diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, Jiasen Wu +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +# mxnet-dataiter + +Here is an example of making a Conduit from MNIST dataset. + +```haskell +mnistIter (add @"image" "data/train-images-idx3-ubyte" $ + add @"label" "data/train-labels-idx1-ubyte" $ + add @"batch_size" 128 + nil) :: ConduitData IO (NDArray Float, NDArray Float) +``` + +The first argument is provides named parameters for the MXNet Data Iterators. Detailed specification can be +found in MXNet API 's python [document](https://mxnet.incubator.apache.org/api/python/io/io.html). + +Below is a snapshot of current support in this package. + +```haskell +type CSVIter_Args = + '[ "data_csv" := String, "data_shape" := [Int], "label_csv" := String, "label_shape" := [Int] + , "batch_size" := Int, "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String] + +type MNISTIter_Args = + '[ "image" := String, "label" := String, "batch_size" := Int, "shuffle" := Bool, "flat" := Bool + , "seed" := Int, "silent" := Bool, "num_parts" := Int, "part_index" := Int + , "prefetch_buffer" := Integer, "dtype" := String] + +type ImageRecordIter_Args = + '[ "path_imglist" := String, "path_imgrec" := String, "path_imgidx" := String, "aug_seq" := String + , "label_width" := Int, "data_shape" := [Int], "preprocess_threads" := Int, "verbose" := Bool + , "num_parts" := Int, "part_index" := Int, "shuffle_chunk_size" := Integer + , "shuffle_chunk_seed" := Int, "shuffle" := Bool, "seed" := Int, "batch_size" := Int + , "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String, "resize" := Int + , "rand_crop" := Bool, "max_rotate_angle" := Int, "max_aspect_ratio" := Float + , "max_shear_ratio" := Float, "max_crop_size" := Int, "min_crop_size" := Int + , "max_random_scale" := Float, "min_random_scale" := Float, "max_img_size" := Float + , "min_img_size" := Float, "random_h" := Int, "random_s" := Int, "random_l" := Int, "rotate" := Int + , "fill_value" := Int, "inter_method" := Int, "pad" := Int, "mirror" := Bool, "rand_mirror" := Bool + , "mean_img" := String, "mean_r" := Float, "mean_g" := Float, "mean_b" := Float, "mean_a" := Float + , "std_r" := Float, "std_g" := Float, "std_b" := Float, "std_a" := Float, "scale" := Float + , "max_random_contrast" := Float, "max_random_illumination" := Float] + +type ImageDetRecordIter_Args = + '[ "path_imglist" := String, "path_imgrec" := String, "aug_seq" := String, "label_width" := Int + , "data_shape" := [Int], "preprocess_threads" := Int, "verbose" := Bool, "num_parts" := Int + , "part_index" := Int, "shuffle_chunk_size" := Integer, "shuffle_chunk_seed" := Int + , "label_pad_width" := Int, "label_pad_value" := Float, "shuffle" := Bool, "seed" := Int + , "batch_size" := Int, "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String + , "resize" := Int, "rand_crop_prob" := Float, "min_crop_scales" := [Float] + , "max_crop_scales" := [Float], "min_crop_aspect_ratios" := [Float] + , "max_crop_aspect_ratios" := [Float], "min_crop_overlaps" := [Float], "max_crop_overlaps" := [Float] + , "min_crop_sample_coverages" := [Float], "max_crop_sample_coverages" := [Float] + , "min_crop_object_coverages" := [Float], "max_crop_object_coverages" := [Float] + , "num_crop_sampler" := Int, "crop_emit_mode" := String, "emit_overlap_thresh" := Float + , "max_crop_trials" := [Int], "rand_pad_prob" := Float, "max_pad_scale" := Float + , "max_random_hue" := Int, "random_hue_prob" := Float, "max_random_saturation" := Int + , "random_saturation_prob" := Float, "max_random_illumination" := Int + , "random_illumination_prob" := Float, "max_random_contrast" := Float, "random_contrast_prob" := Float + , "rand_mirror_prob" := Float, "fill_value" := Int, "inter_method" := Int, "resize_mode" := String + , "mean_img" := String, "mean_r" := Float, "mean_g" := Float, "mean_b" := Float, "mean_a" := Float + , "std_r" := Float, "std_g" := Float, "std_b" := Float, "std_a" := Float, "scale" := Float] + +type ImageRecordUInt8Iter_Args = + '[ "path_imglist" := String, "path_imgrec" := String, "path_imgidx" := String, "aug_seq" := String + , "label_width" := Int, "data_shape" := [Int], "preprocess_threads" := Int, "verbose" := Bool + , "num_parts" := Int, "part_index" := Int, "shuffle_chunk_size" := Integer + , "shuffle_chunk_seed" := Int, "shuffle" := Bool, "seed" := Int, "batch_size" := Int + , "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String, "resize" := Int + , "rand_crop" := Bool, "max_rotate_angle" := Int, "max_aspect_ratio" := Float + , "max_shear_ratio" := Float, "max_crop_size" := Int, "min_crop_size" := Int + , "max_random_scale" := Float, "min_random_scale" := Float, "max_img_size" := Float + , "min_img_size" := Float, "random_h" := Int, "random_s" := Int, "random_l" := Int, "rotate" := Int + , "fill_value" := Int, "inter_method" := Int, "pad" := Int] + +type LibSVMIter_Args = + '[ "data_libsvm" := String, "data_shape" := [Int], "label_libsvm" := String, "label_shape" := [Int] + , "num_parts" := Int, "part_index" := Int, "batch_size" := Int, "round_batch" := Bool + , "prefetch_buffer" := Integer, "dtype" := String] +``` diff --git a/cmdline/Main.hs b/cmdline/Main.hs new file mode 100644 --- /dev/null +++ b/cmdline/Main.hs @@ -0,0 +1,191 @@ +module Main where + +import Options.Applicative +import Data.Semigroup ((<>)) +import Language.Haskell.Exts +import qualified Data.Text as T +import System.Log.Logger +import Control.Monad +import Control.Monad.Writer (Writer, execWriter, tell) +import Data.Either +import Data.Char (toLower, isUpper, isSpace, isAlphaNum) +import Text.Printf (printf) +import Text.ParserCombinators.ReadP +import System.FilePath +import System.Directory + +import MXNet.Base.Raw + +_module_ = "Main" + +data Arguments = Arguments { + output_dir :: FilePath +} + +args_spec = Arguments + <$> strOption (long "output" <> short 'o' <> value "dataiter/src" <> metavar "OUTPUT-DIR") + +main = do + updateGlobalLogger _module_ (setLevel INFO) + args <- execParser opts + let base = output_dir args "MXNet" "NN" "DataIter" + createDirectoryIfMissing True base + + dataitercreators <- mxListDataIters + + infoM _module_ "Generating DataIters..." + dataiters <- concat <$> mapM genDataIter (zip dataitercreators [0..]) + writeFile (base "Raw.hs") $ prettyPrint (modDataIter dataiters) + + where + opts = info (args_spec <**> helper) (fullDesc <> progDesc "Generate MXNet dataiters") + modDataIter = Module () (Just $ ModuleHead () (ModuleName () "MXNet.NN.DataIter.Raw") Nothing Nothing) [] + [ simpleImport "MXNet.Base.Raw" + , simpleImport "MXNet.Base.Spec.Operator" + , simpleImport "MXNet.Base.Spec.HMap" + , simpleImportVars "Data.Maybe" ["catMaybes", "fromMaybe"]] + +simpleImport mod = ImportDecl { + importAnn = (), + importModule = ModuleName () mod, + importQualified = False, + importSrc = False, + importSafe = False, + importPkg = Nothing, + importAs = Nothing, + importSpecs = Nothing +} + +simpleImportVars mod vars = ImportDecl { + importAnn = (), + importModule = ModuleName () mod, + importQualified = False, + importSrc = False, + importSafe = False, + importPkg = Nothing, + importAs = Nothing, + importSpecs = Just $ ImportSpecList () False [IVar () $ Ident () var | var <- vars] +} + +genDataIter :: (DataIterCreator, Integer) -> IO [Decl ()] +genDataIter (dataitercreator, index) = do + (diname, didesc, argnames, argtypes, argdescs) <- mxDataIterGetIterInfo dataitercreator + let diname_ = normalizeName diname + (errs, scalarTypes) = execWriter $ zipWithM_ resolveHaskellType argnames argtypes + + -- parameter list + paramList = map (\(name, typ1, typ2) -> tyPromotedTuple [tyPromotedStr name, tyApp typ1 typ2]) scalarTypes + paramInst = TypeInsDecl () (tyApp (tyCon $ unQual $ name "ParameterList") (tyPromotedStr diname)) + (tyPromotedList paramList) + + -- signature + cxfullfill = appA (name "Fullfilled") [tyPromotedStr diname, tyVarIdent "args"] + tyfun = tyFun (tyApp (tyApp (tyCon $ unQual $ name "ArgsHMap") (tyPromotedStr diname)) (tyVarIdent "args")) + (tyApp (tyCon $ unQual $ name "IO") (tyCon $ unQual $ name "DataIterHandle")) + tysig = tySig [name diname_] $ tyForall [unkindedVar (name "args")] (cxSingle cxfullfill) tyfun + + -- function + fun = sfun (name diname_) [name "args"] (UnGuardedRhs () body) Nothing + body = letE ([ + patBind (pvar $ name "allargs") (function "catMaybes" + `app` listE [ + infixApp (infixApp (tupleSection [Just $ strE argkey, Nothing]) (op $ sym ".") (function "showValue")) (op $ sym "<$>") $ + ExpTypeSig () (infixApp (var $ name "args") (op $ sym "!?") (OverloadedLabel () argkey)) (tyApp (tyCon $ unQual $ name "Maybe") typ) | (argkey, _, typ) <- scalarTypes]) + , patBind (pTuple [pvar $ name "keys", pvar $ name "vals"]) (app (function "unzip") $ var $ name "allargs") + ]) (doE $ [ + genStmt (pvar $ name "dis") $ function "mxListDataIters", + genStmt (pvar $ name "di") $ function "return" `app` (infixApp (var $ name "dis") (op $ sym "!!") (intE index)), + qualStmt $ function "mxDataIterCreateIter" `app` (var $ name "di") `app` (var $ name "keys") `app` (var $ name "vals") + ]) + + return [paramInst, tysig, fun] + + + where + normalizeName :: String -> String + normalizeName name@(c:cs) + | isUpper c = '_' : name + | otherwise = name + +data ParamDesc = ParamDescItem String | ParamDescList Bool [String] deriving (Eq, Show) + +type ResolvedType = (String, Type (), Type ()) +resolveHaskellType :: String -> String -> Writer ([(String, String)], [ResolvedType]) () +resolveHaskellType argname desc = + case head fields of + ParamDescItem "Shape(tuple)" -> scalar $ tyList $ tyCon $ unQual $ name "Int" + ParamDescItem "int" -> scalar $ tyCon $ unQual $ name "Int" + ParamDescItem "int (non-negative)" -> scalar $ tyCon $ unQual $ name "Int" + ParamDescItem "long (non-negative)" -> scalar $ tyCon $ unQual $ name "Int" + ParamDescItem "boolean" -> scalar $ tyCon $ unQual $ name "Bool" + ParamDescItem "float" -> scalar $ tyCon $ unQual $ name "Float" + ParamDescItem "double" -> scalar $ tyCon $ unQual $ name "Double" + ParamDescItem "float32" -> scalar $ tyCon $ unQual $ name "Float" + ParamDescItem "string" -> scalar $ tyCon $ unQual $ name "String" + ParamDescItem "int or None" -> scalar $ tyApp (tyCon $ unQual $ name "Maybe") (tyCon $ unQual $ name "Int") + ParamDescItem "double or None" -> scalar $ tyApp (tyCon $ unQual $ name "Maybe") (tyCon $ unQual $ name "Double") + ParamDescList hasnone vs -> do + let vsprom = map tyPromotedStr vs + typ1 = tyApp (tyCon $ unQual $ name "EnumType") (tyPromotedList vsprom) + typ2 = tyApp (tyCon $ unQual $ name "Maybe") typ1 + scalar $ if hasnone then typ2 else typ1 + + t -> fail $ printf "unsupported arg: %s" (show t) + where + typedesc = do + ds <- sepBy (skipSpaces >> (list1 +++ list2 +++ item)) (char ',') + eof + return ds + list1 = ParamDescList True <$> between (string "{None,") (char '}') (sepBy (skipSpaces >> listItem) (char ',')) + list2 = ParamDescList False <$> between (string "{") (char '}') (sepBy (skipSpaces >> listItem) (char ',')) + listItem = between (char '\'') (char '\'') (munch1 (\c -> isAlphaNum c || c `elem` "_")) + item = ParamDescItem <$> munch1 (\c -> isAlphaNum c || c `elem` " _-()=[]<>'./+") + runP str = case readP_to_S typedesc str of + [(xs, "")] -> xs + other -> error ("cannot parse type description: " ++ str) + + fields = runP desc + required = ParamDescItem "required" `elem` fields + attr = tyCon $ unQual $ name $ if required then "AttrReq" else "AttrOpt" + scalar hstyp = tell ([], [(argname, attr, hstyp)]) + fail msg = tell ([(argname, msg)], []) + +makeParamInst :: String -> [ResolvedType] -> Bool -> Decl () +makeParamInst symname typs symbolapi = + TypeInsDecl () (tyApp (tyCon $ unQual $ name "ParameterList") (tyPromotedStr symname_with_appendix)) + (tyPromotedList paramList) + where + symname_with_appendix = symname ++ (if symbolapi then "(symbol)" else "(ndarray)") + paramList = map (\(name, typ1, typ2) -> tyPromotedTuple [tyPromotedStr name, tyApp typ1 typ2]) typs + + +unQual = UnQual () +unkindedVar = UnkindedVar () + +tyCon = TyCon () +tyVarSymbol = TyVar () . Symbol () +tyVarIdent = TyVar () . Ident () +tyApp = TyApp () +tyFun = TyFun () +tySig names types = TypeSig () names types +tyList = TyList () +tyVar = TyVar () + +tyPromotedInteger s = TyPromoted () (PromotedInteger () s (show s)) +tyPromotedStr s = TyPromoted () (PromotedString () s s) +tyPromotedList s = TyPromoted () (PromotedList () True s) +tyPromotedTuple s = TyPromoted () (PromotedTuple () s) + +tyForall vars cxt typ = TyForall () vars_ cxt_ typ + where + vars_ = if null vars then Nothing else Just vars + cxt_ = if cxt == CxEmpty () then Nothing else Just cxt + +cxSingle = CxSingle () +cxTuple = CxTuple () + +appA = AppA () + +tupleSection = TupleSection () Boxed + +con = Con () diff --git a/dataiter/src/MXNet/NN/DataIter/Conduit.hs b/dataiter/src/MXNet/NN/DataIter/Conduit.hs new file mode 100644 --- /dev/null +++ b/dataiter/src/MXNet/NN/DataIter/Conduit.hs @@ -0,0 +1,77 @@ +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +module MXNet.NN.DataIter.Conduit ( + ConduitData(..), + Dataset(..), + imageRecordIter, mnistIter, csvIter, libSVMIter +) where + +import Data.Conduit +import qualified Data.Conduit.Combinators as C +import qualified Data.Conduit.List as CL +import Control.Applicative +import Control.Monad.IO.Class + +import MXNet.Base +import qualified MXNet.NN.DataIter.Raw as I +import MXNet.NN.DataIter.Class + +data ConduitData m a = ConduitData { + iter_batch_size :: Maybe Int, + getConduit :: ConduitM () a m () +} + +imageRecordIter :: (Fullfilled "ImageRecordIter" args, DType a, MonadIO m) + => ArgsHMap "ImageRecordIter" args -> ConduitData m (NDArray a, NDArray a) +imageRecordIter args = ConduitData { + getConduit = makeIter I._ImageRecordIter args, + iter_batch_size = Just (args ! #batch_size) +} + +mnistIter :: (Fullfilled "MNISTIter" args, DType a, MonadIO m) + => ArgsHMap "MNISTIter" args -> ConduitData m (NDArray a, NDArray a) +mnistIter args = ConduitData { + getConduit = makeIter I._MNISTIter args, + iter_batch_size = (args !? #batch_size) <|> Just 1 +} + +csvIter :: (Fullfilled "CSVIter" args, DType a, MonadIO m) + => ArgsHMap "CSVIter" args -> ConduitData m (NDArray a, NDArray a) +csvIter args = ConduitData { + getConduit = makeIter I._CSVIter args, + iter_batch_size = Just (args ! #batch_size) +} + +libSVMIter :: (Fullfilled "LibSVMIter" args, DType a, MonadIO m) + => ArgsHMap "LibSVMIter" args -> ConduitData m (NDArray a, NDArray a) +libSVMIter args = ConduitData { + getConduit = makeIter I._LibSVMIter args, + iter_batch_size = Just (args ! #batch_size) +} + +makeIter creator args = do + iter <- liftIO (creator args) + let loop = do valid <- liftIO $ mxDataIterNext iter + if valid == 0 + then liftIO (finalizeDataIterHandle iter) + else do + yieldM $ liftIO $ do + dat <- mxDataIterGetData iter + lbl <- mxDataIterGetLabel iter + return (NDArray dat, NDArray lbl) + loop + loop + +type instance DatasetConstraint (ConduitData m1) m2 = m1 ~ m2 + +instance Monad m => Dataset (ConduitData m) where + fromListD = ConduitData Nothing . CL.sourceList + zipD d1 d2 = ConduitData Nothing $ getZipSource $ (,) <$> ZipSource (getConduit d1) <*> ZipSource (getConduit d2) + sizeD d = runConduit (getConduit d .| C.length) + forEachD d proc = sourceToList $ getConduit d .| CL.mapM proc + foldD proc elem d = runConduit (getConduit d .| C.foldM proc elem) + takeD n d = d {getConduit = getConduit d .| C.take n} + +instance DatasetProp (ConduitData m) a where + batchSizeD = return . iter_batch_size diff --git a/dataiter/src/MXNet/NN/DataIter/Raw.hs b/dataiter/src/MXNet/NN/DataIter/Raw.hs new file mode 100644 --- /dev/null +++ b/dataiter/src/MXNet/NN/DataIter/Raw.hs @@ -0,0 +1,741 @@ +module MXNet.NN.DataIter.Raw where +import MXNet.Base.Raw +import MXNet.Base.Spec.Operator +import MXNet.Base.Spec.HMap +import Data.Maybe (catMaybes, fromMaybe) + +type instance ParameterList "CSVIter" = + '[ '("data_csv", AttrReq String), '("data_shape", AttrReq [Int]), + '("label_csv", AttrOpt String), '("label_shape", AttrOpt [Int]), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"])))] + +_CSVIter :: + forall args . Fullfilled "CSVIter" args => + ArgsHMap "CSVIter" args -> IO DataIterHandle +_CSVIter args + = let allargs + = catMaybes + [("data_csv",) . showValue <$> (args !? #data_csv :: Maybe String), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("label_csv",) . showValue <$> + (args !? #label_csv :: Maybe String), + ("label_shape",) . showValue <$> + (args !? #label_shape :: Maybe [Int]), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"])))] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 0) + mxDataIterCreateIter di keys vals + +type instance ParameterList "ImageDetRecordIter" = + '[ '("path_imglist", AttrOpt String), + '("path_imgrec", AttrOpt String), '("aug_seq", AttrOpt String), + '("label_width", AttrOpt Int), '("data_shape", AttrReq [Int]), + '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool), + '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int), + '("shuffle_chunk_size", AttrOpt Int), + '("shuffle_chunk_seed", AttrOpt Int), + '("label_pad_width", AttrOpt Int), + '("label_pad_value", AttrOpt Float), '("shuffle", AttrOpt Bool), + '("seed", AttrOpt Int), '("verbose", AttrOpt Bool), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + '("resize", AttrOpt Int), '("rand_crop_prob", AttrOpt Float), + '("num_crop_sampler", AttrOpt Int), + '("crop_emit_mode", AttrOpt (EnumType '["center", "overlap"])), + '("emit_overlap_thresh", AttrOpt Float), + '("max_crop_trials", AttrOpt [Int]), + '("rand_pad_prob", AttrOpt Float), + '("max_pad_scale", AttrOpt Float), + '("max_random_hue", AttrOpt Int), + '("random_hue_prob", AttrOpt Float), + '("max_random_saturation", AttrOpt Int), + '("random_saturation_prob", AttrOpt Float), + '("max_random_illumination", AttrOpt Int), + '("random_illumination_prob", AttrOpt Float), + '("max_random_contrast", AttrOpt Float), + '("random_contrast_prob", AttrOpt Float), + '("rand_mirror_prob", AttrOpt Float), '("fill_value", AttrOpt Int), + '("inter_method", AttrOpt Int), '("data_shape", AttrReq [Int]), + '("resize_mode", AttrOpt (EnumType '["fit", "force", "shrink"])), + '("seed", AttrOpt Int), '("mean_img", AttrOpt String), + '("mean_r", AttrOpt Float), '("mean_g", AttrOpt Float), + '("mean_b", AttrOpt Float), '("mean_a", AttrOpt Float), + '("std_r", AttrOpt Float), '("std_g", AttrOpt Float), + '("std_b", AttrOpt Float), '("std_a", AttrOpt Float), + '("scale", AttrOpt Float), '("verbose", AttrOpt Bool)] + +_ImageDetRecordIter :: + forall args . Fullfilled "ImageDetRecordIter" args => + ArgsHMap "ImageDetRecordIter" args -> IO DataIterHandle +_ImageDetRecordIter args + = let allargs + = catMaybes + [("path_imglist",) . showValue <$> + (args !? #path_imglist :: Maybe String), + ("path_imgrec",) . showValue <$> + (args !? #path_imgrec :: Maybe String), + ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String), + ("label_width",) . showValue <$> + (args !? #label_width :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("preprocess_threads",) . showValue <$> + (args !? #preprocess_threads :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("shuffle_chunk_size",) . showValue <$> + (args !? #shuffle_chunk_size :: Maybe Int), + ("shuffle_chunk_seed",) . showValue <$> + (args !? #shuffle_chunk_seed :: Maybe Int), + ("label_pad_width",) . showValue <$> + (args !? #label_pad_width :: Maybe Int), + ("label_pad_value",) . showValue <$> + (args !? #label_pad_value :: Maybe Float), + ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + ("resize",) . showValue <$> (args !? #resize :: Maybe Int), + ("rand_crop_prob",) . showValue <$> + (args !? #rand_crop_prob :: Maybe Float), + ("num_crop_sampler",) . showValue <$> + (args !? #num_crop_sampler :: Maybe Int), + ("crop_emit_mode",) . showValue <$> + (args !? #crop_emit_mode :: + Maybe (EnumType '["center", "overlap"])), + ("emit_overlap_thresh",) . showValue <$> + (args !? #emit_overlap_thresh :: Maybe Float), + ("max_crop_trials",) . showValue <$> + (args !? #max_crop_trials :: Maybe [Int]), + ("rand_pad_prob",) . showValue <$> + (args !? #rand_pad_prob :: Maybe Float), + ("max_pad_scale",) . showValue <$> + (args !? #max_pad_scale :: Maybe Float), + ("max_random_hue",) . showValue <$> + (args !? #max_random_hue :: Maybe Int), + ("random_hue_prob",) . showValue <$> + (args !? #random_hue_prob :: Maybe Float), + ("max_random_saturation",) . showValue <$> + (args !? #max_random_saturation :: Maybe Int), + ("random_saturation_prob",) . showValue <$> + (args !? #random_saturation_prob :: Maybe Float), + ("max_random_illumination",) . showValue <$> + (args !? #max_random_illumination :: Maybe Int), + ("random_illumination_prob",) . showValue <$> + (args !? #random_illumination_prob :: Maybe Float), + ("max_random_contrast",) . showValue <$> + (args !? #max_random_contrast :: Maybe Float), + ("random_contrast_prob",) . showValue <$> + (args !? #random_contrast_prob :: Maybe Float), + ("rand_mirror_prob",) . showValue <$> + (args !? #rand_mirror_prob :: Maybe Float), + ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int), + ("inter_method",) . showValue <$> + (args !? #inter_method :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("resize_mode",) . showValue <$> + (args !? #resize_mode :: + Maybe (EnumType '["fit", "force", "shrink"])), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("mean_img",) . showValue <$> (args !? #mean_img :: Maybe String), + ("mean_r",) . showValue <$> (args !? #mean_r :: Maybe Float), + ("mean_g",) . showValue <$> (args !? #mean_g :: Maybe Float), + ("mean_b",) . showValue <$> (args !? #mean_b :: Maybe Float), + ("mean_a",) . showValue <$> (args !? #mean_a :: Maybe Float), + ("std_r",) . showValue <$> (args !? #std_r :: Maybe Float), + ("std_g",) . showValue <$> (args !? #std_g :: Maybe Float), + ("std_b",) . showValue <$> (args !? #std_b :: Maybe Float), + ("std_a",) . showValue <$> (args !? #std_a :: Maybe Float), + ("scale",) . showValue <$> (args !? #scale :: Maybe Float), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool)] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 1) + mxDataIterCreateIter di keys vals + +type instance ParameterList "ImageRecordIter_v1" = + '[ '("path_imglist", AttrOpt String), + '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String), + '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int), + '("data_shape", AttrReq [Int]), + '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool), + '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int), + '("shuffle_chunk_size", AttrOpt Int), + '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool), + '("seed", AttrOpt Int), '("verbose", AttrOpt Bool), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool), + '("max_rotate_angle", AttrOpt Int), + '("max_aspect_ratio", AttrOpt Float), + '("max_shear_ratio", AttrOpt Float), + '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int), + '("max_random_scale", AttrOpt Float), + '("min_random_scale", AttrOpt Float), + '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float), + '("random_h", AttrOpt Int), '("random_s", AttrOpt Int), + '("random_l", AttrOpt Int), '("rotate", AttrOpt Int), + '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]), + '("inter_method", AttrOpt Int), '("pad", AttrOpt Int), + '("seed", AttrOpt Int), '("mirror", AttrOpt Bool), + '("rand_mirror", AttrOpt Bool), '("mean_img", AttrOpt String), + '("mean_r", AttrOpt Float), '("mean_g", AttrOpt Float), + '("mean_b", AttrOpt Float), '("mean_a", AttrOpt Float), + '("std_r", AttrOpt Float), '("std_g", AttrOpt Float), + '("std_b", AttrOpt Float), '("std_a", AttrOpt Float), + '("scale", AttrOpt Float), '("max_random_contrast", AttrOpt Float), + '("max_random_illumination", AttrOpt Float), + '("verbose", AttrOpt Bool)] + +_ImageRecordIter_v1 :: + forall args . Fullfilled "ImageRecordIter_v1" args => + ArgsHMap "ImageRecordIter_v1" args -> IO DataIterHandle +_ImageRecordIter_v1 args + = let allargs + = catMaybes + [("path_imglist",) . showValue <$> + (args !? #path_imglist :: Maybe String), + ("path_imgrec",) . showValue <$> + (args !? #path_imgrec :: Maybe String), + ("path_imgidx",) . showValue <$> + (args !? #path_imgidx :: Maybe String), + ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String), + ("label_width",) . showValue <$> + (args !? #label_width :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("preprocess_threads",) . showValue <$> + (args !? #preprocess_threads :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("shuffle_chunk_size",) . showValue <$> + (args !? #shuffle_chunk_size :: Maybe Int), + ("shuffle_chunk_seed",) . showValue <$> + (args !? #shuffle_chunk_seed :: Maybe Int), + ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + ("resize",) . showValue <$> (args !? #resize :: Maybe Int), + ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool), + ("max_rotate_angle",) . showValue <$> + (args !? #max_rotate_angle :: Maybe Int), + ("max_aspect_ratio",) . showValue <$> + (args !? #max_aspect_ratio :: Maybe Float), + ("max_shear_ratio",) . showValue <$> + (args !? #max_shear_ratio :: Maybe Float), + ("max_crop_size",) . showValue <$> + (args !? #max_crop_size :: Maybe Int), + ("min_crop_size",) . showValue <$> + (args !? #min_crop_size :: Maybe Int), + ("max_random_scale",) . showValue <$> + (args !? #max_random_scale :: Maybe Float), + ("min_random_scale",) . showValue <$> + (args !? #min_random_scale :: Maybe Float), + ("max_img_size",) . showValue <$> + (args !? #max_img_size :: Maybe Float), + ("min_img_size",) . showValue <$> + (args !? #min_img_size :: Maybe Float), + ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int), + ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int), + ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int), + ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int), + ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("inter_method",) . showValue <$> + (args !? #inter_method :: Maybe Int), + ("pad",) . showValue <$> (args !? #pad :: Maybe Int), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("mirror",) . showValue <$> (args !? #mirror :: Maybe Bool), + ("rand_mirror",) . showValue <$> + (args !? #rand_mirror :: Maybe Bool), + ("mean_img",) . showValue <$> (args !? #mean_img :: Maybe String), + ("mean_r",) . showValue <$> (args !? #mean_r :: Maybe Float), + ("mean_g",) . showValue <$> (args !? #mean_g :: Maybe Float), + ("mean_b",) . showValue <$> (args !? #mean_b :: Maybe Float), + ("mean_a",) . showValue <$> (args !? #mean_a :: Maybe Float), + ("std_r",) . showValue <$> (args !? #std_r :: Maybe Float), + ("std_g",) . showValue <$> (args !? #std_g :: Maybe Float), + ("std_b",) . showValue <$> (args !? #std_b :: Maybe Float), + ("std_a",) . showValue <$> (args !? #std_a :: Maybe Float), + ("scale",) . showValue <$> (args !? #scale :: Maybe Float), + ("max_random_contrast",) . showValue <$> + (args !? #max_random_contrast :: Maybe Float), + ("max_random_illumination",) . showValue <$> + (args !? #max_random_illumination :: Maybe Float), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool)] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 2) + mxDataIterCreateIter di keys vals + +type instance ParameterList "ImageRecordUInt8Iter_v1" = + '[ '("path_imglist", AttrOpt String), + '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String), + '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int), + '("data_shape", AttrReq [Int]), + '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool), + '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int), + '("shuffle_chunk_size", AttrOpt Int), + '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool), + '("seed", AttrOpt Int), '("verbose", AttrOpt Bool), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool), + '("max_rotate_angle", AttrOpt Int), + '("max_aspect_ratio", AttrOpt Float), + '("max_shear_ratio", AttrOpt Float), + '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int), + '("max_random_scale", AttrOpt Float), + '("min_random_scale", AttrOpt Float), + '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float), + '("random_h", AttrOpt Int), '("random_s", AttrOpt Int), + '("random_l", AttrOpt Int), '("rotate", AttrOpt Int), + '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]), + '("inter_method", AttrOpt Int), '("pad", AttrOpt Int)] + +_ImageRecordUInt8Iter_v1 :: + forall args . Fullfilled "ImageRecordUInt8Iter_v1" args => + ArgsHMap "ImageRecordUInt8Iter_v1" args -> IO DataIterHandle +_ImageRecordUInt8Iter_v1 args + = let allargs + = catMaybes + [("path_imglist",) . showValue <$> + (args !? #path_imglist :: Maybe String), + ("path_imgrec",) . showValue <$> + (args !? #path_imgrec :: Maybe String), + ("path_imgidx",) . showValue <$> + (args !? #path_imgidx :: Maybe String), + ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String), + ("label_width",) . showValue <$> + (args !? #label_width :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("preprocess_threads",) . showValue <$> + (args !? #preprocess_threads :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("shuffle_chunk_size",) . showValue <$> + (args !? #shuffle_chunk_size :: Maybe Int), + ("shuffle_chunk_seed",) . showValue <$> + (args !? #shuffle_chunk_seed :: Maybe Int), + ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + ("resize",) . showValue <$> (args !? #resize :: Maybe Int), + ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool), + ("max_rotate_angle",) . showValue <$> + (args !? #max_rotate_angle :: Maybe Int), + ("max_aspect_ratio",) . showValue <$> + (args !? #max_aspect_ratio :: Maybe Float), + ("max_shear_ratio",) . showValue <$> + (args !? #max_shear_ratio :: Maybe Float), + ("max_crop_size",) . showValue <$> + (args !? #max_crop_size :: Maybe Int), + ("min_crop_size",) . showValue <$> + (args !? #min_crop_size :: Maybe Int), + ("max_random_scale",) . showValue <$> + (args !? #max_random_scale :: Maybe Float), + ("min_random_scale",) . showValue <$> + (args !? #min_random_scale :: Maybe Float), + ("max_img_size",) . showValue <$> + (args !? #max_img_size :: Maybe Float), + ("min_img_size",) . showValue <$> + (args !? #min_img_size :: Maybe Float), + ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int), + ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int), + ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int), + ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int), + ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("inter_method",) . showValue <$> + (args !? #inter_method :: Maybe Int), + ("pad",) . showValue <$> (args !? #pad :: Maybe Int)] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 3) + mxDataIterCreateIter di keys vals + +type instance ParameterList "MNISTIter" = + '[ '("image", AttrOpt String), '("label", AttrOpt String), + '("batch_size", AttrOpt Int), '("shuffle", AttrOpt Bool), + '("flat", AttrOpt Bool), '("seed", AttrOpt Int), + '("silent", AttrOpt Bool), '("num_parts", AttrOpt Int), + '("part_index", AttrOpt Int), '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"])))] + +_MNISTIter :: + forall args . Fullfilled "MNISTIter" args => + ArgsHMap "MNISTIter" args -> IO DataIterHandle +_MNISTIter args + = let allargs + = catMaybes + [("image",) . showValue <$> (args !? #image :: Maybe String), + ("label",) . showValue <$> (args !? #label :: Maybe String), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool), + ("flat",) . showValue <$> (args !? #flat :: Maybe Bool), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("silent",) . showValue <$> (args !? #silent :: Maybe Bool), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"])))] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 4) + mxDataIterCreateIter di keys vals + +type instance ParameterList "ImageRecordIter" = + '[ '("path_imglist", AttrOpt String), + '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String), + '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int), + '("data_shape", AttrReq [Int]), + '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool), + '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int), + '("shuffle_chunk_size", AttrOpt Int), + '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool), + '("seed", AttrOpt Int), '("verbose", AttrOpt Bool), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool), + '("max_rotate_angle", AttrOpt Int), + '("max_aspect_ratio", AttrOpt Float), + '("max_shear_ratio", AttrOpt Float), + '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int), + '("max_random_scale", AttrOpt Float), + '("min_random_scale", AttrOpt Float), + '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float), + '("random_h", AttrOpt Int), '("random_s", AttrOpt Int), + '("random_l", AttrOpt Int), '("rotate", AttrOpt Int), + '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]), + '("inter_method", AttrOpt Int), '("pad", AttrOpt Int), + '("seed", AttrOpt Int), '("mirror", AttrOpt Bool), + '("rand_mirror", AttrOpt Bool), '("mean_img", AttrOpt String), + '("mean_r", AttrOpt Float), '("mean_g", AttrOpt Float), + '("mean_b", AttrOpt Float), '("mean_a", AttrOpt Float), + '("std_r", AttrOpt Float), '("std_g", AttrOpt Float), + '("std_b", AttrOpt Float), '("std_a", AttrOpt Float), + '("scale", AttrOpt Float), '("max_random_contrast", AttrOpt Float), + '("max_random_illumination", AttrOpt Float), + '("verbose", AttrOpt Bool)] + +_ImageRecordIter :: + forall args . Fullfilled "ImageRecordIter" args => + ArgsHMap "ImageRecordIter" args -> IO DataIterHandle +_ImageRecordIter args + = let allargs + = catMaybes + [("path_imglist",) . showValue <$> + (args !? #path_imglist :: Maybe String), + ("path_imgrec",) . showValue <$> + (args !? #path_imgrec :: Maybe String), + ("path_imgidx",) . showValue <$> + (args !? #path_imgidx :: Maybe String), + ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String), + ("label_width",) . showValue <$> + (args !? #label_width :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("preprocess_threads",) . showValue <$> + (args !? #preprocess_threads :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("shuffle_chunk_size",) . showValue <$> + (args !? #shuffle_chunk_size :: Maybe Int), + ("shuffle_chunk_seed",) . showValue <$> + (args !? #shuffle_chunk_seed :: Maybe Int), + ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + ("resize",) . showValue <$> (args !? #resize :: Maybe Int), + ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool), + ("max_rotate_angle",) . showValue <$> + (args !? #max_rotate_angle :: Maybe Int), + ("max_aspect_ratio",) . showValue <$> + (args !? #max_aspect_ratio :: Maybe Float), + ("max_shear_ratio",) . showValue <$> + (args !? #max_shear_ratio :: Maybe Float), + ("max_crop_size",) . showValue <$> + (args !? #max_crop_size :: Maybe Int), + ("min_crop_size",) . showValue <$> + (args !? #min_crop_size :: Maybe Int), + ("max_random_scale",) . showValue <$> + (args !? #max_random_scale :: Maybe Float), + ("min_random_scale",) . showValue <$> + (args !? #min_random_scale :: Maybe Float), + ("max_img_size",) . showValue <$> + (args !? #max_img_size :: Maybe Float), + ("min_img_size",) . showValue <$> + (args !? #min_img_size :: Maybe Float), + ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int), + ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int), + ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int), + ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int), + ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("inter_method",) . showValue <$> + (args !? #inter_method :: Maybe Int), + ("pad",) . showValue <$> (args !? #pad :: Maybe Int), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("mirror",) . showValue <$> (args !? #mirror :: Maybe Bool), + ("rand_mirror",) . showValue <$> + (args !? #rand_mirror :: Maybe Bool), + ("mean_img",) . showValue <$> (args !? #mean_img :: Maybe String), + ("mean_r",) . showValue <$> (args !? #mean_r :: Maybe Float), + ("mean_g",) . showValue <$> (args !? #mean_g :: Maybe Float), + ("mean_b",) . showValue <$> (args !? #mean_b :: Maybe Float), + ("mean_a",) . showValue <$> (args !? #mean_a :: Maybe Float), + ("std_r",) . showValue <$> (args !? #std_r :: Maybe Float), + ("std_g",) . showValue <$> (args !? #std_g :: Maybe Float), + ("std_b",) . showValue <$> (args !? #std_b :: Maybe Float), + ("std_a",) . showValue <$> (args !? #std_a :: Maybe Float), + ("scale",) . showValue <$> (args !? #scale :: Maybe Float), + ("max_random_contrast",) . showValue <$> + (args !? #max_random_contrast :: Maybe Float), + ("max_random_illumination",) . showValue <$> + (args !? #max_random_illumination :: Maybe Float), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool)] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 5) + mxDataIterCreateIter di keys vals + +type instance ParameterList "ImageRecordUInt8Iter" = + '[ '("path_imglist", AttrOpt String), + '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String), + '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int), + '("data_shape", AttrReq [Int]), + '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool), + '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int), + '("shuffle_chunk_size", AttrOpt Int), + '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool), + '("seed", AttrOpt Int), '("verbose", AttrOpt Bool), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool), + '("max_rotate_angle", AttrOpt Int), + '("max_aspect_ratio", AttrOpt Float), + '("max_shear_ratio", AttrOpt Float), + '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int), + '("max_random_scale", AttrOpt Float), + '("min_random_scale", AttrOpt Float), + '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float), + '("random_h", AttrOpt Int), '("random_s", AttrOpt Int), + '("random_l", AttrOpt Int), '("rotate", AttrOpt Int), + '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]), + '("inter_method", AttrOpt Int), '("pad", AttrOpt Int)] + +_ImageRecordUInt8Iter :: + forall args . Fullfilled "ImageRecordUInt8Iter" args => + ArgsHMap "ImageRecordUInt8Iter" args -> IO DataIterHandle +_ImageRecordUInt8Iter args + = let allargs + = catMaybes + [("path_imglist",) . showValue <$> + (args !? #path_imglist :: Maybe String), + ("path_imgrec",) . showValue <$> + (args !? #path_imgrec :: Maybe String), + ("path_imgidx",) . showValue <$> + (args !? #path_imgidx :: Maybe String), + ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String), + ("label_width",) . showValue <$> + (args !? #label_width :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("preprocess_threads",) . showValue <$> + (args !? #preprocess_threads :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("shuffle_chunk_size",) . showValue <$> + (args !? #shuffle_chunk_size :: Maybe Int), + ("shuffle_chunk_seed",) . showValue <$> + (args !? #shuffle_chunk_seed :: Maybe Int), + ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool), + ("seed",) . showValue <$> (args !? #seed :: Maybe Int), + ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"]))), + ("resize",) . showValue <$> (args !? #resize :: Maybe Int), + ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool), + ("max_rotate_angle",) . showValue <$> + (args !? #max_rotate_angle :: Maybe Int), + ("max_aspect_ratio",) . showValue <$> + (args !? #max_aspect_ratio :: Maybe Float), + ("max_shear_ratio",) . showValue <$> + (args !? #max_shear_ratio :: Maybe Float), + ("max_crop_size",) . showValue <$> + (args !? #max_crop_size :: Maybe Int), + ("min_crop_size",) . showValue <$> + (args !? #min_crop_size :: Maybe Int), + ("max_random_scale",) . showValue <$> + (args !? #max_random_scale :: Maybe Float), + ("min_random_scale",) . showValue <$> + (args !? #min_random_scale :: Maybe Float), + ("max_img_size",) . showValue <$> + (args !? #max_img_size :: Maybe Float), + ("min_img_size",) . showValue <$> + (args !? #min_img_size :: Maybe Float), + ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int), + ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int), + ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int), + ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int), + ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("inter_method",) . showValue <$> + (args !? #inter_method :: Maybe Int), + ("pad",) . showValue <$> (args !? #pad :: Maybe Int)] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 6) + mxDataIterCreateIter di keys vals + +type instance ParameterList "LibSVMIter" = + '[ '("data_libsvm", AttrReq String), '("data_shape", AttrReq [Int]), + '("label_libsvm", AttrOpt String), '("label_shape", AttrOpt [Int]), + '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int), + '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool), + '("prefetch_buffer", AttrOpt Int), + '("dtype", + AttrOpt + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"])))] + +_LibSVMIter :: + forall args . Fullfilled "LibSVMIter" args => + ArgsHMap "LibSVMIter" args -> IO DataIterHandle +_LibSVMIter args + = let allargs + = catMaybes + [("data_libsvm",) . showValue <$> + (args !? #data_libsvm :: Maybe String), + ("data_shape",) . showValue <$> + (args !? #data_shape :: Maybe [Int]), + ("label_libsvm",) . showValue <$> + (args !? #label_libsvm :: Maybe String), + ("label_shape",) . showValue <$> + (args !? #label_shape :: Maybe [Int]), + ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int), + ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int), + ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int), + ("round_batch",) . showValue <$> + (args !? #round_batch :: Maybe Bool), + ("prefetch_buffer",) . showValue <$> + (args !? #prefetch_buffer :: Maybe Int), + ("dtype",) . showValue <$> + (args !? #dtype :: + Maybe + (Maybe + (EnumType '["float16", "float32", "float64", "int32", "uint8"])))] + (keys, vals) = unzip allargs + in + do dis <- mxListDataIters + di <- return (dis !! 7) + mxDataIterCreateIter di keys vals diff --git a/dataiter/src/MXNet/NN/DataIter/Streaming.hs b/dataiter/src/MXNet/NN/DataIter/Streaming.hs new file mode 100644 --- /dev/null +++ b/dataiter/src/MXNet/NN/DataIter/Streaming.hs @@ -0,0 +1,78 @@ +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +module MXNet.NN.DataIter.Streaming ( + StreamData(..), + Dataset(..), + imageRecordIter, mnistIter, csvIter, libSVMIter +) where + +import Streaming +import Streaming.Prelude (Of(..), yield, length_, toList_) +import qualified Streaming.Prelude as S + +import MXNet.Base +import qualified MXNet.NN.DataIter.Raw as I +import MXNet.NN.DataIter.Class + +data StreamData m a = StreamData { + iter_batch_size :: Maybe Int, + getStream :: Stream (Of a) m () +} + +imageRecordIter :: (Fullfilled "ImageRecordIter" args, DType a, MonadIO m) + => ArgsHMap "ImageRecordIter" args -> StreamData m (NDArray a, NDArray a) +imageRecordIter args = StreamData { + getStream = makeIter I._ImageRecordIter args, + iter_batch_size = Just (args ! #batch_size) +} + +mnistIter :: (Fullfilled "MNISTIter" args, DType a, MonadIO m) + => ArgsHMap "MNISTIter" args -> StreamData m (NDArray a, NDArray a) +mnistIter args = StreamData { + getStream = makeIter I._MNISTIter args, + iter_batch_size = (args !? #batch_size) <|> Just 1 +} + +csvIter :: (Fullfilled "CSVIter" args, DType a, MonadIO m) + => ArgsHMap "CSVIter" args -> StreamData m (NDArray a, NDArray a) +csvIter args = StreamData { + getStream = makeIter I._CSVIter args, + iter_batch_size = Just (args ! #batch_size) +} + +libSVMIter :: (Fullfilled "LibSVMIter" args, DType a, MonadIO m) + => ArgsHMap "LibSVMIter" args -> StreamData m (NDArray a, NDArray a) +libSVMIter args = StreamData { + getStream = makeIter I._LibSVMIter args, + iter_batch_size = Just (args ! #batch_size) +} + + +makeIter creator args = do + iter <- liftIO (creator args) + let loop = do valid <- liftIO $ mxDataIterNext iter + if valid == 0 + then liftIO (finalizeDataIterHandle iter) + else do + item <- liftIO $ do + dat <- mxDataIterGetData iter + lbl <- mxDataIterGetLabel iter + return (NDArray dat, NDArray lbl) + yield item + loop + loop + +type instance DatasetConstraint (StreamData m1) m2 = m1 ~ m2 + +instance Monad m => Dataset (StreamData m) where + fromListD = StreamData Nothing . S.each + zipD s1 s2 = StreamData Nothing $ S.zip (getStream s1) (getStream s2) + sizeD = length_ . getStream + forEachD dat proc = toList_ $ void $ S.mapM proc (getStream dat) + foldD proc elem dat = S.foldM_ proc (return elem) return (getStream dat) + takeD n dat = dat { getStream = S.take n (getStream dat) } + +instance DatasetProp (StreamData m) a where + batchSizeD = return . iter_batch_size + diff --git a/dataiter/test/conduit.hs b/dataiter/test/conduit.hs new file mode 100644 --- /dev/null +++ b/dataiter/test/conduit.hs @@ -0,0 +1,38 @@ +module Main where + +import Test.Hspec + +import MXNet.Base +import MXNet.NN.DataIter.Conduit + +type DS = ConduitData IO (NDArray Float, NDArray Float) + +main :: IO () +main = hspec $ do + describe "MNISTIter" $ do + it "batch-size = 1" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 1 .& Nil) + sizeD sr `shouldReturn` 60000 + it "batch-size = 32" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 32 .& Nil) + sizeD sr `shouldReturn` 1875 + it "batch-size = 128" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 128 .& Nil) + sizeD sr `shouldReturn` 468 + describe "ImageRecordIter" $ do + it "batch-size = 32" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 32 .& Nil) + sizeD sr `shouldReturn` 313 + it "batch-size = 128" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 128 .& Nil) + sizeD sr `shouldReturn` 79 diff --git a/dataiter/test/streaming.hs b/dataiter/test/streaming.hs new file mode 100644 --- /dev/null +++ b/dataiter/test/streaming.hs @@ -0,0 +1,39 @@ +module Main where + +import Test.Hspec +import Streaming.Prelude +import MXNet.Core.Base + +import MXNet.Core.IO.DataIter.Streaming + +type DS = StreamData IO (NDArray Float, NDArray Float) + +main :: IO () +main = hspec $ do + describe "MNISTIter" $ do + it "batch-size = 1" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 1 .& Nil) :: DS + sizeD sr `shouldReturn` 60000 + it "batch-size = 32" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 32 .& Nil) :: DS + sizeD sr `shouldReturn` 1875 + it "batch-size = 128" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 128 .& Nil) :: DS + sizeD sr `shouldReturn` 468 + describe "ImageRecordIter" $ do + it "batch-size = 32" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 32 .& Nil) :: DS + sizeD sr `shouldReturn` 313 + it "batch-size = 128" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 128 .& Nil) :: DS + sizeD sr `shouldReturn` 79 diff --git a/fei-dataiter.cabal b/fei-dataiter.cabal new file mode 100644 --- /dev/null +++ b/fei-dataiter.cabal @@ -0,0 +1,87 @@ +name: fei-dataiter +version: 0.2.0.0 +synopsis: mxnet dataiters +description: Providing the mxnet dataiters as Stream or Conduit +homepage: https://github.com/pierric/fei-dataiter#readme +bug-reports: https://github.com/pierric/fei-dataiter/issues +author: Jiasen Wu +maintainer: jiasenwu@hotmail.com +copyright: 2018 Jiasen Wu +license: BSD3 +license-file: LICENSE +category: Machine Learning, AI +build-type: Simple +cabal-version: >= 1.10 + +extra-source-files: + README.md + +source-repository head + type: git + location: https://github.com/pierric/fei-dataiter + +library + hs-source-dirs: dataiter/src + build-depends: base >=4.7 && <5, + template-haskell >= 2.10.0.0, + streaming >= 0.1.4.5, + conduit >= 1.2 && < 1.4, + conduit-combinators >= 1.1.2 && < 1.4, + fei-base, + fei-nn + exposed-modules: MXNet.NN.DataIter.Streaming, + MXNet.NN.DataIter.Conduit + other-modules: Paths_fei_dataiter + MXNet.NN.DataIter.Raw + default-extensions: FlexibleContexts, + DataKinds, + TypeOperators, + ExplicitForAll, + TupleSections, + OverloadedLabels, + TypeFamilies + default-language: Haskell2010 + +executable mxnet-dataiter-gen + hs-source-dirs: cmdline + main-is: Main.hs + build-depends: base >= 4.7 && < 5, + optparse-applicative >= 0.14 && < 0.15, + haskell-src-exts >= 1.20, + hslogger >= 1.2, + text >= 1.2.0 && < 1.3, + mtl, + filepath >= 1.4 && < 1.5, + directory >= 1.3 && < 1.4, + fei-base + default-language: Haskell2010 + default-extensions: FlexibleContexts + extra-libraries: mxnet + +test-suite streaming + type: exitcode-stdio-1.0 + main-is: streaming.hs + hs-source-dirs: dataiter/test + build-depends: base>=4.7 && <5, + hspec==2.*, + streaming >= 0.1.4.5, + fei-base, + fei-dataiter + default-language: Haskell2010 + default-extensions: FlexibleContexts, + DataKinds, + TypeApplications + +test-suite conduit + type: exitcode-stdio-1.0 + main-is: conduit.hs + hs-source-dirs: dataiter/test + build-depends: base>=4.7 && <5, + hspec==2.*, + fei-base, + fei-dataiter + default-language: Haskell2010 + default-extensions: FlexibleContexts, + DataKinds, + TypeApplications +