dawdle 0.1.0.1 → 0.1.0.2

raw patch · 7 files changed

+108/−44 lines, 7 filesdep ~dawdlePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: dawdle

API changes (from Hackage documentation)

Files

• dawdle.cabal +2/−2
• src/Database/Dawdle/Dawdle.lhs +10/−7
• src/Database/Dawdle/Options.lhs +25/−7
• src/Database/Dawdle/Parser.lhs +6/−1
• src/Database/Dawdle/PrettyPrint.lhs +10/−1
• src/Database/Dawdle/Types.lhs +42/−18
• src/Database/Dawdle/Utils.lhs +13/−8

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/
 
 name:                dawdle
-version:             0.1.0.1
+version:             0.1.0.2
 synopsis:            Generates DDL suggestions based on a CSV file
 description:         Generates DDL suggestions based on a CSV file
 homepage:            https://github.com/arnons1/dawdle
@@ -47,5 +47,5 @@                        pretty >= 1.0 && < 1.2,
                        text >= 0.11.1.13 && < 1.3,
                        filepath >= 1.4.0.0,
-                       dawdle == 0.1.0.1
+                       dawdle == 0.1.0.2
   ghc-options:         -Wall -O3

@@ -18,7 +18,10 @@ > import           Data.Time.Calendar
 > import           Text.Read (readMaybe)
 
-> analyzeFile :: Options -> [[String]] -> Either String [CellType]
+> -- | Analyze a file and return a list of types, or an error.
+> analyzeFile :: Options -- ^ Options from 'getOpts' (or 'defaultOptions') in "Database.Dawdle.Options"
+>             -> [[String]] -- ^ Table containing parsed CSV text, one element per cell
+>             -> Either String [CellType] -- ^ Returned type is either error message or list of types, each item representing one column
 > analyzeFile o mtrx = do
 >   let cols = transpose mtrx
 >   -- sanity on structure
@@ -33,12 +36,12 @@ 
 > analyzeCell :: CellType -> String -> Either String CellType
 > analyzeCell mp s
->  | null s || isNullText s = Right $ composeMaxTypesWithNulls mp $ Nullable Unknown
+>  | null s || isNullText s = Right $ addNullable $ composeMaxTypesWithNulls mp Unknown
 >  | b <- map toLower s
 >  , b `elem` ["false","true"]
->  , removeNull mp `elem` [CTBool,Unknown] =
+>  , removeNullable mp `elem` [CTBool,Unknown] =
 >      Right $ composeMaxTypesWithNulls mp CTBool
->  | isDateOrUnknown $ removeNull mp
+>  | isDateOrUnknown $ removeNullable mp
 >  , Just fFmt <- msum
 >       [ ptm fmt s | fmt <- iso8601DateFormat Nothing : 
 >                            ["%Y%m%d" | length s == 8] ] = 
@@ -46,7 +49,7 @@ >        Just oFmt |
 >          oFmt /= fFmt -> return $ composeMaxTypesWithNulls mp $ CTChar (length s)
 >        _ -> return $ composeMaxTypesWithNulls mp $ CTDate fFmt
->  | isDateTimeOrUnknown $ removeNull mp
+>  | isDateTimeOrUnknown $ removeNullable mp
 >  , Just fFmt <- msum
 >       [ ptm fmt s | fmt <- [iso8601DateFormat $ Just "%H:%M:%S"
 >                                      ,"%Y-%m-%d %H:%M:%S%Q"
@@ -57,11 +60,11 @@ >          oFmt /= fFmt -> return $ composeMaxTypesWithNulls mp $ CTChar (length s)
 >        _ -> return $ composeMaxTypesWithNulls mp $ CTDateTime fFmt
 >  | isInt s
->  , removeNull mp < CTDateTime ""
+>  , removeNullable mp < CTDateTime ""
 >  , Right s' <- maybeToEither "Can't read integer as Integer (??)" (readMaybe s :: Maybe Integer) =
 >       composeMaxTypesWithNulls mp <$> intType' s'
 >  | (not . isInt) s && isCTNumber s
->  , removeNull mp < CTChar 1
+>  , removeNullable mp < CTChar 1
 >  , Right s' <- maybeToEither "Can't read float type as Double (??)" (readMaybe s :: Maybe Double) =
 >      composeMaxTypesWithNulls mp <$> floatType s'
 >   | otherwise = Right $ composeMaxTypesWithNulls mp $ CTChar (length s)

@@ -8,16 +8,32 @@ >
 > import           System.Console.GetOpt
 > import           Text.Read ( readMaybe )
->
+
+> -- | Options structure for passing to the main module's 'analyzeFile'
 > data Options = Options
->  { optVerbose     :: Bool
->  , optInput       :: Maybe FilePath
->  , optStopAfter   :: Maybe Int
->  , optWithHeader  :: Bool
->  , optSepChar     :: Char
->  , optVerifyIntegrity :: Bool
+>  { optVerbose     :: Bool -- ^ Verbose mode
+>  , optInput       :: Maybe FilePath -- ^ stdin (Nothing) or file to read (Just FilePath)
+>  , optStopAfter   :: Maybe Int -- ^ Threshold for stopping (Just Int)
+>                                --   or consume entire file (Nothing)
+>  , optWithHeader  :: Bool -- ^ Is the first line a header? (Boolean)
+>  , optSepChar     :: Char -- ^ Separator character for CSV (Default ',')
+>  , optVerifyIntegrity :: Bool -- ^ Verify integrity of CSV file first? (Default True)
 >  } deriving Show
 
+> -- | Default set of options:
+> --
+> --   * Verbosity: False
+> --
+> --   * Input: Nothing (stdin)
+> --
+> --   * Stop after threshold: Just 10000
+> --
+> --   * With header: False
+> --
+> --   * Separator char: ,
+> --
+> --   * Verify integrity: True
+> --
 > defaultOptions :: Options
 > defaultOptions    = Options
 >  { optVerbose     = False
@@ -54,6 +70,8 @@ > headErr' (a:_) = a
 > headErr' _ = error "Head error in Options.lhs"
 
+> -- | getOpts constructs a set of flags based on the user's input
+> --   and the default options set in the 'defaultOptions' structure
 > getOpts :: [String] -> IO (Options, [String])
 > getOpts argv =
 >   case getOpt Permute options argv of

@@ -10,7 +10,12 @@ > import Data.Text.Lazy (Text)
 > import Text.Parsec
 
-> parseCsv :: Char -> String -> Text -> Either ParseError [[String]]
+> -- | Parses a CSV file and returns a 2D list of parsed strings
+> parseCsv :: Char -- ^ Separator character for CSV
+>          -> String -- ^ Source of the text (stdin/file name)
+>          -> Text -- ^ "Data.Text.Lazy"'s 'Text' of the source
+>          -> Either ParseError [[String]] -- ^ Either "Text.Parsec"'s 'ParseError'
+>                                          --   or a 2D list of parsed strings
 > parseCsv sepChar = parse (csvFile sepChar)
 
 > newLines :: String

@@ -4,19 +4,28 @@ >              FlexibleContexts, RankNTypes #-}
 
 > module Database.Dawdle.PrettyPrint
+>        (pretty)
 > where
 > import Database.Dawdle.Types
 > import Text.PrettyPrint
 
-> pretty :: String -> [String] -> [CellType] -> String
+TODO: Add different syntax support (ANSI/Oracle/Microsoft/PostgreSQL,...)
+
+> -- | PrettyPrint the types as a Create Table statement
+> pretty :: String -- ^ Source of the data (stdin / filename)
+>        -> [String] -- ^ Column names to assign
+>        -> [CellType] -- ^ List of types
+>        -> String -- ^ create table statement returned
 > pretty fn colns = render . ddl fn colns
 
+> -- | Create a "Text.PrettyPrint" 'Doc' to be rendered
 > ddl :: String -> [String] -> [CellType] -> Doc
 > ddl fn colns cts = text "create table" <+> text fn <+> body
 >   where
 >     body = parens $ nest 3 $ vcat $ punctuate comma clnms
 >     clnms = zipWith (\a b -> text a <+> b) colns $ map (prettyTypes True) cts
 
+> -- | Prettyprint the types.
 > prettyTypes :: Bool -> CellType -> Doc
 > prettyTypes isNullPass = \case
 >   Nullable c -> prettyTypes False c <+> text "null"

@@ -5,9 +5,9 @@ >        (SParser
 >        ,CellType(..)
 >        ,isNullable
->        ,applyNullIfNeeded
+>        ,addNullable
 >        ,getDateFormatFromCT
->        ,removeNull
+>        ,removeNullable
 >        ,composeNull
 >        ,composeMaxTypesWithNulls
 >        ,isDateOrUnknown
@@ -17,42 +17,66 @@ > import Text.Parsec
 
 > type SParser a = forall s u (m :: * -> *). Stream s m Char => ParsecT s u m a
-> data CellType = Nullable CellType | Unknown | 
->   CTBool | CTInt8 | CTInt16 | CTInt32 | CTInt64 | CTFloat | CTDouble | CTDate String | CTDateTime String | CTChar Int
+> -- ^ Used to prettify the type signatures
+
+> -- | The CellType contains the types available for inferrence.
+> --   The order matters, as it represents the order in which the
+> --   algorithm will try to infer.
+> data CellType =
+>                 Nullable CellType -- ^ Nullable cell (Recursive definition)
+>               | Unknown -- ^ Initial type is unknown
+>               | CTBool
+>               | CTInt8
+>               | CTInt16
+>               | CTInt32
+>               | CTInt64
+>               | CTFloat
+>               | CTDouble
+>               | CTDate String -- ^ The string represents the format
+>               | CTDateTime String -- ^ The string represents the format
+>               | CTChar Int -- ^ String with size
 >   deriving (Eq,Show,Ord,Data,Typeable)
 
+> -- | Is this cell nullable?
 > isNullable :: CellType -> Bool
-> isNullable (Nullable _) = True
+> isNullable Nullable{} = True
 > isNullable _ = False
 
-> applyNullIfNeeded :: Bool -> CellType -> CellType
-> applyNullIfNeeded True r@Nullable{} = r
-> applyNullIfNeeded True r = Nullable r
-> applyNullIfNeeded False r = r
-
-> getDateFormatFromCT :: CellType -> Maybe String
-> getDateFormatFromCT (CTDateTime s) = Just s
-> getDateFormatFromCT (CTDate s) = Just s
-> getDateFormatFromCT _ = Nothing
+> -- | Add nullability to a cell
+> addNullable :: CellType -> CellType
+> addNullable r@Nullable{} = r
+> addNullable r = Nullable r
 
-> removeNull :: CellType -> CellType
-> removeNull (Nullable x) = x
-> removeNull x = x
+> -- | Remove nullability from a cell
+> removeNullable :: CellType -> CellType
+> removeNullable (Nullable x) = x
+> removeNullable x = x
 
+> -- | Compose second cell with first cell's nullability
 > composeNull :: CellType -> CellType -> CellType
 > composeNull _ r@Nullable{} = r
 > composeNull Nullable{} r = Nullable r
 > composeNull _ r = r
 
+> -- | Select the maximum type between first and second cell, and apply
+> --   nullability if needed (according to first cell)
 > composeMaxTypesWithNulls :: CellType -> CellType -> CellType
 > composeMaxTypesWithNulls mp nt =
->   composeNull mp (maximum $ map removeNull [mp,nt])
+>   composeNull mp (maximum $ map removeNullable [mp,nt])
 
+> -- | Pull out format from date or datetime cells
+> getDateFormatFromCT :: CellType -> Maybe String
+> getDateFormatFromCT (CTDateTime s) = Just s
+> getDateFormatFromCT (CTDate s) = Just s
+> getDateFormatFromCT _ = Nothing
+
+> -- | Is this cell a Date or Unknown?
 > isDateOrUnknown :: CellType -> Bool
 > isDateOrUnknown CTDate{} = True
 > isDateOrUnknown Unknown = True
 > isDateOrUnknown _ = False
 
+> -- | Is this cell a DateTime or Unknown?
 > isDateTimeOrUnknown :: CellType -> Bool
 > isDateTimeOrUnknown CTDateTime{} = True
 > isDateTimeOrUnknown Unknown = True

@@ -1,7 +1,6 @@ 
 > module Database.Dawdle.Utils
->  (getFstIfJust
->  ,isCTNumber
+>  (isCTNumber
 >  ,isInt
 >  ,maybeToEither
 >  ,allTheSame
@@ -11,24 +10,30 @@ > import Data.Char
 > import Data.List
 
-> getFstIfJust :: (a, Maybe b) -> Maybe a
-> getFstIfJust (a,Just _) = Just a
-> getFstIfJust _ = Nothing
-
+> -- | Is this a number or a float(y) number? (Can give incorrect
+> --   results, for example with version numbers like 0.5.1.1)
 > isCTNumber :: String -> Bool
 > isCTNumber = all (\x -> isNumber x || x `elem` ['-','.'])
 
+> -- | Is this number an int? (Similar to 'isCTNumber', but without a decimal point)
 > isInt :: String -> Bool
 > isInt = all (\x -> isNumber x || x =='-')
 
-> maybeToEither :: String -> Maybe a -> Either String a
+> -- | Promote a Maybe to an Either with an error string
+> maybeToEither :: String -- ^ Error string
+>               -> Maybe a -- ^ Maybe value to check
+>               -> Either String a -- ^ Either Error string or the Just value from the maybe
 > maybeToEither _ (Just x) = Right x
 > maybeToEither s Nothing = Left s
 
+> -- | Check that all the items in a list are identical. This is used to verify lengths in the algorithm.
 > allTheSame :: (Eq a) => [a] -> Bool
 > allTheSame xs = all (== head xs) (tail xs)
 
-> normalizeNames :: [String] -> [String]
+> -- | Normalize names by removing special characters, replacing
+> --   spaces with underscores and lowercasing everything.
+> normalizeNames :: [String] -- ^ List of strings to normalize
+>                -> [String] -- ^ List of normalized strings
 > normalizeNames = map normalize
 >  where
 >    normalize = map toLower . filter (\x -> x=='_' || isAlphaNum x) . intercalate "_" . words