diff --git a/language-pig.cabal b/language-pig.cabal --- a/language-pig.cabal +++ b/language-pig.cabal @@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: language-pig -version: 0.2.0.1 +version: 0.2.0.2 synopsis: Pig parser in haskell. description: Parser and pretty printer for the Apache Pig scripting language (http://pig.apache.org/). The current version is implemented using Parsec parser combinators. -- description: @@ -42,6 +42,8 @@ hs-source-dirs: test main-is: tests.hs Type: exitcode-stdio-1.0 + other-modules: Language.Pig.Parser.Test + Language.Pig.Pretty.Test build-depends: base ==4.6.*, Cabal >= 1.16.0 , language-pig , QuickCheck diff --git a/test/Language/Pig/Parser/Test.hs b/test/Language/Pig/Parser/Test.hs new file mode 100644 --- /dev/null +++ b/test/Language/Pig/Parser/Test.hs @@ -0,0 +1,90 @@ +module Language.Pig.Parser.Test + where + +import Test.Framework (testGroup, Test) +import Test.Framework.Providers.HUnit +import Test.HUnit hiding (Test) + +import Control.Monad (liftM, guard) +import Control.Exception +import Language.Pig.Parser +import Language.Pig.Parser.Parser +import Language.Pig.Parser.AST + +parserSuite :: Test +parserSuite = testGroup "Parser" + [testCase "load statement 1" (testStmt "users = LOAD 'sorted_log/user_registration/$date/*' USING LogStorage() AS (date:chararray, time:chararray, user_id:long);" + "Seq [Assignment (Identifier \"users\") (LoadClause (Filename \"sorted_log/user_registration/$date/*\") (Function \"LogStorage\" []) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"time\") CharArray,Field (Identifier \"user_id\") Long]))]") + + , testCase "load statement 2" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);" + "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long]))]") + + , testCase "foreach stmt with flatten" (testStmt "users = FOREACH users GENERATE FLATTEN(group) AS (date, herd);" + "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"])]))]") + + , testCase "foreach stmt with expression" (testStmt "users = FOREACH users GENERATE *, ((user_id % 100) / 10) AS cohort;" + "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [TupleFieldGlob,ExpressionTransform (Binary Divide (Binary Modulo (AliasTerm (Identifier \"user_id\")) (ScalarTerm (Number (Left 100)))) (ScalarTerm (Number (Left 10)))) (Identifier \"cohort\")]))]") + + , testCase "foreach stmt with ternary if-then-else" (testStmt "users = FOREACH users GENERATE *, (cohort <= 4 ? '04' : '59') AS herd;" + "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [TupleFieldGlob,ExpressionTransform (BinCond (BooleanExpression LessEqual (AliasTerm (Identifier \"cohort\")) (ScalarTerm (Number (Left 4)))) (ScalarTerm (String \"04\")) (ScalarTerm (String \"59\"))) (Identifier \"herd\")]))]") + , testCase "foreach stmt with flatten and function" (testStmt "report = FOREACH report GENERATE FLATTEN(group) AS (date, herd), COUNT(active_users) AS day_visits;" + "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"]),FunctionTransform (Function \"COUNT\" [AliasArgument (Identifier \"active_users\")]) (Identifier \"day_visits\")]))]") + , testCase "foreach stmt with qualified field names" (testStmt "report = FOREACH report GENERATE report::date AS date, report::herd AS herd, report::day_visits AS day_visits, visits::visits AS visits;" + "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [AliasTransform (Identifier \"report::date\") (Identifier \"date\"),AliasTransform (Identifier \"report::herd\") (Identifier \"herd\"),AliasTransform (Identifier \"report::day_visits\") (Identifier \"day_visits\"),AliasTransform (Identifier \"visits::visits\") (Identifier \"visits\")]))]") + , testCase "foreach stmt with quoted string" (testStmt "report = FOREACH report GENERATE '$date' AS date, *;" + "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [EnvTransform (String \"$date\") (Identifier \"date\"),TupleFieldGlob]))]") + + , testCase "join stmt" (testStmt "active_users = JOIN users BY user_id, active_users BY user_id;" + "Seq [Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]") + + , testCase "group stmt by one field" (testStmt "visits = GROUP active_users BY herd;" + "Seq [Assignment (Identifier \"visits\") (GroupClause (Identifier \"active_users\") (SingleColumn (Identifier \"herd\")))]") + + , testCase "group stmt by several fields" (testStmt "report = GROUP active_users BY (date, herd);" + "Seq [Assignment (Identifier \"report\") (GroupClause (Identifier \"active_users\") (MultipleColumn (Tuple [Identifier \"date\",Identifier \"herd\"])))]") + + , testCase "describe stmt" (testStmt "DESCRIBE visits;" + "Seq [Describe (Identifier \"visits\")]") + + , testCase "define stmt" (testStmt "define RESOLVE `python delta.py $date` SHIP('delta.py');" + "Seq [DefineUDF (Identifier \"RESOLVE\") (Exec \"python delta.py $date\") (Ship (Filename \"delta.py\"))]") + + , testCase "stream stmt" (testStmt "report = STREAM report THROUGH RESOLVE AS (day:chararray, herd:chararray, day_visits:int, visits:int);" + "Seq [Assignment (Identifier \"report\") (StreamClause (Identifier \"report\") (Identifier \"RESOLVE\") (TupleDef [Field (Identifier \"day\") CharArray,Field (Identifier \"herd\") CharArray,Field (Identifier \"day_visits\") Int,Field (Identifier \"visits\") Int]))]") + + , testCase "store stmt" (testStmt "STORE report INTO '$output' USING ColumnStorage(',');" + "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]") + + , testCase "several statements" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);\nactive_users = JOIN users BY user_id, active_users BY user_id;" + "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long])),Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]") + , testCase "case insensitivity of keywords" (testStmt "store report into '$output' using ColumnStorage(',');" + "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]") + , testCase "input file path" (testFilePath "example.pig" "example.pig") +-- , testCase "non existant file path" (testFileError "dummy.pig" "") + ] + +testStmt :: String -> String -> Assertion +testStmt str expected = expected @=? (show $ parseString str) + +testFilePath :: String -> String -> Assertion +testFilePath str expected = do path <- getPath $ parseFile str + assertEqual "same file" expected path + +{- +testFileError :: String -> String -> Assertion +testFileError str expected = assertException IOError $ parseFile str + +assertException :: (Exception e, Eq e) => e -> IO a -> IO () +assertException ex action = + handleJust isWanted (const $ return ()) $ do + action + assertFailure $ "Expected exception: " ++ show ex + where isWanted = guard . (== ex) +-} + +getPath :: IO PigFile -> IO String +getPath pig = liftM filePath pig + +filePath :: PigFile -> String +filePath (PigFile path _) = path + diff --git a/test/Language/Pig/Pretty/Test.hs b/test/Language/Pig/Pretty/Test.hs new file mode 100644 --- /dev/null +++ b/test/Language/Pig/Pretty/Test.hs @@ -0,0 +1,135 @@ +module Language.Pig.Pretty.Test +where + +import Test.Framework (testGroup, Test) +import Test.Framework.Providers.HUnit +import Test.Framework.Providers.QuickCheck2 (testProperty) +import Test.HUnit hiding (Test) +import Test.QuickCheck (Arbitrary, arbitrary, oneof) +import Control.Applicative ((<$>), (<*>)) +import Data.Text (isInfixOf, pack) + +import Language.Pig.Parser +import Language.Pig.Parser.Parser +import Language.Pig.Pretty + +prettyPrintSuite :: Test +prettyPrintSuite = testGroup "pretty print" + [ + testCase "no statements" (testPrint (Seq []) "no statements\n") + , testCase "load statement" (testPrint (Seq [Assignment (Identifier "active_users") (LoadClause (Filename "warehouse/active_users/daily/point/{$visit_dates}*") (Function "ColumnStorage" [StringArgument (String " ")]) (TupleDef [Field (Identifier "date") CharArray,Field (Identifier "user_id") Long]))]) + "sequence of statements:\n assignment \n | \n -------------------------------------------------------------------------------------- \n / \\ \nidentifier: active_users LOAD clause \n | \n ------------------------------------------------------------------------------------- \n / | \\ \n filename: \"warehouse/active_users/daily/point/{$visit_dates}*\" function ColumnStorage tuple def \n | | \n string argument: \" \" ----------------------------- \n / \\ \n field: date of type CharArray field: user_id of type Long\n") + + , testCase "expression" (testPrint (Seq [Assignment (Identifier "users") (ForeachClause (Identifier "users") (GenBlock [TupleFieldGlob,ExpressionTransform (Binary Divide (Binary Modulo (AliasTerm (Identifier "user_id")) (ScalarTerm (Number (Left 100)))) (ScalarTerm (Number (Left 10)))) (Identifier "cohort")]))]) + "sequence of statements:\n assignment \n | \n ----------------------------------------------------------- \n / \\ \nidentifier: users FOREACH clause \n | \n -------------------------------------------------- \n / \\ \n identifier: users transformation block \n | \n ---------------------------------------- \n / \\ \n * calculate \n | \n --------------------------------------- \n / \\ \n binary expression identifier: cohort\n | \n -------------------------------------------------- \n / | \\ \n Divide binary expression double:10 \n | \n ------------------------------ \n / | \\ \n Modulo identifier: user_id double:100 \n") +-- , testProperty "pretty prints to ast" prop_printed + ] + +testPrint :: Root -> String -> Assertion +testPrint tree expected = expected @=? prettyPrint tree + +prop_printed tree = (pack "statement") `isInfixOf` (pack $ prettyPrint tree) + where types = (tree :: Root) + +instance Arbitrary Root where + arbitrary = Seq <$> arbitrary + +instance Arbitrary Statement where + arbitrary = oneof [Assignment <$> arbitrary <*> arbitrary, + Describe <$> arbitrary, + DefineUDF <$> arbitrary <*> arbitrary <*> arbitrary, + Store <$> arbitrary <*> arbitrary <*> arbitrary] + +instance Arbitrary OpClause where + arbitrary = oneof [ LoadClause <$> arbitrary <*> arbitrary <*> arbitrary + , ForeachClause <$> arbitrary <*> arbitrary + , GroupClause <$> arbitrary <*> arbitrary + , InnerJoinClause <$> arbitrary + , StreamClause <$> arbitrary <*> arbitrary <*> arbitrary] + +instance Arbitrary GenBlock where + arbitrary = GenBlock <$> arbitrary + +instance Arbitrary GroupBy where + arbitrary = oneof [ SingleColumn <$> arbitrary + , MultipleColumn <$> arbitrary ] + +instance Arbitrary Transform where + arbitrary = oneof [ Flatten <$> arbitrary <*> arbitrary + , return TupleFieldGlob + , AliasTransform <$> arbitrary <*> arbitrary + , ExpressionTransform <$> arbitrary <*> arbitrary + , FunctionTransform <$> arbitrary <*> arbitrary + , EnvTransform <$> arbitrary <*> arbitrary] + +instance Arbitrary Join where + arbitrary = Join <$> arbitrary <*> arbitrary + +instance Arbitrary DefineSpec where + arbitrary = Ship <$> arbitrary + +instance Arbitrary Alias where + arbitrary = Identifier <$> arbitrary + +instance Arbitrary Language.Pig.Parser.Parser.Path where + arbitrary = oneof [ Filename <$> arbitrary + , Directory <$> arbitrary ] + +instance Arbitrary Command where + arbitrary = Exec <$> arbitrary + +instance Arbitrary Function where + arbitrary = Function <$> arbitrary <*> arbitrary + +instance Arbitrary Argument where + arbitrary = oneof [ StringArgument <$> arbitrary, + AliasArgument <$> arbitrary] + +instance Arbitrary TupleDef where + arbitrary = TupleDef <$> arbitrary + +instance Arbitrary Tuple where + arbitrary = Tuple <$> arbitrary + +instance Arbitrary Field where + arbitrary = Field <$> arbitrary <*> arbitrary + +instance Arbitrary Expression where + arbitrary = oneof [ -- Unary <$> arbitrary <*> arbitrary +-- , Binary <$> arbitrary <*> arbitrary <*> arbitrary +-- , BinCond <$> arbitrary <*> arbitrary <*> arbitrary + ScalarTerm <$> arbitrary + , AliasTerm <$> arbitrary ] + +{- + arbitrary = sized arbExpression' +arbExpression :: Int -> Gen a +arbExpression 0 = ScalarTerm . Number <*> arbitrary +arbExpression n = do + (Positive m) <- arbitrary + let n' = n / (m + 1) + f <- mapM (arbExpression n') [1..m] + return $ (Binary <$> arbitrary <*> f <*> f + +-} + +instance Arbitrary BooleanExpression where + arbitrary = oneof [ BooleanExpression <$> arbitrary <*> arbitrary <*> arbitrary + , BooleanUnary <$> arbitrary <*> arbitrary + , BooleanBinary <$> arbitrary <*> arbitrary <*> arbitrary ] + +instance Arbitrary Scalar where + arbitrary = oneof [ Number <$> arbitrary + , String <$> arbitrary ] + +instance Arbitrary SimpleType where + arbitrary = oneof [ return Int , return Long , return Float , return Double , return CharArray , return ByteArray] + +instance Arbitrary Operator where + arbitrary = oneof [ return Neg , return Add , return Subtract , return Multiply , return Divide , return Modulo ] + +instance Arbitrary BooleanOperator where + arbitrary = oneof [ return And, return Or, return Not ] + +instance Arbitrary ComparisonOperator where + arbitrary = oneof [ return Equal, return NotEqual, return Greater, return Less, return GreaterEqual, return LessEqual ]