language-pig 0.2.0.1 → 0.2.0.2
raw patch · 3 files changed
+228/−1 lines, 3 files
Files
- language-pig.cabal +3/−1
- test/Language/Pig/Parser/Test.hs +90/−0
- test/Language/Pig/Pretty/Test.hs +135/−0
language-pig.cabal view
@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: language-pig-version: 0.2.0.1+version: 0.2.0.2 synopsis: Pig parser in haskell. description: Parser and pretty printer for the Apache Pig scripting language (http://pig.apache.org/). The current version is implemented using Parsec parser combinators. -- description: @@ -42,6 +42,8 @@ hs-source-dirs: test main-is: tests.hs Type: exitcode-stdio-1.0+ other-modules: Language.Pig.Parser.Test+ Language.Pig.Pretty.Test build-depends: base ==4.6.*, Cabal >= 1.16.0 , language-pig , QuickCheck
+ test/Language/Pig/Parser/Test.hs view
@@ -0,0 +1,90 @@+module Language.Pig.Parser.Test+ where++import Test.Framework (testGroup, Test)+import Test.Framework.Providers.HUnit+import Test.HUnit hiding (Test)++import Control.Monad (liftM, guard)+import Control.Exception+import Language.Pig.Parser+import Language.Pig.Parser.Parser+import Language.Pig.Parser.AST++parserSuite :: Test+parserSuite = testGroup "Parser"+ [testCase "load statement 1" (testStmt "users = LOAD 'sorted_log/user_registration/$date/*' USING LogStorage() AS (date:chararray, time:chararray, user_id:long);" + "Seq [Assignment (Identifier \"users\") (LoadClause (Filename \"sorted_log/user_registration/$date/*\") (Function \"LogStorage\" []) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"time\") CharArray,Field (Identifier \"user_id\") Long]))]")++ , testCase "load statement 2" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);"+ "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long]))]")++ , testCase "foreach stmt with flatten" (testStmt "users = FOREACH users GENERATE FLATTEN(group) AS (date, herd);" + "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"])]))]")++ , testCase "foreach stmt with expression" (testStmt "users = FOREACH users GENERATE *, ((user_id % 100) / 10) AS cohort;"+ "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [TupleFieldGlob,ExpressionTransform (Binary Divide (Binary Modulo (AliasTerm (Identifier \"user_id\")) (ScalarTerm (Number (Left 100)))) (ScalarTerm (Number (Left 10)))) (Identifier \"cohort\")]))]")++ , testCase "foreach stmt with ternary if-then-else" (testStmt "users = FOREACH users GENERATE *, (cohort <= 4 ? '04' : '59') AS herd;"+ "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [TupleFieldGlob,ExpressionTransform (BinCond (BooleanExpression LessEqual (AliasTerm (Identifier \"cohort\")) (ScalarTerm (Number (Left 4)))) (ScalarTerm (String \"04\")) (ScalarTerm (String \"59\"))) (Identifier \"herd\")]))]")+ , testCase "foreach stmt with flatten and function" (testStmt "report = FOREACH report GENERATE FLATTEN(group) AS (date, herd), COUNT(active_users) AS day_visits;"+ "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"]),FunctionTransform (Function \"COUNT\" [AliasArgument (Identifier \"active_users\")]) (Identifier \"day_visits\")]))]")+ , testCase "foreach stmt with qualified field names" (testStmt "report = FOREACH report GENERATE report::date AS date, report::herd AS herd, report::day_visits AS day_visits, visits::visits AS visits;"+ "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [AliasTransform (Identifier \"report::date\") (Identifier \"date\"),AliasTransform (Identifier \"report::herd\") (Identifier \"herd\"),AliasTransform (Identifier \"report::day_visits\") (Identifier \"day_visits\"),AliasTransform (Identifier \"visits::visits\") (Identifier \"visits\")]))]")+ , testCase "foreach stmt with quoted string" (testStmt "report = FOREACH report GENERATE '$date' AS date, *;"+ "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [EnvTransform (String \"$date\") (Identifier \"date\"),TupleFieldGlob]))]")++ , testCase "join stmt" (testStmt "active_users = JOIN users BY user_id, active_users BY user_id;"+ "Seq [Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]")++ , testCase "group stmt by one field" (testStmt "visits = GROUP active_users BY herd;"+ "Seq [Assignment (Identifier \"visits\") (GroupClause (Identifier \"active_users\") (SingleColumn (Identifier \"herd\")))]")++ , testCase "group stmt by several fields" (testStmt "report = GROUP active_users BY (date, herd);"+ "Seq [Assignment (Identifier \"report\") (GroupClause (Identifier \"active_users\") (MultipleColumn (Tuple [Identifier \"date\",Identifier \"herd\"])))]")++ , testCase "describe stmt" (testStmt "DESCRIBE visits;"+ "Seq [Describe (Identifier \"visits\")]")++ , testCase "define stmt" (testStmt "define RESOLVE `python delta.py $date` SHIP('delta.py');"+ "Seq [DefineUDF (Identifier \"RESOLVE\") (Exec \"python delta.py $date\") (Ship (Filename \"delta.py\"))]")++ , testCase "stream stmt" (testStmt "report = STREAM report THROUGH RESOLVE AS (day:chararray, herd:chararray, day_visits:int, visits:int);"+ "Seq [Assignment (Identifier \"report\") (StreamClause (Identifier \"report\") (Identifier \"RESOLVE\") (TupleDef [Field (Identifier \"day\") CharArray,Field (Identifier \"herd\") CharArray,Field (Identifier \"day_visits\") Int,Field (Identifier \"visits\") Int]))]")++ , testCase "store stmt" (testStmt "STORE report INTO '$output' USING ColumnStorage(',');"+ "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]")++ , testCase "several statements" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);\nactive_users = JOIN users BY user_id, active_users BY user_id;" + "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long])),Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]")+ , testCase "case insensitivity of keywords" (testStmt "store report into '$output' using ColumnStorage(',');"+ "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]")+ , testCase "input file path" (testFilePath "example.pig" "example.pig")+-- , testCase "non existant file path" (testFileError "dummy.pig" "")+ ]++testStmt :: String -> String -> Assertion+testStmt str expected = expected @=? (show $ parseString str)++testFilePath :: String -> String -> Assertion+testFilePath str expected = do path <- getPath $ parseFile str+ assertEqual "same file" expected path++{-+testFileError :: String -> String -> Assertion+testFileError str expected = assertException IOError $ parseFile str++assertException :: (Exception e, Eq e) => e -> IO a -> IO ()+assertException ex action =+ handleJust isWanted (const $ return ()) $ do+ action+ assertFailure $ "Expected exception: " ++ show ex+ where isWanted = guard . (== ex)+-}++getPath :: IO PigFile -> IO String+getPath pig = liftM filePath pig++filePath :: PigFile -> String+filePath (PigFile path _) = path+
+ test/Language/Pig/Pretty/Test.hs view
@@ -0,0 +1,135 @@+module Language.Pig.Pretty.Test+where++import Test.Framework (testGroup, Test)+import Test.Framework.Providers.HUnit+import Test.Framework.Providers.QuickCheck2 (testProperty)+import Test.HUnit hiding (Test)+import Test.QuickCheck (Arbitrary, arbitrary, oneof)+import Control.Applicative ((<$>), (<*>))+import Data.Text (isInfixOf, pack)++import Language.Pig.Parser+import Language.Pig.Parser.Parser+import Language.Pig.Pretty++prettyPrintSuite :: Test+prettyPrintSuite = testGroup "pretty print"+ [+ testCase "no statements" (testPrint (Seq []) "no statements\n")+ , testCase "load statement" (testPrint (Seq [Assignment (Identifier "active_users") (LoadClause (Filename "warehouse/active_users/daily/point/{$visit_dates}*") (Function "ColumnStorage" [StringArgument (String " ")]) (TupleDef [Field (Identifier "date") CharArray,Field (Identifier "user_id") Long]))])+ "sequence of statements:\n assignment \n | \n -------------------------------------------------------------------------------------- \n / \\ \nidentifier: active_users LOAD clause \n | \n ------------------------------------------------------------------------------------- \n / | \\ \n filename: \"warehouse/active_users/daily/point/{$visit_dates}*\" function ColumnStorage tuple def \n | | \n string argument: \" \" ----------------------------- \n / \\ \n field: date of type CharArray field: user_id of type Long\n")++ , testCase "expression" (testPrint (Seq [Assignment (Identifier "users") (ForeachClause (Identifier "users") (GenBlock [TupleFieldGlob,ExpressionTransform (Binary Divide (Binary Modulo (AliasTerm (Identifier "user_id")) (ScalarTerm (Number (Left 100)))) (ScalarTerm (Number (Left 10)))) (Identifier "cohort")]))]) + "sequence of statements:\n assignment \n | \n ----------------------------------------------------------- \n / \\ \nidentifier: users FOREACH clause \n | \n -------------------------------------------------- \n / \\ \n identifier: users transformation block \n | \n ---------------------------------------- \n / \\ \n * calculate \n | \n --------------------------------------- \n / \\ \n binary expression identifier: cohort\n | \n -------------------------------------------------- \n / | \\ \n Divide binary expression double:10 \n | \n ------------------------------ \n / | \\ \n Modulo identifier: user_id double:100 \n")+-- , testProperty "pretty prints to ast" prop_printed+ ]++testPrint :: Root -> String -> Assertion+testPrint tree expected = expected @=? prettyPrint tree++prop_printed tree = (pack "statement") `isInfixOf` (pack $ prettyPrint tree)+ where types = (tree :: Root)++instance Arbitrary Root where+ arbitrary = Seq <$> arbitrary++instance Arbitrary Statement where+ arbitrary = oneof [Assignment <$> arbitrary <*> arbitrary,+ Describe <$> arbitrary,+ DefineUDF <$> arbitrary <*> arbitrary <*> arbitrary,+ Store <$> arbitrary <*> arbitrary <*> arbitrary]++instance Arbitrary OpClause where+ arbitrary = oneof [ LoadClause <$> arbitrary <*> arbitrary <*> arbitrary+ , ForeachClause <$> arbitrary <*> arbitrary+ , GroupClause <$> arbitrary <*> arbitrary+ , InnerJoinClause <$> arbitrary+ , StreamClause <$> arbitrary <*> arbitrary <*> arbitrary]++instance Arbitrary GenBlock where+ arbitrary = GenBlock <$> arbitrary++instance Arbitrary GroupBy where+ arbitrary = oneof [ SingleColumn <$> arbitrary+ , MultipleColumn <$> arbitrary ]++instance Arbitrary Transform where+ arbitrary = oneof [ Flatten <$> arbitrary <*> arbitrary+ , return TupleFieldGlob+ , AliasTransform <$> arbitrary <*> arbitrary+ , ExpressionTransform <$> arbitrary <*> arbitrary+ , FunctionTransform <$> arbitrary <*> arbitrary+ , EnvTransform <$> arbitrary <*> arbitrary]++instance Arbitrary Join where+ arbitrary = Join <$> arbitrary <*> arbitrary++instance Arbitrary DefineSpec where+ arbitrary = Ship <$> arbitrary++instance Arbitrary Alias where+ arbitrary = Identifier <$> arbitrary++instance Arbitrary Language.Pig.Parser.Parser.Path where+ arbitrary = oneof [ Filename <$> arbitrary+ , Directory <$> arbitrary ]++instance Arbitrary Command where+ arbitrary = Exec <$> arbitrary++instance Arbitrary Function where+ arbitrary = Function <$> arbitrary <*> arbitrary++instance Arbitrary Argument where+ arbitrary = oneof [ StringArgument <$> arbitrary,+ AliasArgument <$> arbitrary]+ +instance Arbitrary TupleDef where+ arbitrary = TupleDef <$> arbitrary++instance Arbitrary Tuple where+ arbitrary = Tuple <$> arbitrary++instance Arbitrary Field where+ arbitrary = Field <$> arbitrary <*> arbitrary++instance Arbitrary Expression where+ arbitrary = oneof [ -- Unary <$> arbitrary <*> arbitrary+-- , Binary <$> arbitrary <*> arbitrary <*> arbitrary+-- , BinCond <$> arbitrary <*> arbitrary <*> arbitrary+ ScalarTerm <$> arbitrary+ , AliasTerm <$> arbitrary ]++{-+ arbitrary = sized arbExpression'+arbExpression :: Int -> Gen a+arbExpression 0 = ScalarTerm . Number <*> arbitrary+arbExpression n = do+ (Positive m) <- arbitrary+ let n' = n / (m + 1)+ f <- mapM (arbExpression n') [1..m]+ return $ (Binary <$> arbitrary <*> f <*> f++-}++instance Arbitrary BooleanExpression where+ arbitrary = oneof [ BooleanExpression <$> arbitrary <*> arbitrary <*> arbitrary+ , BooleanUnary <$> arbitrary <*> arbitrary+ , BooleanBinary <$> arbitrary <*> arbitrary <*> arbitrary ]++instance Arbitrary Scalar where+ arbitrary = oneof [ Number <$> arbitrary+ , String <$> arbitrary ]+ +instance Arbitrary SimpleType where+ arbitrary = oneof [ return Int , return Long , return Float , return Double , return CharArray , return ByteArray]++instance Arbitrary Operator where+ arbitrary = oneof [ return Neg , return Add , return Subtract , return Multiply , return Divide , return Modulo ]++instance Arbitrary BooleanOperator where+ arbitrary = oneof [ return And, return Or, return Not ]++instance Arbitrary ComparisonOperator where+ arbitrary = oneof [ return Equal, return NotEqual, return Greater, return Less, return GreaterEqual, return LessEqual ]