language-pig 0.2.0.2 → 0.3.0.0
raw patch · 8 files changed
+98/−41 lines, 8 files
Files
- README.md +2/−2
- language-pig.cabal +1/−1
- src/Language/Pig/Parser.hs +1/−0
- src/Language/Pig/Parser/AST.hs +11/−6
- src/Language/Pig/Parser/Parser.hs +49/−14
- src/Language/Pig/Pretty.hs +11/−5
- test/Language/Pig/Parser/Test.hs +17/−6
- test/Language/Pig/Pretty/Test.hs +6/−7
README.md view
@@ -32,7 +32,7 @@ ``` parseFile :: FilePath -> IO PigFile ```-PigFile contains the Root (of AST) and the file name.+PigFile contains the Root (of AST) and the file name. If you want just the AST, parseFileForAST would do it. Pretty print the produced tree: ```@@ -42,5 +42,5 @@ So to round it up, if you want to parse and pretty print the parsed AST of a Pig file (using Control.Applicative (<$>)) ```-prettyPrint <$> parseFile "example.pig" >>= putStrLn+prettyPrint <$> parseFileForAST "example.pig" >>= putStrLn ```
language-pig.cabal view
@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: language-pig-version: 0.2.0.2+version: 0.3.0.0 synopsis: Pig parser in haskell. description: Parser and pretty printer for the Apache Pig scripting language (http://pig.apache.org/). The current version is implemented using Parsec parser combinators. -- description:
src/Language/Pig/Parser.hs view
@@ -2,6 +2,7 @@ ( PA.parseString , PA.parseFile+ , PA.parseFileForAST ) where import qualified Language.Pig.Parser.Parser as PA
src/Language/Pig/Parser/AST.hs view
@@ -17,8 +17,9 @@ data Statement = Assignment Alias OpClause | Describe Alias- | DefineUDF Alias Command DefineSpec+ | DefineUDF Alias Aliasable [DefineSpec] | Store Alias Path Function+ | Register Library DERIVE data OpClause = LoadClause Path Function TupleDef@@ -56,16 +57,19 @@ | Directory String DERIVE +data Library = Library String+ DERIVE++data Aliasable = AliasCommand Command+ | AliasFunction Function+ DERIVE+ data Command = Exec String DERIVE -data Function = Function String [Argument]+data Function = Function String [Expression] DERIVE -data Argument = StringArgument Scalar- | AliasArgument Alias- DERIVE- data TupleDef = TupleDef [Field] DERIVE @@ -80,6 +84,7 @@ | BinCond BooleanExpression Expression Expression | ScalarTerm Scalar | AliasTerm Alias+ | FunctionTerm Function DERIVE data BooleanExpression = BooleanExpression ComparisonOperator Expression Expression
src/Language/Pig/Parser/Parser.hs view
@@ -1,6 +1,8 @@ module Language.Pig.Parser.Parser ( parseString , parseFile+ , parseFileForAST+ , parseTst , module Language.Pig.Parser.AST ) where @@ -27,7 +29,7 @@ , Token.nestedComments = True , Token.identStart = letter , Token.identLetter = alphaNum <|> specialChar- , Token.reservedNames = ["LOAD", "USING", "AS", -- todo case insensitivity of these keywords+ , Token.reservedNames = ["LOAD", "USING", "AS", "FOREACH", "GENERATE", "FLATTEN", "JOIN", "BY", "GROUP",@@ -35,8 +37,9 @@ "DEFINE", "STREAM", "THROUGH", "STORE", "INTO", "USING",+ "REGISTER", "int", "long", "float", "double", "chararray", "bytearray", "*"]- , Token.reservedOpNames = ["=", "+", "-", "*", "/", "%", "?", ":"]+ , Token.reservedOpNames = ["=", "+", "-", "*", "/", "%", "?", ":", "and", "or", "not"] , Token.caseSensitive = False } @@ -60,7 +63,12 @@ (intercalate "::") <$> sepBy1 identifierPart (string "::") identifierPart = (:) <$> letter <*> many1 (alphaNum <|> specialChar) +functionIdentifier = try(dottedIdentifier) <|> identifier+dottedIdentifier :: Parser String+dottedIdentifier = lexeme $+ (intercalate ".") <$> sepBy1 identifierPart (string ".") + -- Parser: top-down parseString :: [Char] -> Root@@ -71,9 +79,17 @@ parseFile :: FilePath -> IO PigFile parseFile filename = ((PigFile filename) . parseString) <$> readFile (filename) +parseFileForAST filename = getAST <$> parseFile filename++getAST :: PigFile -> Root+getAST (PigFile _ ast) = ast+ parsePig :: String -> Either ParseError Root parsePig input = parse pigParser "pigParser error" input +parseTst :: String -> Either ParseError Expression+parseTst input = parse conditional "test error" input+ pigParser :: Parser Root pigParser = whiteSpace >> statements @@ -81,7 +97,7 @@ statements = Seq <$> endBy statement semi statement :: Parser Statement-statement = query <|> describe <|> define <|> store+statement = query <|> describe <|> define <|> store <|> register query :: Parser Statement query = Assignment <$> @@ -95,9 +111,14 @@ define = DefineUDF <$> (reserved "define" *> pigVar) <*>- executable <*>- shipClause -- could be input, output, ship, cache, stderr in full pig grammar+ aliasable <*>+ defineSpec -- could be input, output, ship, cache, stderr in full pig grammar +aliasable :: Parser Aliasable+aliasable = try (AliasCommand <$>+ executable) <|>+ (AliasFunction <$> pigFunc)+ store :: Parser Statement store = Store <$> (reserved "STORE" *>@@ -107,7 +128,12 @@ (reserved "USING" *> pigFunc) +register :: Parser Statement+register = Register <$>+ (reserved "REGISTER" *>+ pigQuotedString Library) + opClause :: Parser OpClause opClause = loadClause <|> foreachClause@@ -124,6 +150,7 @@ (reserved "AS" *> pigTupleDef) + -- foreach: only the block (outer bag) version foreachClause :: Parser OpClause foreachClause = ForeachClause <$>@@ -159,6 +186,9 @@ (reserved "BY" *> pigIdentifier) +defineSpec :: Parser [DefineSpec]+defineSpec = many shipClause+ shipClause :: Parser DefineSpec shipClause = Ship . Filename <$> (reserved "SHIP" *>@@ -172,16 +202,16 @@ pigFunc :: Parser Function pigFunc = Function <$>- identifier <*>+ functionIdentifier <*> parens arguments -arguments :: Parser [Argument]+arguments :: Parser [Expression] arguments = sepBy argument comma -argument :: Parser Argument-argument = (StringArgument . String <$> quotedString) <|> - (AliasArgument <$> pigVar)+argument :: Parser Expression+argument = calculation + quotedString :: Parser String quotedString = (char '\'' *> (many $ noneOf "\'")) <* char '\'' <* whiteSpace -- doesn't take into account escaped quotes @@ -196,9 +226,10 @@ field :: Parser Field field = Field <$>- pigVar- <* char ':'- <*> pigType+ pigVar <*>+ ( char ':' *>+ whiteSpace *>+ pigType ) pigType :: Parser SimpleType pigType = pigSimpleType "int" Int <|>@@ -271,6 +302,7 @@ pigTerm = (ScalarTerm . String <$> quotedString) <|> (ScalarTerm <$> number) <|> generalExpression+ <|> try(FunctionTerm <$> pigFunc) <|> (AliasTerm <$> name) number = Number <$> naturalOrFloat -- for now - could be naturalOrFloat for inclusion@@ -289,8 +321,11 @@ <|> comparisonExpression booleanOperators = [ [Prefix (reservedOp "not" >> return (BooleanUnary Not))]+ , [Prefix (reservedOp "NOT" >> return (BooleanUnary Not))] , [Infix (reservedOp "and" >> return (BooleanBinary And)) AssocLeft]- , [Infix (reservedOp "or" >> return (BooleanBinary Or)) AssocLeft]]+ , [Infix (reservedOp "AND" >> return (BooleanBinary And)) AssocLeft]+ , [Infix (reservedOp "or" >> return (BooleanBinary Or)) AssocLeft]+ , [Infix (reservedOp "OR" >> return (BooleanBinary Or)) AssocLeft]] comparisonExpression :: Parser BooleanExpression comparisonExpression = flippedBooleanExpression <$> pigTerm <*> relation <*> pigTerm
src/Language/Pig/Pretty.hs view
@@ -28,6 +28,7 @@ toTree (Describe a) = Node "describe statement" [toTree a] toTree (DefineUDF a b c) = Node "define UDF statement" [toTree a, toTree b, toTree c] toTree (Store a b c) = Node "store statement" [toTree a, toTree b, toTree c]+ toTree (Register a) = Node "register statement" [toTree a] instance Treeable OpClause where toTree (LoadClause a b c) = Node "LOAD clause" [toTree a, toTree b, toTree c]@@ -54,6 +55,13 @@ instance Treeable Join where toTree (Join a b) = Node ("join " ++ a ++ " by " ++ b) [] +instance Treeable Aliasable where+ toTree (AliasCommand p) = toTree p+ toTree (AliasFunction p) = toTree p++instance Treeable [DefineSpec] where+ toTree specs = Node "define specs" (map toTree specs)+ instance Treeable DefineSpec where toTree (Ship p) = Node "SHIP" [toTree p] @@ -70,11 +78,6 @@ instance Treeable Function where toTree (Function s a) = Node ("function " ++ s) (map toTree a) -instance Treeable Argument where- toTree (StringArgument (String s)) = Node ("string argument: \"" ++ s ++ "\"") []- toTree (StringArgument (Number s)) = Node ("number argument: " ++ show s) []- toTree (AliasArgument (Identifier s)) = Node ("identifier argument: \"" ++ s ++ "\"") []- instance Treeable TupleDef where toTree (TupleDef f) = Node "tuple def" (map toTree f) @@ -112,6 +115,9 @@ toTree c = Node (show c) [] instance Treeable ComparisonOperator where+ toTree c = Node (show c) []++instance Treeable Library where toTree c = Node (show c) [] prettyPrint :: Root -> String
test/Language/Pig/Parser/Test.hs view
@@ -17,7 +17,7 @@ "Seq [Assignment (Identifier \"users\") (LoadClause (Filename \"sorted_log/user_registration/$date/*\") (Function \"LogStorage\" []) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"time\") CharArray,Field (Identifier \"user_id\") Long]))]") , testCase "load statement 2" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);"- "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long]))]")+ "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [ScalarTerm (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long]))]") , testCase "foreach stmt with flatten" (testStmt "users = FOREACH users GENERATE FLATTEN(group) AS (date, herd);" "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"])]))]")@@ -27,8 +27,13 @@ , testCase "foreach stmt with ternary if-then-else" (testStmt "users = FOREACH users GENERATE *, (cohort <= 4 ? '04' : '59') AS herd;" "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [TupleFieldGlob,ExpressionTransform (BinCond (BooleanExpression LessEqual (AliasTerm (Identifier \"cohort\")) (ScalarTerm (Number (Left 4)))) (ScalarTerm (String \"04\")) (ScalarTerm (String \"59\"))) (Identifier \"herd\")]))]")+ , testCase "foreach stmt with ternary if-then-else complex" (testStmt "desktop_client_dates = FOREACH desktop_client GENERATE server_date AS server_date, device_date AS device_date, user_id AS user_id, (server_date != '0' AND device_date != '0' ? (ISOToUnix(CustomFormatToISO(server_date,'YYYY-MM-dd'))-ISOToUnix(CustomFormatToISO(device_date,'YYYY-MM-dd')))/86400000 : 0) AS datediff;"+ "Seq [Assignment (Identifier \"desktop_client_dates\") (ForeachClause (Identifier \"desktop_client\") (GenBlock [AliasTransform (Identifier \"server_date\") (Identifier \"server_date\"),AliasTransform (Identifier \"device_date\") (Identifier \"device_date\"),AliasTransform (Identifier \"user_id\") (Identifier \"user_id\"),ExpressionTransform (BinCond (BooleanBinary And (BooleanExpression NotEqual (AliasTerm (Identifier \"server_date\")) (ScalarTerm (String \"0\"))) (BooleanExpression NotEqual (AliasTerm (Identifier \"device_date\")) (ScalarTerm (String \"0\")))) (Binary Divide (Binary Subtract (FunctionTerm (Function \"ISOToUnix\" [FunctionTerm (Function \"CustomFormatToISO\" [AliasTerm (Identifier \"server_date\"),ScalarTerm (String \"YYYY-MM-dd\")])])) (FunctionTerm (Function \"ISOToUnix\" [FunctionTerm (Function \"CustomFormatToISO\" [AliasTerm (Identifier \"device_date\"),ScalarTerm (String \"YYYY-MM-dd\")])]))) (ScalarTerm (Number (Left 86400000)))) (ScalarTerm (Number (Left 0)))) (Identifier \"datediff\")]))]")+{-+desktop_client_dates = FOREACH desktop_client GENERATE server_date AS server_date, device_date AS device_date, user_id AS user_id, (server_date != '0' AND device_date != '0' ? (ISOToUnix(CustomFormatToISO(server_date,'YYYY-MM-dd'))-ISOToUnix(CustomFormatToISO(device_date,'YYYY-MM-dd')))/86400000 : 0) AS datediff;+-} , testCase "foreach stmt with flatten and function" (testStmt "report = FOREACH report GENERATE FLATTEN(group) AS (date, herd), COUNT(active_users) AS day_visits;"- "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"]),FunctionTransform (Function \"COUNT\" [AliasArgument (Identifier \"active_users\")]) (Identifier \"day_visits\")]))]")+ "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"]),FunctionTransform (Function \"COUNT\" [AliasTerm (Identifier \"active_users\")]) (Identifier \"day_visits\")]))]") , testCase "foreach stmt with qualified field names" (testStmt "report = FOREACH report GENERATE report::date AS date, report::herd AS herd, report::day_visits AS day_visits, visits::visits AS visits;" "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [AliasTransform (Identifier \"report::date\") (Identifier \"date\"),AliasTransform (Identifier \"report::herd\") (Identifier \"herd\"),AliasTransform (Identifier \"report::day_visits\") (Identifier \"day_visits\"),AliasTransform (Identifier \"visits::visits\") (Identifier \"visits\")]))]") , testCase "foreach stmt with quoted string" (testStmt "report = FOREACH report GENERATE '$date' AS date, *;"@@ -47,18 +52,24 @@ "Seq [Describe (Identifier \"visits\")]") , testCase "define stmt" (testStmt "define RESOLVE `python delta.py $date` SHIP('delta.py');"- "Seq [DefineUDF (Identifier \"RESOLVE\") (Exec \"python delta.py $date\") (Ship (Filename \"delta.py\"))]")+ "Seq [DefineUDF (Identifier \"RESOLVE\") (AliasCommand (Exec \"python delta.py $date\")) [Ship (Filename \"delta.py\")]]") + , testCase "define stmt second type" (testStmt "DEFINE ISOToUnix org.apache.pig.piggybank.evaluation.datetime.convert.ISOToUnix();"+ "Seq [DefineUDF (Identifier \"ISOToUnix\") (AliasFunction (Function \"org.apache.pig.piggybank.evaluation.datetime.convert.ISOToUnix\" [])) []]")+ , testCase "stream stmt" (testStmt "report = STREAM report THROUGH RESOLVE AS (day:chararray, herd:chararray, day_visits:int, visits:int);" "Seq [Assignment (Identifier \"report\") (StreamClause (Identifier \"report\") (Identifier \"RESOLVE\") (TupleDef [Field (Identifier \"day\") CharArray,Field (Identifier \"herd\") CharArray,Field (Identifier \"day_visits\") Int,Field (Identifier \"visits\") Int]))]") , testCase "store stmt" (testStmt "STORE report INTO '$output' USING ColumnStorage(',');"- "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]")+ "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [ScalarTerm (String \",\")])]") + , testCase "register stmt" (testStmt "REGISTER 'lib/datafu-0.0.10.jar';"+ "Seq [Register (Library \"lib/datafu-0.0.10.jar\")]")+ , testCase "several statements" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);\nactive_users = JOIN users BY user_id, active_users BY user_id;" - "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long])),Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]")+ "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [ScalarTerm (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long])),Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]") , testCase "case insensitivity of keywords" (testStmt "store report into '$output' using ColumnStorage(',');"- "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]")+ "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [ScalarTerm (String \",\")])]") , testCase "input file path" (testFilePath "example.pig" "example.pig") -- , testCase "non existant file path" (testFileError "dummy.pig" "") ]
test/Language/Pig/Pretty/Test.hs view
@@ -17,9 +17,8 @@ prettyPrintSuite = testGroup "pretty print" [ testCase "no statements" (testPrint (Seq []) "no statements\n")- , testCase "load statement" (testPrint (Seq [Assignment (Identifier "active_users") (LoadClause (Filename "warehouse/active_users/daily/point/{$visit_dates}*") (Function "ColumnStorage" [StringArgument (String " ")]) (TupleDef [Field (Identifier "date") CharArray,Field (Identifier "user_id") Long]))])- "sequence of statements:\n assignment \n | \n -------------------------------------------------------------------------------------- \n / \\ \nidentifier: active_users LOAD clause \n | \n ------------------------------------------------------------------------------------- \n / | \\ \n filename: \"warehouse/active_users/daily/point/{$visit_dates}*\" function ColumnStorage tuple def \n | | \n string argument: \" \" ----------------------------- \n / \\ \n field: date of type CharArray field: user_id of type Long\n")-+ , testCase "load statement" (testPrint (Seq [Assignment (Identifier "active_users") (LoadClause (Filename "warehouse/active_users/daily/point/{$visit_dates}*") (Function "ColumnStorage" [ScalarTerm (String " ")]) (TupleDef [Field (Identifier "date") CharArray,Field (Identifier "user_id") Long]))])+ "sequence of statements:\n assignment \n | \n -------------------------------------------------------------------------------------- \n / \\ \nidentifier: active_users LOAD clause \n | \n ------------------------------------------------------------------------------------- \n / | \\ \n filename: \"warehouse/active_users/daily/point/{$visit_dates}*\" function ColumnStorage tuple def \n | | \n scalar: string ----------------------------- \n / \\ \n field: date of type CharArray field: user_id of type Long\n") , testCase "expression" (testPrint (Seq [Assignment (Identifier "users") (ForeachClause (Identifier "users") (GenBlock [TupleFieldGlob,ExpressionTransform (Binary Divide (Binary Modulo (AliasTerm (Identifier "user_id")) (ScalarTerm (Number (Left 100)))) (ScalarTerm (Number (Left 10)))) (Identifier "cohort")]))]) "sequence of statements:\n assignment \n | \n ----------------------------------------------------------- \n / \\ \nidentifier: users FOREACH clause \n | \n -------------------------------------------------- \n / \\ \n identifier: users transformation block \n | \n ---------------------------------------- \n / \\ \n * calculate \n | \n --------------------------------------- \n / \\ \n binary expression identifier: cohort\n | \n -------------------------------------------------- \n / | \\ \n Divide binary expression double:10 \n | \n ------------------------------ \n / | \\ \n Modulo identifier: user_id double:100 \n") -- , testProperty "pretty prints to ast" prop_printed@@ -54,6 +53,10 @@ arbitrary = oneof [ SingleColumn <$> arbitrary , MultipleColumn <$> arbitrary ] +instance Arbitrary Aliasable where+ arbitrary = oneof [ AliasCommand <$> arbitrary+ , AliasFunction <$> arbitrary ]+ instance Arbitrary Transform where arbitrary = oneof [ Flatten <$> arbitrary <*> arbitrary , return TupleFieldGlob@@ -81,10 +84,6 @@ instance Arbitrary Function where arbitrary = Function <$> arbitrary <*> arbitrary -instance Arbitrary Argument where- arbitrary = oneof [ StringArgument <$> arbitrary,- AliasArgument <$> arbitrary]- instance Arbitrary TupleDef where arbitrary = TupleDef <$> arbitrary