diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ ``` parseFile :: FilePath -> IO PigFile ``` -PigFile contains the Root (of AST) and the file name. +PigFile contains the Root (of AST) and the file name. If you want just the AST, parseFileForAST would do it. Pretty print the produced tree: ``` @@ -42,5 +42,5 @@ So to round it up, if you want to parse and pretty print the parsed AST of a Pig file (using Control.Applicative (<$>)) ``` -prettyPrint <$> parseFile "example.pig" >>= putStrLn +prettyPrint <$> parseFileForAST "example.pig" >>= putStrLn ``` diff --git a/language-pig.cabal b/language-pig.cabal --- a/language-pig.cabal +++ b/language-pig.cabal @@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: language-pig -version: 0.2.0.2 +version: 0.3.0.0 synopsis: Pig parser in haskell. description: Parser and pretty printer for the Apache Pig scripting language (http://pig.apache.org/). The current version is implemented using Parsec parser combinators. -- description: diff --git a/src/Language/Pig/Parser.hs b/src/Language/Pig/Parser.hs --- a/src/Language/Pig/Parser.hs +++ b/src/Language/Pig/Parser.hs @@ -2,6 +2,7 @@ ( PA.parseString , PA.parseFile + , PA.parseFileForAST ) where import qualified Language.Pig.Parser.Parser as PA diff --git a/src/Language/Pig/Parser/AST.hs b/src/Language/Pig/Parser/AST.hs --- a/src/Language/Pig/Parser/AST.hs +++ b/src/Language/Pig/Parser/AST.hs @@ -17,8 +17,9 @@ data Statement = Assignment Alias OpClause | Describe Alias - | DefineUDF Alias Command DefineSpec + | DefineUDF Alias Aliasable [DefineSpec] | Store Alias Path Function + | Register Library DERIVE data OpClause = LoadClause Path Function TupleDef @@ -56,16 +57,19 @@ | Directory String DERIVE +data Library = Library String + DERIVE + +data Aliasable = AliasCommand Command + | AliasFunction Function + DERIVE + data Command = Exec String DERIVE -data Function = Function String [Argument] +data Function = Function String [Expression] DERIVE -data Argument = StringArgument Scalar - | AliasArgument Alias - DERIVE - data TupleDef = TupleDef [Field] DERIVE @@ -80,6 +84,7 @@ | BinCond BooleanExpression Expression Expression | ScalarTerm Scalar | AliasTerm Alias + | FunctionTerm Function DERIVE data BooleanExpression = BooleanExpression ComparisonOperator Expression Expression diff --git a/src/Language/Pig/Parser/Parser.hs b/src/Language/Pig/Parser/Parser.hs --- a/src/Language/Pig/Parser/Parser.hs +++ b/src/Language/Pig/Parser/Parser.hs @@ -1,6 +1,8 @@ module Language.Pig.Parser.Parser ( parseString , parseFile + , parseFileForAST + , parseTst , module Language.Pig.Parser.AST ) where @@ -27,7 +29,7 @@ , Token.nestedComments = True , Token.identStart = letter , Token.identLetter = alphaNum <|> specialChar - , Token.reservedNames = ["LOAD", "USING", "AS", -- todo case insensitivity of these keywords + , Token.reservedNames = ["LOAD", "USING", "AS", "FOREACH", "GENERATE", "FLATTEN", "JOIN", "BY", "GROUP", @@ -35,8 +37,9 @@ "DEFINE", "STREAM", "THROUGH", "STORE", "INTO", "USING", + "REGISTER", "int", "long", "float", "double", "chararray", "bytearray", "*"] - , Token.reservedOpNames = ["=", "+", "-", "*", "/", "%", "?", ":"] + , Token.reservedOpNames = ["=", "+", "-", "*", "/", "%", "?", ":", "and", "or", "not"] , Token.caseSensitive = False } @@ -60,7 +63,12 @@ (intercalate "::") <$> sepBy1 identifierPart (string "::") identifierPart = (:) <$> letter <*> many1 (alphaNum <|> specialChar) +functionIdentifier = try(dottedIdentifier) <|> identifier +dottedIdentifier :: Parser String +dottedIdentifier = lexeme $ + (intercalate ".") <$> sepBy1 identifierPart (string ".") + -- Parser: top-down parseString :: [Char] -> Root @@ -71,9 +79,17 @@ parseFile :: FilePath -> IO PigFile parseFile filename = ((PigFile filename) . parseString) <$> readFile (filename) +parseFileForAST filename = getAST <$> parseFile filename + +getAST :: PigFile -> Root +getAST (PigFile _ ast) = ast + parsePig :: String -> Either ParseError Root parsePig input = parse pigParser "pigParser error" input +parseTst :: String -> Either ParseError Expression +parseTst input = parse conditional "test error" input + pigParser :: Parser Root pigParser = whiteSpace >> statements @@ -81,7 +97,7 @@ statements = Seq <$> endBy statement semi statement :: Parser Statement -statement = query <|> describe <|> define <|> store +statement = query <|> describe <|> define <|> store <|> register query :: Parser Statement query = Assignment <$> @@ -95,9 +111,14 @@ define = DefineUDF <$> (reserved "define" *> pigVar) <*> - executable <*> - shipClause -- could be input, output, ship, cache, stderr in full pig grammar + aliasable <*> + defineSpec -- could be input, output, ship, cache, stderr in full pig grammar +aliasable :: Parser Aliasable +aliasable = try (AliasCommand <$> + executable) <|> + (AliasFunction <$> pigFunc) + store :: Parser Statement store = Store <$> (reserved "STORE" *> @@ -107,7 +128,12 @@ (reserved "USING" *> pigFunc) +register :: Parser Statement +register = Register <$> + (reserved "REGISTER" *> + pigQuotedString Library) + opClause :: Parser OpClause opClause = loadClause <|> foreachClause @@ -124,6 +150,7 @@ (reserved "AS" *> pigTupleDef) + -- foreach: only the block (outer bag) version foreachClause :: Parser OpClause foreachClause = ForeachClause <$> @@ -159,6 +186,9 @@ (reserved "BY" *> pigIdentifier) +defineSpec :: Parser [DefineSpec] +defineSpec = many shipClause + shipClause :: Parser DefineSpec shipClause = Ship . Filename <$> (reserved "SHIP" *> @@ -172,16 +202,16 @@ pigFunc :: Parser Function pigFunc = Function <$> - identifier <*> + functionIdentifier <*> parens arguments -arguments :: Parser [Argument] +arguments :: Parser [Expression] arguments = sepBy argument comma -argument :: Parser Argument -argument = (StringArgument . String <$> quotedString) <|> - (AliasArgument <$> pigVar) +argument :: Parser Expression +argument = calculation + quotedString :: Parser String quotedString = (char '\'' *> (many $ noneOf "\'")) <* char '\'' <* whiteSpace -- doesn't take into account escaped quotes @@ -196,9 +226,10 @@ field :: Parser Field field = Field <$> - pigVar - <* char ':' - <*> pigType + pigVar <*> + ( char ':' *> + whiteSpace *> + pigType ) pigType :: Parser SimpleType pigType = pigSimpleType "int" Int <|> @@ -271,6 +302,7 @@ pigTerm = (ScalarTerm . String <$> quotedString) <|> (ScalarTerm <$> number) <|> generalExpression + <|> try(FunctionTerm <$> pigFunc) <|> (AliasTerm <$> name) number = Number <$> naturalOrFloat -- for now - could be naturalOrFloat for inclusion @@ -289,8 +321,11 @@ <|> comparisonExpression booleanOperators = [ [Prefix (reservedOp "not" >> return (BooleanUnary Not))] + , [Prefix (reservedOp "NOT" >> return (BooleanUnary Not))] , [Infix (reservedOp "and" >> return (BooleanBinary And)) AssocLeft] - , [Infix (reservedOp "or" >> return (BooleanBinary Or)) AssocLeft]] + , [Infix (reservedOp "AND" >> return (BooleanBinary And)) AssocLeft] + , [Infix (reservedOp "or" >> return (BooleanBinary Or)) AssocLeft] + , [Infix (reservedOp "OR" >> return (BooleanBinary Or)) AssocLeft]] comparisonExpression :: Parser BooleanExpression comparisonExpression = flippedBooleanExpression <$> pigTerm <*> relation <*> pigTerm diff --git a/src/Language/Pig/Pretty.hs b/src/Language/Pig/Pretty.hs --- a/src/Language/Pig/Pretty.hs +++ b/src/Language/Pig/Pretty.hs @@ -28,6 +28,7 @@ toTree (Describe a) = Node "describe statement" [toTree a] toTree (DefineUDF a b c) = Node "define UDF statement" [toTree a, toTree b, toTree c] toTree (Store a b c) = Node "store statement" [toTree a, toTree b, toTree c] + toTree (Register a) = Node "register statement" [toTree a] instance Treeable OpClause where toTree (LoadClause a b c) = Node "LOAD clause" [toTree a, toTree b, toTree c] @@ -54,6 +55,13 @@ instance Treeable Join where toTree (Join a b) = Node ("join " ++ a ++ " by " ++ b) [] +instance Treeable Aliasable where + toTree (AliasCommand p) = toTree p + toTree (AliasFunction p) = toTree p + +instance Treeable [DefineSpec] where + toTree specs = Node "define specs" (map toTree specs) + instance Treeable DefineSpec where toTree (Ship p) = Node "SHIP" [toTree p] @@ -70,11 +78,6 @@ instance Treeable Function where toTree (Function s a) = Node ("function " ++ s) (map toTree a) -instance Treeable Argument where - toTree (StringArgument (String s)) = Node ("string argument: \"" ++ s ++ "\"") [] - toTree (StringArgument (Number s)) = Node ("number argument: " ++ show s) [] - toTree (AliasArgument (Identifier s)) = Node ("identifier argument: \"" ++ s ++ "\"") [] - instance Treeable TupleDef where toTree (TupleDef f) = Node "tuple def" (map toTree f) @@ -112,6 +115,9 @@ toTree c = Node (show c) [] instance Treeable ComparisonOperator where + toTree c = Node (show c) [] + +instance Treeable Library where toTree c = Node (show c) [] prettyPrint :: Root -> String diff --git a/test/Language/Pig/Parser/Test.hs b/test/Language/Pig/Parser/Test.hs --- a/test/Language/Pig/Parser/Test.hs +++ b/test/Language/Pig/Parser/Test.hs @@ -17,7 +17,7 @@ "Seq [Assignment (Identifier \"users\") (LoadClause (Filename \"sorted_log/user_registration/$date/*\") (Function \"LogStorage\" []) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"time\") CharArray,Field (Identifier \"user_id\") Long]))]") , testCase "load statement 2" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);" - "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long]))]") + "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [ScalarTerm (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long]))]") , testCase "foreach stmt with flatten" (testStmt "users = FOREACH users GENERATE FLATTEN(group) AS (date, herd);" "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"])]))]") @@ -27,8 +27,13 @@ , testCase "foreach stmt with ternary if-then-else" (testStmt "users = FOREACH users GENERATE *, (cohort <= 4 ? '04' : '59') AS herd;" "Seq [Assignment (Identifier \"users\") (ForeachClause (Identifier \"users\") (GenBlock [TupleFieldGlob,ExpressionTransform (BinCond (BooleanExpression LessEqual (AliasTerm (Identifier \"cohort\")) (ScalarTerm (Number (Left 4)))) (ScalarTerm (String \"04\")) (ScalarTerm (String \"59\"))) (Identifier \"herd\")]))]") + , testCase "foreach stmt with ternary if-then-else complex" (testStmt "desktop_client_dates = FOREACH desktop_client GENERATE server_date AS server_date, device_date AS device_date, user_id AS user_id, (server_date != '0' AND device_date != '0' ? (ISOToUnix(CustomFormatToISO(server_date,'YYYY-MM-dd'))-ISOToUnix(CustomFormatToISO(device_date,'YYYY-MM-dd')))/86400000 : 0) AS datediff;" + "Seq [Assignment (Identifier \"desktop_client_dates\") (ForeachClause (Identifier \"desktop_client\") (GenBlock [AliasTransform (Identifier \"server_date\") (Identifier \"server_date\"),AliasTransform (Identifier \"device_date\") (Identifier \"device_date\"),AliasTransform (Identifier \"user_id\") (Identifier \"user_id\"),ExpressionTransform (BinCond (BooleanBinary And (BooleanExpression NotEqual (AliasTerm (Identifier \"server_date\")) (ScalarTerm (String \"0\"))) (BooleanExpression NotEqual (AliasTerm (Identifier \"device_date\")) (ScalarTerm (String \"0\")))) (Binary Divide (Binary Subtract (FunctionTerm (Function \"ISOToUnix\" [FunctionTerm (Function \"CustomFormatToISO\" [AliasTerm (Identifier \"server_date\"),ScalarTerm (String \"YYYY-MM-dd\")])])) (FunctionTerm (Function \"ISOToUnix\" [FunctionTerm (Function \"CustomFormatToISO\" [AliasTerm (Identifier \"device_date\"),ScalarTerm (String \"YYYY-MM-dd\")])]))) (ScalarTerm (Number (Left 86400000)))) (ScalarTerm (Number (Left 0)))) (Identifier \"datediff\")]))]") +{- +desktop_client_dates = FOREACH desktop_client GENERATE server_date AS server_date, device_date AS device_date, user_id AS user_id, (server_date != '0' AND device_date != '0' ? (ISOToUnix(CustomFormatToISO(server_date,'YYYY-MM-dd'))-ISOToUnix(CustomFormatToISO(device_date,'YYYY-MM-dd')))/86400000 : 0) AS datediff; +-} , testCase "foreach stmt with flatten and function" (testStmt "report = FOREACH report GENERATE FLATTEN(group) AS (date, herd), COUNT(active_users) AS day_visits;" - "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"]),FunctionTransform (Function \"COUNT\" [AliasArgument (Identifier \"active_users\")]) (Identifier \"day_visits\")]))]") + "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [Flatten \"group\" (Tuple [Identifier \"date\",Identifier \"herd\"]),FunctionTransform (Function \"COUNT\" [AliasTerm (Identifier \"active_users\")]) (Identifier \"day_visits\")]))]") , testCase "foreach stmt with qualified field names" (testStmt "report = FOREACH report GENERATE report::date AS date, report::herd AS herd, report::day_visits AS day_visits, visits::visits AS visits;" "Seq [Assignment (Identifier \"report\") (ForeachClause (Identifier \"report\") (GenBlock [AliasTransform (Identifier \"report::date\") (Identifier \"date\"),AliasTransform (Identifier \"report::herd\") (Identifier \"herd\"),AliasTransform (Identifier \"report::day_visits\") (Identifier \"day_visits\"),AliasTransform (Identifier \"visits::visits\") (Identifier \"visits\")]))]") , testCase "foreach stmt with quoted string" (testStmt "report = FOREACH report GENERATE '$date' AS date, *;" @@ -47,18 +52,24 @@ "Seq [Describe (Identifier \"visits\")]") , testCase "define stmt" (testStmt "define RESOLVE `python delta.py $date` SHIP('delta.py');" - "Seq [DefineUDF (Identifier \"RESOLVE\") (Exec \"python delta.py $date\") (Ship (Filename \"delta.py\"))]") + "Seq [DefineUDF (Identifier \"RESOLVE\") (AliasCommand (Exec \"python delta.py $date\")) [Ship (Filename \"delta.py\")]]") + , testCase "define stmt second type" (testStmt "DEFINE ISOToUnix org.apache.pig.piggybank.evaluation.datetime.convert.ISOToUnix();" + "Seq [DefineUDF (Identifier \"ISOToUnix\") (AliasFunction (Function \"org.apache.pig.piggybank.evaluation.datetime.convert.ISOToUnix\" [])) []]") + , testCase "stream stmt" (testStmt "report = STREAM report THROUGH RESOLVE AS (day:chararray, herd:chararray, day_visits:int, visits:int);" "Seq [Assignment (Identifier \"report\") (StreamClause (Identifier \"report\") (Identifier \"RESOLVE\") (TupleDef [Field (Identifier \"day\") CharArray,Field (Identifier \"herd\") CharArray,Field (Identifier \"day_visits\") Int,Field (Identifier \"visits\") Int]))]") , testCase "store stmt" (testStmt "STORE report INTO '$output' USING ColumnStorage(',');" - "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]") + "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [ScalarTerm (String \",\")])]") + , testCase "register stmt" (testStmt "REGISTER 'lib/datafu-0.0.10.jar';" + "Seq [Register (Library \"lib/datafu-0.0.10.jar\")]") + , testCase "several statements" (testStmt "active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);\nactive_users = JOIN users BY user_id, active_users BY user_id;" - "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [StringArgument (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long])),Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]") + "Seq [Assignment (Identifier \"active_users\") (LoadClause (Filename \"warehouse/active_users/daily/point/{$visit_dates}*\") (Function \"ColumnStorage\" [ScalarTerm (String \" \")]) (TupleDef [Field (Identifier \"date\") CharArray,Field (Identifier \"user_id\") Long])),Assignment (Identifier \"active_users\") (InnerJoinClause [Join \"users\" \"user_id\",Join \"active_users\" \"user_id\"])]") , testCase "case insensitivity of keywords" (testStmt "store report into '$output' using ColumnStorage(',');" - "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [StringArgument (String \",\")])]") + "Seq [Store (Identifier \"report\") (Directory \"$output\") (Function \"ColumnStorage\" [ScalarTerm (String \",\")])]") , testCase "input file path" (testFilePath "example.pig" "example.pig") -- , testCase "non existant file path" (testFileError "dummy.pig" "") ] diff --git a/test/Language/Pig/Pretty/Test.hs b/test/Language/Pig/Pretty/Test.hs --- a/test/Language/Pig/Pretty/Test.hs +++ b/test/Language/Pig/Pretty/Test.hs @@ -17,9 +17,8 @@ prettyPrintSuite = testGroup "pretty print" [ testCase "no statements" (testPrint (Seq []) "no statements\n") - , testCase "load statement" (testPrint (Seq [Assignment (Identifier "active_users") (LoadClause (Filename "warehouse/active_users/daily/point/{$visit_dates}*") (Function "ColumnStorage" [StringArgument (String " ")]) (TupleDef [Field (Identifier "date") CharArray,Field (Identifier "user_id") Long]))]) - "sequence of statements:\n assignment \n | \n -------------------------------------------------------------------------------------- \n / \\ \nidentifier: active_users LOAD clause \n | \n ------------------------------------------------------------------------------------- \n / | \\ \n filename: \"warehouse/active_users/daily/point/{$visit_dates}*\" function ColumnStorage tuple def \n | | \n string argument: \" \" ----------------------------- \n / \\ \n field: date of type CharArray field: user_id of type Long\n") - + , testCase "load statement" (testPrint (Seq [Assignment (Identifier "active_users") (LoadClause (Filename "warehouse/active_users/daily/point/{$visit_dates}*") (Function "ColumnStorage" [ScalarTerm (String " ")]) (TupleDef [Field (Identifier "date") CharArray,Field (Identifier "user_id") Long]))]) + "sequence of statements:\n assignment \n | \n -------------------------------------------------------------------------------------- \n / \\ \nidentifier: active_users LOAD clause \n | \n ------------------------------------------------------------------------------------- \n / | \\ \n filename: \"warehouse/active_users/daily/point/{$visit_dates}*\" function ColumnStorage tuple def \n | | \n scalar: string ----------------------------- \n / \\ \n field: date of type CharArray field: user_id of type Long\n") , testCase "expression" (testPrint (Seq [Assignment (Identifier "users") (ForeachClause (Identifier "users") (GenBlock [TupleFieldGlob,ExpressionTransform (Binary Divide (Binary Modulo (AliasTerm (Identifier "user_id")) (ScalarTerm (Number (Left 100)))) (ScalarTerm (Number (Left 10)))) (Identifier "cohort")]))]) "sequence of statements:\n assignment \n | \n ----------------------------------------------------------- \n / \\ \nidentifier: users FOREACH clause \n | \n -------------------------------------------------- \n / \\ \n identifier: users transformation block \n | \n ---------------------------------------- \n / \\ \n * calculate \n | \n --------------------------------------- \n / \\ \n binary expression identifier: cohort\n | \n -------------------------------------------------- \n / | \\ \n Divide binary expression double:10 \n | \n ------------------------------ \n / | \\ \n Modulo identifier: user_id double:100 \n") -- , testProperty "pretty prints to ast" prop_printed @@ -54,6 +53,10 @@ arbitrary = oneof [ SingleColumn <$> arbitrary , MultipleColumn <$> arbitrary ] +instance Arbitrary Aliasable where + arbitrary = oneof [ AliasCommand <$> arbitrary + , AliasFunction <$> arbitrary ] + instance Arbitrary Transform where arbitrary = oneof [ Flatten <$> arbitrary <*> arbitrary , return TupleFieldGlob @@ -81,10 +84,6 @@ instance Arbitrary Function where arbitrary = Function <$> arbitrary <*> arbitrary -instance Arbitrary Argument where - arbitrary = oneof [ StringArgument <$> arbitrary, - AliasArgument <$> arbitrary] - instance Arbitrary TupleDef where arbitrary = TupleDef <$> arbitrary