language-pig (empty) → 0.1.0.0
raw patch · 10 files changed
+704/−0 lines, 10 filesdep +Cabaldep +HUnitdep +QuickChecksetup-changed
Dependencies added: Cabal, HUnit, QuickCheck, base, containers, language-pig, parsec, pretty-tree, test-framework, test-framework-hunit, test-framework-quickcheck2, text
Files
- LICENSE +1/−0
- README.md +48/−0
- Setup.hs +2/−0
- example.pig +34/−0
- language-pig.cabal +52/−0
- src/Language/Pig/Parser.hs +7/−0
- src/Language/Pig/Parser/AST.hs +119/−0
- src/Language/Pig/Parser/Parser.hs +314/−0
- src/Language/Pig/Pretty.hs +116/−0
- test/tests.hs +11/−0
+ LICENSE view
@@ -0,0 +1,1 @@+license here BSD-like
+ README.md view
@@ -0,0 +1,48 @@+# language-pig #++Parser and pretty printer for the Apache Pig scripting language (http://pig.apache.org/). The current version is implemented using Parsec parser combinators.++# Install #++Cabal project, now on hackage, so the usual++```+cabal install language-pig+```++Or from source++```+git clone ...+cd language-pig+cabal install+```++# Use #++Parse an expression:++```+parseString :: [Char] -> Root+```++Returns an AST (type is the root node).++Parse a file:++```+parseFile :: FilePath -> IO Root+```++Returns an AST (type = Root, which is the root node).++Pretty print the produced tree:+```+putStrLn $ prettyPrint tree+```++So to round it up, if you want to parse and pretty print the parsed AST of a Pig file (using Control.Applicative (<$>))++```+prettyPrint <$> parseFile "example.pig" >>= putStrLn+```
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ example.pig view
@@ -0,0 +1,34 @@+users = LOAD 'sorted_log/user_registration/$date/*' USING LogStorage() AS (date:chararray, time:chararray, user_id:long);++users = FOREACH users GENERATE *, ((user_id % 100) / 10) AS cohort;+users = FOREACH users GENERATE *, (cohort <= 4 ? '04' : '59') AS herd;+++active_users = LOAD 'warehouse/active_users/daily/point/{$visit_dates}*' USING ColumnStorage(' ') AS (date:chararray, user_id:long);++active_users = JOIN users BY user_id, active_users BY user_id;+active_users = FOREACH active_users GENERATE active_users::date AS date, active_users::user_id AS user_id, users::herd AS herd;+++visits = GROUP active_users BY herd;+visits = FOREACH visits GENERATE group AS herd, COUNT(active_users) AS visits;++DESCRIBE visits;++report = GROUP active_users BY (date, herd);++report = FOREACH report GENERATE FLATTEN(group) AS (date, herd), COUNT(active_users) AS day_visits;+DESCRIBE report;+++report = JOIN report BY herd, visits BY herd;+report = FOREACH report GENERATE report::date AS date, report::herd AS herd, report::day_visits AS day_visits, visits::visits AS visits;+++define RESOLVE `python delta.py $date` SHIP('delta.py');++report = STREAM report THROUGH RESOLVE AS (day:chararray, herd:chararray, day_visits:int, visits:int);++report = FOREACH report GENERATE '$date' AS date, *;++STORE report INTO '$output' USING ColumnStorage(',');
+ language-pig.cabal view
@@ -0,0 +1,52 @@+-- Initial language-pig.cabal generated by cabal init. For further +-- documentation, see http://haskell.org/cabal/users-guide/++name: language-pig+version: 0.1.0.0+synopsis: Pig parser in haskell.+description: Parser and pretty printer for the Apache Pig scripting language (http://pig.apache.org/). The current version is implemented using Parsec parser combinators.+-- description: +license: MIT+license-file: LICENSE+author: Elise Huard+maintainer: elise@jabberwocky.eu+-- copyright: +category: Language+build-type: Simple+cabal-version: >=1.8+Stability: Experimental++Extra-Source-Files:+ example.pig+ README.md++source-repository head+ type: git+ location: https://github.com/elisehuard/language-pig++library+ hs-source-dirs: src+ -- exposed-modules: + Exposed-modules: Language.Pig.Parser+ Language.Pig.Parser.Parser+ Language.Pig.Parser.AST+ Language.Pig.Pretty+ -- other-modules: + -- Other-modules: Language.Pig.Parser.Token+ build-depends: base ==4.6.*, Cabal >= 1.16.0+ , parsec >= 3.1.3+ , containers+ , pretty-tree++test-suite Tests+ hs-source-dirs: test+ main-is: tests.hs+ Type: exitcode-stdio-1.0+ build-depends: base ==4.6.*, Cabal >= 1.16.0+ , language-pig+ , QuickCheck+ , HUnit+ , test-framework+ , test-framework-hunit+ , test-framework-quickcheck2+ , text
+ src/Language/Pig/Parser.hs view
@@ -0,0 +1,7 @@+module Language.Pig.Parser+ (+ PA.parseString+ , PA.parseFile+ ) where++import qualified Language.Pig.Parser.Parser as PA
+ src/Language/Pig/Parser/AST.hs view
@@ -0,0 +1,119 @@+{-# LANGUAGE CPP, DeriveDataTypeable #-}+module Language.Pig.Parser.AST where++#define DERIVE deriving (Eq,Ord,Show,Typeable,Data)++import Data.Data+import Data.Typeable++-- source:+-- http://wiki.apache.org/pig/PigLexer+-- http://wiki.apache.org/pig/PigParser++data Root = Seq [Statement]+ DERIVE++data Statement = Assignment Alias OpClause+ | Describe Alias+ | DefineUDF Alias Command DefineSpec+ | Store Alias Path Function+ DERIVE++data OpClause = LoadClause Path Function TupleDef+ | ForeachClause Alias GenBlock+ | GroupClause Alias GroupBy+ | InnerJoinClause [Join]+ | StreamClause Alias Alias TupleDef+ DERIVE++data GenBlock = GenBlock [Transform]+ DERIVE++data GroupBy = SingleColumn Alias+ | MultipleColumn Tuple+ DERIVE++data Transform = Flatten String Tuple -- foreach flatten transform+ | TupleFieldGlob+ | AliasTransform Alias Alias+ | ExpressionTransform Expression Alias+ | FunctionTransform Function Alias+ | EnvTransform Scalar Alias+ DERIVE++data Join = Join String String+ DERIVE++data DefineSpec = Ship Path+ DERIVE++data Alias = Identifier String+ DERIVE++data Path = Filename String+ | Directory String+ DERIVE++data Command = Exec String+ DERIVE++data Function = Function String [Argument]+ DERIVE++data Argument = StringArgument Scalar+ | AliasArgument Alias+ DERIVE++data TupleDef = TupleDef [Field]+ DERIVE++data Tuple = Tuple [Alias]+ DERIVE++data Field = Field Alias SimpleType+ DERIVE++data Expression = Unary Operator Expression+ | Binary Operator Expression Expression+ | BinCond BooleanExpression Expression Expression+ | ScalarTerm Scalar+ | AliasTerm Alias+ DERIVE++data BooleanExpression = BooleanExpression ComparisonOperator Expression Expression+ | BooleanUnary BooleanOperator BooleanExpression+ | BooleanBinary BooleanOperator BooleanExpression BooleanExpression+ DERIVE++data Scalar = Number (Either Integer Double)+ | String String+ DERIVE++data SimpleType = Int+ | Long+ | Float+ | Double+ | CharArray+ | ByteArray+ DERIVE++data BooleanOperator = And+ | Or+ | Not+ DERIVE++data Operator = Neg+ | Add+ | Subtract+ | Multiply+ | Divide+ | Modulo+ DERIVE++data ComparisonOperator = Equal+ | NotEqual+ | Greater+ | Less+ | GreaterEqual+ | LessEqual+ DERIVE
+ src/Language/Pig/Parser/Parser.hs view
@@ -0,0 +1,314 @@+module Language.Pig.Parser.Parser (+ parseString+ , parseFile+ , module Language.Pig.Parser.AST+) where++import System.IO+import Text.ParserCombinators.Parsec+import Text.ParserCombinators.Parsec.Expr+import Text.ParserCombinators.Parsec.Language+import qualified Text.ParserCombinators.Parsec.Token as Token+import Data.List (intercalate)+import Control.Applicative ((<$>), (<*>), (*>), (<*))++import Language.Pig.Parser.AST+++-- Lexer++specialChar = oneOf "_"++pigLanguageDef :: LanguageDef st+pigLanguageDef = emptyDef {+ Token.commentStart = "/*"+ , Token.commentEnd = "*/"+ , Token.commentLine = "--"+ , Token.nestedComments = True+ , Token.identStart = letter+ , Token.identLetter = alphaNum <|> specialChar+ , Token.reservedNames = ["LOAD", "USING", "AS", -- todo case insensitivity of these keywords+ "FOREACH", "GENERATE", "FLATTEN",+ "JOIN", "BY",+ "GROUP",+ "DESCRIBE", "SHIP",+ "DEFINE",+ "STREAM", "THROUGH",+ "STORE", "INTO", "USING",+ "int", "long", "float", "double", "chararray", "bytearray", "*"]+ , Token.reservedOpNames = ["=", "+", "-", "*", "/", "%", "?", ":"]+ , Token.caseSensitive = False+ }++lexer = Token.makeTokenParser pigLanguageDef++identifier = Token.identifier lexer+reserved = Token.reserved lexer+reservedOp = Token.reservedOp lexer+integer = Token.integer lexer+naturalOrFloat = Token.naturalOrFloat lexer+semi = Token.semi lexer+comma = Token.comma lexer+whiteSpace = Token.whiteSpace lexer+parens = Token.parens lexer+lexeme = Token.lexeme lexer++-- parser to handle double colon.+pigIdentifier = try(detailedIdentifier) <|> identifier+detailedIdentifier :: Parser String+detailedIdentifier = lexeme $+ (intercalate "::") <$> sepBy1 identifierPart (string "::")+identifierPart = (:) <$> letter <*> many1 (alphaNum <|> specialChar)+++-- Parser: top-down++parseString :: [Char] -> Root+parseString input = case parsePig input of+ Left msg -> error (show msg)+ Right p -> p++parseFile :: FilePath -> IO Root+parseFile filename = parseString <$> readFile (filename)++parsePig :: String -> Either ParseError Root+parsePig input = parse pigParser "pigParser error" input++pigParser :: Parser Root+pigParser = whiteSpace >> statements++statements :: Parser Root+statements = Seq <$> endBy statement semi++statement :: Parser Statement+statement = query <|> describe <|> define <|> store++query :: Parser Statement+query = Assignment <$> + (Identifier <$> identifier) <*>+ (reservedOp "=" *> opClause)++describe :: Parser Statement+describe = Describe <$> (reserved "DESCRIBE" *> pigVar)++define :: Parser Statement+define = DefineUDF <$>+ (reserved "define" *>+ pigVar) <*>+ executable <*>+ shipClause -- could be input, output, ship, cache, stderr in full pig grammar++store :: Parser Statement+store = Store <$>+ (reserved "STORE" *>+ pigVar) <*>+ (reserved "INTO" *>+ pigQuotedString Directory) <*>+ (reserved "USING" *>+ pigFunc)+++opClause :: Parser OpClause+opClause = loadClause+ <|> foreachClause+ <|> innerJoinClause+ <|> groupClause+ <|> streamClause++loadClause :: Parser OpClause+loadClause = LoadClause <$>+ (reserved "LOAD" *>+ pigQuotedString Filename) <*>+ (reserved "USING" *>+ pigFunc) <*>+ (reserved "AS" *>+ pigTupleDef)++-- foreach: only the block (outer bag) version+foreachClause :: Parser OpClause+foreachClause = ForeachClause <$>+ (reserved "FOREACH" *>+ pigVar) <*>+ (reserved "GENERATE" *>+ (GenBlock <$> sepBy transform comma))++innerJoinClause :: Parser OpClause+innerJoinClause = InnerJoinClause <$>+ (reserved "JOIN" *>+ sepBy joinTable comma)++groupClause :: Parser OpClause+groupClause = GroupClause <$>+ (reserved "GROUP" *>+ pigVar) <*>+ (reserved "BY" *>+ (MultipleColumn <$> tuple <|> SingleColumn <$> name))++streamClause :: Parser OpClause+streamClause = StreamClause <$>+ (reserved "STREAM" *>+ pigVar) <*>+ (reserved "THROUGH" *>+ pigVar) <*>+ (reserved "AS" *>+ pigTupleDef)++joinTable :: Parser Join+joinTable = Join <$>+ pigIdentifier <*>+ (reserved "BY" *>+ pigIdentifier)++shipClause :: Parser DefineSpec+shipClause = Ship . Filename <$> + (reserved "SHIP" *>+ parens quotedString)++pigVar :: Parser Alias+pigVar = Identifier <$> pigIdentifier++pigQuotedString :: (String -> a) -> Parser a+pigQuotedString constructor = constructor <$> quotedString++pigFunc :: Parser Function+pigFunc = Function <$>+ identifier <*>+ parens arguments++arguments :: Parser [Argument]+arguments = sepBy argument comma++argument :: Parser Argument+argument = (StringArgument . String <$> quotedString) <|> + (AliasArgument <$> pigVar)++quotedString :: Parser String+quotedString = (char '\'' *> (many $ noneOf "\'")) <* char '\'' <* whiteSpace -- doesn't take into account escaped quotes++executable :: Parser Command+executable = Exec <$> (char '`' *> (many $ noneOf "`") <* char '`' <* whiteSpace)++pigTupleDef :: Parser TupleDef+pigTupleDef = TupleDef <$> parens tupleDef++tupleDef :: Parser [Field]+tupleDef = sepBy field comma++field :: Parser Field+field = Field <$>+ pigVar+ <* char ':'+ <*> pigType++pigType :: Parser SimpleType+pigType = pigSimpleType "int" Int <|>+ pigSimpleType "long" Long <|>+ pigSimpleType "float" Float <|>+ pigSimpleType "double" Double <|>+ pigSimpleType "chararray" CharArray <|>+ pigSimpleType "bytearray" ByteArray++pigSimpleType :: String -> SimpleType -> Parser SimpleType+pigSimpleType typeString constructor = reserved typeString >> return constructor+ +transform :: Parser Transform+transform = try(aliasTransform)+ <|> flattenTransform+ <|> tupleFieldGlob+ <|> expressionTransform+ <|> functionTransform+ <|> envTransform++flattenTransform :: Parser Transform+flattenTransform = Flatten <$>+ (reserved "FLATTEN" *>+ parens pigIdentifier) <*>+ (reserved "AS" *>+ tuple)++expressionTransform :: Parser Transform+expressionTransform = ExpressionTransform <$>+ generalExpression <*>+ (reserved "AS" *>+ (Identifier <$> identifier))++functionTransform :: Parser Transform+functionTransform = FunctionTransform <$>+ pigFunc <*>+ (reserved "AS" *>+ (Identifier <$> identifier))++aliasTransform :: Parser Transform+aliasTransform = AliasTransform <$>+ (Identifier <$> pigIdentifier) <*>+ (reserved "AS" *>+ (Identifier <$> identifier))++envTransform :: Parser Transform+envTransform = EnvTransform <$>+ pigQuotedString String <*>+ (reserved "AS" *>+ (Identifier <$> identifier))++-- general expression:+-- fieldExpression or literal or function or binary operation (+-*/%) or bincond (?:)+-- bincond: boolean expression (==, !=, >, <, >=, <=) (and, or, not)+generalExpression :: Parser Expression+generalExpression = parens calculation++-- conditional is ternary operator, so lookahead to try and parse it first.+calculation :: Parser Expression+calculation = try(conditional) <|> buildExpressionParser pigOperators pigTerm++pigOperators = [[Prefix (reservedOp "-" >> return (Unary Neg))]+ ,[Infix (reservedOp "*" >> return (Binary Multiply)) AssocLeft]+ ,[Infix (reservedOp "/" >> return (Binary Divide)) AssocLeft]+ ,[Infix (reservedOp "%" >> return (Binary Modulo)) AssocLeft]+ ,[Infix (reservedOp "+" >> return (Binary Add)) AssocLeft]+ ,[Infix (reservedOp "-" >> return (Binary Subtract)) AssocLeft]]++pigTerm :: Parser Expression+pigTerm = (ScalarTerm . String <$> quotedString)+ <|> (ScalarTerm <$> number)+ <|> generalExpression+ <|> (AliasTerm <$> name)++number = Number <$> naturalOrFloat -- for now - could be naturalOrFloat for inclusion++conditional :: Parser Expression+conditional = BinCond <$>+ booleanExpression <*>+ (reserved "?" *>+ calculation) <*>+ (reserved ":" *>+ calculation)++booleanExpression = buildExpressionParser booleanOperators booleanTerm++booleanTerm = parens booleanExpression+ <|> comparisonExpression++booleanOperators = [ [Prefix (reservedOp "not" >> return (BooleanUnary Not))]+ , [Infix (reservedOp "and" >> return (BooleanBinary And)) AssocLeft]+ , [Infix (reservedOp "or" >> return (BooleanBinary Or)) AssocLeft]]++comparisonExpression :: Parser BooleanExpression+comparisonExpression = flippedBooleanExpression <$> pigTerm <*> relation <*> pigTerm+ where flippedBooleanExpression expr1 op expr2 = BooleanExpression op expr1 expr2++-- bincond: boolean expression (==, !=, >, <, >=, <=) (and, or, not)+relation = (reservedOp ">" >> return Greater) <|>+ (reservedOp "<" >> return Less) <|>+ (reservedOp "<=" >> return LessEqual) <|>+ (reservedOp ">=" >> return GreaterEqual) <|>+ (reservedOp "==" >> return Equal) <|>+ (reservedOp "!=" >> return NotEqual)++tupleFieldGlob :: Parser Transform+tupleFieldGlob = reserved "*" >> return TupleFieldGlob++tuple :: Parser Tuple+tuple = Tuple <$> parens (sepBy name comma)++name :: Parser Alias+name = Identifier <$> pigIdentifier
+ src/Language/Pig/Pretty.hs view
@@ -0,0 +1,116 @@+{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}+module Language.Pig.Pretty+ ( prettyPrint )+where++import Language.Pig.Parser.Parser+import Text.ParserCombinators.Parsec++import Data.List (intercalate)+import Data.Tree+import Data.Tree.Pretty++-- patterns:+-- node name + list nodes+-- node name + literal to display = terminalnode+-- node name + arguments used as lists to use as nodes++class Treeable a where+ toTree :: a -> (Tree String)++instance Treeable Root where+ toTree (Seq stmts) = Node "sequence of statements" (map toTree stmts)++instance Treeable Statement where+ toTree (Assignment a b) = Node "assignment" [toTree a, toTree b]+ toTree (Describe a) = Node "describe statement" [toTree a]+ toTree (DefineUDF a b c) = Node "define UDF statement" [toTree a, toTree b, toTree c]+ toTree (Store a b c) = Node "store statement" [toTree a, toTree b, toTree c]++instance Treeable OpClause where+ toTree (LoadClause a b c) = Node "LOAD clause" [toTree a, toTree b, toTree c]+ toTree (ForeachClause a b) = Node "FOREACH clause" [toTree a, toTree b]+ toTree (GroupClause a b) = Node "GROUP clause" [toTree a, toTree b]+ toTree (InnerJoinClause joins) = Node "JOIN clause" (map toTree joins)+ toTree (StreamClause a b c) = Node "STREAM clause" [toTree a, toTree b, toTree c]++instance Treeable GenBlock where+ toTree (GenBlock transforms) = Node "transformation block" (map toTree transforms)++instance Treeable GroupBy where+ toTree (SingleColumn a) = Node "group by" [toTree a]+ toTree (MultipleColumn a) = Node "group by" [toTree a]++instance Treeable Transform where+ toTree (Flatten a b) = Node ("FLATTEN: " ++ a) [toTree b]+ toTree (TupleFieldGlob) = Node "*" []+ toTree (AliasTransform a b) = Node "alias" [toTree a, toTree b]+ toTree (ExpressionTransform a b) = Node "calculate" [toTree a, toTree b]+ toTree (FunctionTransform a b) = Node "function expression" [toTree a, toTree b]+ toTree (EnvTransform a b) = Node "name variable" [toTree a, toTree b]++instance Treeable Join where+ toTree (Join a b) = Node ("join " ++ a ++ " by " ++ b) []++instance Treeable DefineSpec where+ toTree (Ship p) = Node "SHIP" [toTree p]++instance Treeable Alias where+ toTree (Identifier s) = Node ("identifier: " ++ s) []++instance Treeable Path where+ toTree (Filename s) = Node ("filename: \"" ++ s ++ "\"") []+ toTree (Directory s) = Node ("directory: \"" ++ s ++ "\"") []++instance Treeable Command where+ toTree (Exec s) = Node ("execute command: " ++ s) []++instance Treeable Function where+ toTree (Function s a) = Node ("function " ++ s) (map toTree a)++instance Treeable Argument where+ toTree (StringArgument (String s)) = Node ("string argument: \"" ++ s ++ "\"") []+ toTree (StringArgument (Number s)) = Node ("number argument: " ++ show s) []+ toTree (AliasArgument (Identifier s)) = Node ("identifier argument: \"" ++ s ++ "\"") []++instance Treeable TupleDef where+ toTree (TupleDef f) = Node "tuple def" (map toTree f)++instance Treeable Tuple where+ toTree (Tuple t) = Node "tuple" (map toTree t)++instance Treeable Field where+ toTree (Field (Identifier s) t) = Node ("field: " ++ s ++ " of type " ++ show t) []++instance Treeable Expression where+ toTree (Unary o e) = Node "unary expression" [toTree o, toTree e]+ toTree (Binary o e1 e2) = Node "binary expression" [toTree o, toTree e1, toTree e2]+ toTree (BinCond e1 e2 e3) = Node "ternary conditional expression" [toTree e1, toTree e2, toTree e3]+ toTree (ScalarTerm (String s)) = Node ("scalar: string " ++ s) []+ toTree (ScalarTerm number) = toTree number+ toTree (AliasTerm alias) = toTree alias++instance Treeable BooleanExpression where+ toTree (BooleanExpression o e1 e2) = Node "comparison expression" [toTree o, toTree e1, toTree e2]+ toTree (BooleanUnary o e) = Node "boolean unary expression" [toTree o, toTree e]+ toTree (BooleanBinary o e1 e2) = Node "boolean binary expression" [toTree o, toTree e1, toTree e2]++instance Treeable Scalar where+ toTree (Number (Right i)) = Node ("integer:" ++ show i) []+ toTree (Number (Left f)) = Node ("double:" ++ show f) []+ toTree (String s) = Node ("string: \"" ++ s ++ "\"") []++instance Treeable SimpleType where+ toTree c = Node (show c) []++instance Treeable Operator where+ toTree c = Node (show c) []++instance Treeable BooleanOperator where+ toTree c = Node (show c) []++instance Treeable ComparisonOperator where+ toTree c = Node (show c) []++prettyPrint :: Root -> String+prettyPrint ast = drawVerticalTree $ toTree ast
+ test/tests.hs view
@@ -0,0 +1,11 @@+module Main+ where++import Test.Framework (defaultMain)++import Language.Pig.Parser.Test+import Language.Pig.Pretty.Test+++main :: IO ()+main = defaultMain [parserSuite, prettyPrintSuite]