queryparser-hive 0.1.0.0 → 0.1.0.1
raw patch · 6 files changed
+177/−57 lines, 6 filesdep ~base
Dependency ranges changed: base
Files
- CHANGELOG.md +6/−0
- queryparser-hive.cabal +5/−5
- src/Database/Sql/Hive/Parser.hs +44/−26
- src/Database/Sql/Hive/Parser/Token.hs +26/−21
- src/Database/Sql/Hive/Scanner.hs +1/−0
- src/Database/Sql/Hive/Type.hs +95/−5
+ CHANGELOG.md view
@@ -0,0 +1,6 @@+# 0.1 to 0.2++## [Hive] 0.1.0 to 0.1.1+- Added support for SET statements+- Added support for table properties+- Upgraded to GHC8, base 4.9
queryparser-hive.cabal view
@@ -6,7 +6,7 @@ -- PVP summary: +-+------- breaking API changes -- | | +----- non-breaking API additions -- | | | +--- code changes with no API change-version: 0.1.0.0+version: 0.1.0.1 -- A short (one-line) description of the package. synopsis: Parsing for Hive SQL queries@@ -41,7 +41,7 @@ -- Extra files to be distributed with the package, such as examples or a -- README.--- extra-source-files:+extra-source-files: CHANGELOG.md -- Constraint on the version of Cabal needed to build this package. cabal-version: >=1.10@@ -62,7 +62,7 @@ , Database.Sql.Hive.Parser.Internal -- Modules included in this library but not exported.- -- other-modules: + -- other-modules: default-extensions: OverloadedStrings , LambdaCase@@ -72,7 +72,7 @@ , FlexibleInstances -- Other library packages from which modules are imported.- build-depends: base >=4.8 && <4.9+ build-depends: base >=4.9 && <4.11 , text >=1.2 && <1.3 , bytestring , queryparser@@ -94,7 +94,7 @@ hs-source-dirs: src - ghc-options: -Wall+ ghc-options: -Wall -Wno-redundant-constraints if flag(development) ghc-options: -Werror
src/Database/Sql/Hive/Parser.hs view
@@ -48,7 +48,7 @@ import qualified Data.Set as S import qualified Data.List as L -import Data.Maybe (fromMaybe)+import Data.Maybe (fromMaybe, listToMaybe) import Data.Monoid (Endo (..)) import qualified Text.Parsec as P import Text.Parsec ( chainl1, choice, many@@ -92,7 +92,7 @@ , try $ HiveAlterTableSetLocationStmt <$> alterTableSetLocationP , try $ HiveUnhandledStatement <$> alterTableSetTblPropertiesP , alterPartitionP- , HiveUnhandledStatement <$> setP+ , HiveSetPropertyStmt <$> setP , HiveUnhandledStatement <$> reloadFunctionP ] case maybeStmt of@@ -463,17 +463,18 @@ ] -setP :: Parser Range+setP :: Parser (SetProperty Range) setP = do s <- Tok.setP- e <- option s $ choice- [ Tok.propertyNameP >> Tok.equalP >> last <$> propertyValueP- , Tok.symbolP "-" >> Tok.keywordP "v"- ]- return $ s <> e- where- propertyValueP = P.many1 Tok.notSemicolonP-+ option (PrintProperties s "") $ choice $+ [ Tok.minusP >> Tok.keywordP "v" >> pure (PrintProperties s "-v")+ , do+ (name, _) <- Tok.propertyNameP+ _ <- Tok.equalP+ (setConfigValue, e) <- Tok.propertyValuePartP+ let details = SetPropertyDetails (s <> e) name setConfigValue+ pure (SetProperty details)+ ] reloadFunctionP :: Parser Range reloadFunctionP = do@@ -882,12 +883,12 @@ return CreateTable{..} -propertyP :: Parser Range+propertyP :: Parser (HiveMetadataProperty Range) propertyP = do- s <- snd <$> Tok.stringP+ (k, s) <- Tok.stringP _ <- Tok.equalP- e <- snd <$> Tok.stringP- return $ s <> e+ (v, e) <- Tok.stringP+ return $ HiveMetadataProperty (s <> e) k v storedAsP :: Parser Range storedAsP = do@@ -929,7 +930,6 @@ , rowFormatP , storedAsP , getInfo <$> locationP- , tblPropertiesP ] createTableDefinition <- case tableDefColumns of@@ -939,10 +939,17 @@ , createTableNoColumnInfoP e2 ] - let createTableExtra = Nothing- e3 = getInfo createTableDefinition- createTableInfo = s <> e1 <> e2 <> e3+ tblProperties <- option Nothing (Just <$> tblPropertiesP) + let e3 = getInfo createTableDefinition+ e4 = fromMaybe e2 (hiveMetadataPropertiesInfo <$> tblProperties)+ createTableExtra =+ Just HiveCreateTableExtra+ { hiveCreateTableExtraInfo = e3 <> e4+ , hiveCreateTableExtraTableProperties = tblProperties+ }+ createTableInfo = s <> e1 <> e2 <> e3 <> e4+ pure CreateTable{..} where@@ -997,12 +1004,6 @@ _ <- Tok.formatP delimitedP <|> serdeP - tblPropertiesP = do- _ <- Tok.tblPropertiesP- _ <- Tok.openP- _ <- propertyP `sepBy1` Tok.commaP- Tok.closeP- createTableColumnsP = do s <- Tok.openP c:cs <- (ColumnOrConstraintColumn <$> columnDefinitionP) `sepBy1` Tok.commaP@@ -1020,6 +1021,17 @@ pure $ TableNoColumnInfo r +tblPropertiesP :: Parser (HiveMetadataProperties Range)+tblPropertiesP = do+ s <- Tok.tblPropertiesP+ _ <- Tok.openP+ l <- propertyP `sepBy1` Tok.commaP+ e <- Tok.closeP+ let hiveMetadataPropertiesInfo = s <> e+ hiveMetadataPropertiesProperties = l+ pure $ HiveMetadataProperties{..}++ delimitedP :: Parser Range delimitedP = do s <- Tok.delimitedP@@ -2034,8 +2046,14 @@ opP = choice $ map binOpP [ "+", "-" ] +stringExprP :: Parser (Expr RawNames Range)+stringExprP = sumExprP `chainl1` opP+ where+ opP = choice $ map binOpP [ "||" ]++ bitwiseAndExprP :: Parser (Expr RawNames Range)-bitwiseAndExprP = sumExprP `chainl1` binOpP "&"+bitwiseAndExprP = stringExprP `chainl1` binOpP "&" bitwiseOrExprP :: Parser (Expr RawNames Range)
src/Database/Sql/Hive/Parser/Token.hs view
@@ -166,29 +166,34 @@ _ -> Nothing +propertyValuePartP :: Parser (Text, Range)+propertyValuePartP = textUntilP [";"] --- a property name part may include dashes-propertyNamePartP :: Parser (Text, Range)-propertyNamePartP = do- xs <- namePartPartP `P.sepBy1` minusP- let namePartParts = map fst xs- ranges = map snd xs- namePart = intercalate "-" namePartParts- info = head ranges <> last ranges- return $ (namePart, info)- where- namePartPartP = P.tokenPrim showTok posFromTok testNameTok+-- Hive supports property names and values contianing+-- equal signs and spaces. Here we stop on the first equal sign.+propertyNameP :: Parser (Text, Range)+propertyNameP = textUntilP ["=", ";"] --- a property name is composed of name-parts separated by dots-propertyNameP :: Parser ([Text], Range)-propertyNameP = do- firstPart <- propertyNamePartP- subsequentParts <- P.many $ dotP >> propertyNamePartP- let allParts = firstPart:subsequentParts- s = snd $ head allParts- e = snd $ last allParts- nameParts = map fst allParts- return $ (nameParts, s <> e)+-- Parses all tokens until a token equal to a given string is found+-- returns parsed text.+textUntilP :: [Text] -> Parser (Text, Range)+textUntilP x = do+ res <- P.many anyTokenExceptX+ let name = Data.Text.Lazy.concat $ fst <$> res+ s = snd $ head res+ e = snd $ last res+ pure (name, s <> e)+ where+ anyTokenExceptX :: Parser (Text, Range)+ anyTokenExceptX = P.tokenPrim showTok posFromTok $+ \ (tok, s, e) -> case tok of+ TokSymbol t | not (t `elem` x) ->+ Just (t, Range s e)+ TokWord _ t | not (t `elem` x) ->+ Just (t, Range s e)+ TokNumber t | not (t `elem` x) ->+ Just (t, Range s e)+ _ -> Nothing keywordP :: Text -> Parser Range
src/Database/Sql/Hive/Scanner.hs view
@@ -49,6 +49,7 @@ operators :: [Text] operators = sortBy (flip compare) [ "+", "-", "*", "/", "%"+ , "||" , "&", "|", "^", "~" , "!" , ":"
src/Database/Sql/Hive/Type.hs view
@@ -27,6 +27,7 @@ {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE UndecidableInstances #-}+{-# LANGUAGE RecordWildCards #-} module Database.Sql.Hive.Type where @@ -47,6 +48,7 @@ import Data.Aeson import qualified Data.Map as M import qualified Data.Set as S+import Data.Text.Lazy (Text) import qualified Data.Text.Lazy.Encoding as TL import Data.ByteString.Lazy (ByteString) import qualified Data.ByteString.Lazy as BL@@ -64,10 +66,12 @@ dialectProxy = Proxy instance Dialect Hive where+ type DialectCreateTableExtra Hive r = HiveCreateTableExtra r shouldCTEsShadowTables _ = False - resolveCreateTableExtra _ Unused = pure Unused+ -- Nothing to resolve in the table extra+ resolveCreateTableExtra _ HiveCreateTableExtra{..} = pure HiveCreateTableExtra{..} getSelectScope _ fromColumns selectionAliases = SelectScope { bindForWhere = bindFromColumns fromColumns@@ -87,6 +91,7 @@ | HiveTruncatePartitionStmt (TruncatePartition r a) | HiveAlterTableSetLocationStmt (AlterTableSetLocation r a) | HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation r a)+ | HiveSetPropertyStmt (SetProperty a) | HiveUnhandledStatement a deriving instance (ConstrainSNames Data r a, Data r) => Data (HiveStatement r a)@@ -97,6 +102,40 @@ deriving instance ConstrainSASNames Foldable r => Foldable (HiveStatement r) deriving instance ConstrainSASNames Traversable r => Traversable (HiveStatement r) +data SetProperty a = SetProperty (SetPropertyDetails a)+ | PrintProperties a Text+ deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)++data SetPropertyDetails a = SetPropertyDetails+ { setPropertyDetailsInfo :: a+ , setPropertyDetailsName :: Text+ , setPropertyDetailsValue :: Text+ } deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)++data HiveCreateTableExtra r a = HiveCreateTableExtra+ { hiveCreateTableExtraInfo :: a+ , hiveCreateTableExtraTableProperties :: Maybe (HiveMetadataProperties a)+ }++deriving instance (ConstrainSNames Data r a, Data r) => Data (HiveCreateTableExtra r a)+deriving instance Generic (HiveCreateTableExtra r a)+deriving instance ConstrainSNames Eq r a => Eq (HiveCreateTableExtra r a)+deriving instance ConstrainSNames Show r a => Show (HiveCreateTableExtra r a)+deriving instance ConstrainSASNames Functor r => Functor (HiveCreateTableExtra r)+deriving instance ConstrainSASNames Foldable r => Foldable (HiveCreateTableExtra r)+deriving instance ConstrainSASNames Traversable r => Traversable (HiveCreateTableExtra r)++data HiveMetadataProperties a = HiveMetadataProperties+ { hiveMetadataPropertiesInfo :: a+ , hiveMetadataPropertiesProperties :: [HiveMetadataProperty a]+ } deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)++data HiveMetadataProperty a = HiveMetadataProperty+ { hiveMetadataPropertyInfo :: a+ , hiveMetadataPropertyKey :: ByteString+ , hiveMetadataPropertyValue :: ByteString+ } deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)+ -- Important terminology note: -- Hive "databases" are schemas. -- https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Create/Drop/Alter/UseDatabase@@ -215,6 +254,7 @@ getJoins (HiveTruncatePartitionStmt _) = S.empty getJoins (HiveAlterTableSetLocationStmt _) = S.empty getJoins (HiveAlterPartitionSetLocationStmt _) = S.empty+ getJoins (HiveSetPropertyStmt _) = S.empty getJoins (HiveUnhandledStatement _) = S.empty instance HasTableLineage (HiveStatement ResolvedNames a) where@@ -227,6 +267,7 @@ in M.singleton table $ S.singleton table getTableLineage (HiveAlterTableSetLocationStmt _) = M.empty getTableLineage (HiveAlterPartitionSetLocationStmt _) = M.empty+ getTableLineage (HiveSetPropertyStmt _) = M.empty getTableLineage (HiveUnhandledStatement _) = M.empty instance HasColumnLineage (HiveStatement ResolvedNames Range) where@@ -240,6 +281,7 @@ $ M.fromList $ map ((\ fqcn -> (Right fqcn, singleColumnSet (getInfo t) fqcn)) . fqcnToFQCN . qualifyColumnName t) columnsList getColumnLineage (HiveAlterTableSetLocationStmt _) = returnNothing M.empty getColumnLineage (HiveAlterPartitionSetLocationStmt _) = returnNothing M.empty+ getColumnLineage (HiveSetPropertyStmt _) = returnNothing M.empty getColumnLineage (HiveUnhandledStatement _) = returnNothing M.empty resolveHiveStatement :: HiveStatement RawNames a -> Resolver (HiveStatement ResolvedNames) a@@ -250,6 +292,7 @@ resolveHiveStatement (HiveTruncatePartitionStmt stmt) = HiveTruncatePartitionStmt <$> resolveTruncatePartition stmt resolveHiveStatement (HiveAlterTableSetLocationStmt stmt) = HiveAlterTableSetLocationStmt <$> resolveAlterTableSetLocation stmt resolveHiveStatement (HiveAlterPartitionSetLocationStmt stmt) = HiveAlterPartitionSetLocationStmt <$> resolveAlterPartitionSetLocation stmt+resolveHiveStatement (HiveSetPropertyStmt stmt) = pure $ HiveSetPropertyStmt stmt resolveHiveStatement (HiveUnhandledStatement stmt) = pure $ HiveUnhandledStatement stmt resolveAnalyze :: Analyze RawNames a -> Resolver (Analyze ResolvedNames) a@@ -314,6 +357,7 @@ getSchemaChange (HiveAlterTableSetLocationStmt _) = [] -- In fact, it may very well have a schema change but -- we can't know without hitting the metastore. getSchemaChange (HiveAlterPartitionSetLocationStmt _) = []+ getSchemaChange (HiveSetPropertyStmt _) = [] getSchemaChange (HiveUnhandledStatement _) = [] instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (HiveStatement r a) where@@ -342,6 +386,10 @@ [ "tag" .= String "HiveAlterPartitionSetLocationStmt" , "statement" .= alterPartitionSetLocation ]+ toJSON (HiveSetPropertyStmt stmt) = object+ [ "tag" .= String "HiveSetPropertyStmt"+ , "info" .= stmt+ ] toJSON (HiveUnhandledStatement info) = object [ "tag" .= String "HiveUnhandledStatement" , "info" .= info@@ -350,6 +398,33 @@ typeExample :: () typeExample = const () $ toJSON (undefined :: HiveStatement ResolvedNames Range) +instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (HiveCreateTableExtra r a) where+ toJSON HiveCreateTableExtra{..} = object+ [ "tag" .= String "HiveCreateTableExtra"+ , "info" .= hiveCreateTableExtraInfo+ , "properties" .= hiveCreateTableExtraTableProperties+ ]++instance ToJSON a => ToJSON (HiveMetadataProperties a) where+ toJSON HiveMetadataProperties{..} = object+ [ "tag" .= String "HiveMetadataProperties"+ , "info" .= hiveMetadataPropertiesInfo+ , "properties" .= hiveMetadataPropertiesProperties+ ]++instance ToJSON a => ToJSON (HiveMetadataProperty a) where+ toJSON HiveMetadataProperty{..} = object+ [ "tag" .= String "MetadataProperty"+ , "info" .= hiveMetadataPropertyInfo+ , "key" .= bsToJSON hiveMetadataPropertyKey+ , "value" .= bsToJSON hiveMetadataPropertyValue+ ]++bsToJSON :: ByteString -> Value+bsToJSON bs = case TL.decodeUtf8' bs of+ Left _ -> toJSON $ BL.unpack bs+ Right str -> toJSON str+ instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (InsertDirectory r a) where toJSON stmt = object [ "tag" .= String "InsertDirectory"@@ -360,10 +435,7 @@ ] instance ToJSON a => ToJSON (Location a) where- toJSON (HDFSPath _ p) =- case TL.decodeUtf8' p of- Left _ -> toJSON $ BL.unpack p- Right str -> toJSON str+ toJSON (HDFSPath _ p) = bsToJSON p instance ToJSON a => ToJSON (InsertDirectoryLocale a) where toJSON (InsertDirectoryLocal _) = String "Local"@@ -373,6 +445,22 @@ toJSON (UseDatabase dbn) = toJSON dbn toJSON (UseDefault _) = String "Default" +instance ToJSON a => ToJSON (SetProperty a) where+ toJSON (SetProperty details) = toJSON details+ toJSON (PrintProperties info t) = object+ [ "tag" .= String "Set"+ , "info" .= info+ , "property" .= t+ ]++instance ToJSON a => ToJSON (SetPropertyDetails a) where+ toJSON d@SetPropertyDetails{} = object+ [ "tag" .= String "Set"+ , "info" .= setPropertyDetailsInfo d+ , "property" .= setPropertyDetailsName d+ , "value" .= setPropertyDetailsValue d+ ]+ instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (Analyze r a) where toJSON (Analyze info tbn) = object [ "tag" .= String "Analyze"@@ -429,6 +517,7 @@ goTables (HiveTruncatePartitionStmt s) = goTables s goTables (HiveAlterTableSetLocationStmt s) = goTables s goTables (HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation _ (RTableName fqtn _) _ _)) = tell $ S.singleton $ TableUse WriteMeta $ fqtnToFQTN fqtn+ goTables (HiveSetPropertyStmt _) = return () goTables (HiveUnhandledStatement _) = return () instance HasTables (InsertDirectory ResolvedNames a) where@@ -452,6 +541,7 @@ goColumns (HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation _ _ items _)) = forM_ items $ \ (StaticPartitionSpecItem _ column _) -> tell [clauseObservation (void column) "PARTITION"]+ goColumns (HiveSetPropertyStmt _) = return () goColumns (HiveUnhandledStatement _) = return () instance HasColumns (InsertDirectory ResolvedNames a) where