diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# 0.1 to 0.2 + +## [Hive] 0.1.0 to 0.1.1 +- Added support for SET statements +- Added support for table properties +- Upgraded to GHC8, base 4.9 diff --git a/queryparser-hive.cabal b/queryparser-hive.cabal --- a/queryparser-hive.cabal +++ b/queryparser-hive.cabal @@ -6,7 +6,7 @@ -- PVP summary: +-+------- breaking API changes -- | | +----- non-breaking API additions -- | | | +--- code changes with no API change -version: 0.1.0.0 +version: 0.1.0.1 -- A short (one-line) description of the package. synopsis: Parsing for Hive SQL queries @@ -41,7 +41,7 @@ -- Extra files to be distributed with the package, such as examples or a -- README. --- extra-source-files: +extra-source-files: CHANGELOG.md -- Constraint on the version of Cabal needed to build this package. cabal-version: >=1.10 @@ -62,7 +62,7 @@ , Database.Sql.Hive.Parser.Internal -- Modules included in this library but not exported. - -- other-modules: + -- other-modules: default-extensions: OverloadedStrings , LambdaCase @@ -72,7 +72,7 @@ , FlexibleInstances -- Other library packages from which modules are imported. - build-depends: base >=4.8 && <4.9 + build-depends: base >=4.9 && <4.11 , text >=1.2 && <1.3 , bytestring , queryparser @@ -94,7 +94,7 @@ hs-source-dirs: src - ghc-options: -Wall + ghc-options: -Wall -Wno-redundant-constraints if flag(development) ghc-options: -Werror diff --git a/src/Database/Sql/Hive/Parser.hs b/src/Database/Sql/Hive/Parser.hs --- a/src/Database/Sql/Hive/Parser.hs +++ b/src/Database/Sql/Hive/Parser.hs @@ -48,7 +48,7 @@ import qualified Data.Set as S import qualified Data.List as L -import Data.Maybe (fromMaybe) +import Data.Maybe (fromMaybe, listToMaybe) import Data.Monoid (Endo (..)) import qualified Text.Parsec as P import Text.Parsec ( chainl1, choice, many @@ -92,7 +92,7 @@ , try $ HiveAlterTableSetLocationStmt <$> alterTableSetLocationP , try $ HiveUnhandledStatement <$> alterTableSetTblPropertiesP , alterPartitionP - , HiveUnhandledStatement <$> setP + , HiveSetPropertyStmt <$> setP , HiveUnhandledStatement <$> reloadFunctionP ] case maybeStmt of @@ -463,17 +463,18 @@ ] -setP :: Parser Range +setP :: Parser (SetProperty Range) setP = do s <- Tok.setP - e <- option s $ choice - [ Tok.propertyNameP >> Tok.equalP >> last <$> propertyValueP - , Tok.symbolP "-" >> Tok.keywordP "v" - ] - return $ s <> e - where - propertyValueP = P.many1 Tok.notSemicolonP - + option (PrintProperties s "") $ choice $ + [ Tok.minusP >> Tok.keywordP "v" >> pure (PrintProperties s "-v") + , do + (name, _) <- Tok.propertyNameP + _ <- Tok.equalP + (setConfigValue, e) <- Tok.propertyValuePartP + let details = SetPropertyDetails (s <> e) name setConfigValue + pure (SetProperty details) + ] reloadFunctionP :: Parser Range reloadFunctionP = do @@ -882,12 +883,12 @@ return CreateTable{..} -propertyP :: Parser Range +propertyP :: Parser (HiveMetadataProperty Range) propertyP = do - s <- snd <$> Tok.stringP + (k, s) <- Tok.stringP _ <- Tok.equalP - e <- snd <$> Tok.stringP - return $ s <> e + (v, e) <- Tok.stringP + return $ HiveMetadataProperty (s <> e) k v storedAsP :: Parser Range storedAsP = do @@ -929,7 +930,6 @@ , rowFormatP , storedAsP , getInfo <$> locationP - , tblPropertiesP ] createTableDefinition <- case tableDefColumns of @@ -939,10 +939,17 @@ , createTableNoColumnInfoP e2 ] - let createTableExtra = Nothing - e3 = getInfo createTableDefinition - createTableInfo = s <> e1 <> e2 <> e3 + tblProperties <- option Nothing (Just <$> tblPropertiesP) + let e3 = getInfo createTableDefinition + e4 = fromMaybe e2 (hiveMetadataPropertiesInfo <$> tblProperties) + createTableExtra = + Just HiveCreateTableExtra + { hiveCreateTableExtraInfo = e3 <> e4 + , hiveCreateTableExtraTableProperties = tblProperties + } + createTableInfo = s <> e1 <> e2 <> e3 <> e4 + pure CreateTable{..} where @@ -997,12 +1004,6 @@ _ <- Tok.formatP delimitedP <|> serdeP - tblPropertiesP = do - _ <- Tok.tblPropertiesP - _ <- Tok.openP - _ <- propertyP `sepBy1` Tok.commaP - Tok.closeP - createTableColumnsP = do s <- Tok.openP c:cs <- (ColumnOrConstraintColumn <$> columnDefinitionP) `sepBy1` Tok.commaP @@ -1020,6 +1021,17 @@ pure $ TableNoColumnInfo r +tblPropertiesP :: Parser (HiveMetadataProperties Range) +tblPropertiesP = do + s <- Tok.tblPropertiesP + _ <- Tok.openP + l <- propertyP `sepBy1` Tok.commaP + e <- Tok.closeP + let hiveMetadataPropertiesInfo = s <> e + hiveMetadataPropertiesProperties = l + pure $ HiveMetadataProperties{..} + + delimitedP :: Parser Range delimitedP = do s <- Tok.delimitedP @@ -2034,8 +2046,14 @@ opP = choice $ map binOpP [ "+", "-" ] +stringExprP :: Parser (Expr RawNames Range) +stringExprP = sumExprP `chainl1` opP + where + opP = choice $ map binOpP [ "||" ] + + bitwiseAndExprP :: Parser (Expr RawNames Range) -bitwiseAndExprP = sumExprP `chainl1` binOpP "&" +bitwiseAndExprP = stringExprP `chainl1` binOpP "&" bitwiseOrExprP :: Parser (Expr RawNames Range) diff --git a/src/Database/Sql/Hive/Parser/Token.hs b/src/Database/Sql/Hive/Parser/Token.hs --- a/src/Database/Sql/Hive/Parser/Token.hs +++ b/src/Database/Sql/Hive/Parser/Token.hs @@ -166,29 +166,34 @@ _ -> Nothing +propertyValuePartP :: Parser (Text, Range) +propertyValuePartP = textUntilP [";"] --- a property name part may include dashes -propertyNamePartP :: Parser (Text, Range) -propertyNamePartP = do - xs <- namePartPartP `P.sepBy1` minusP - let namePartParts = map fst xs - ranges = map snd xs - namePart = intercalate "-" namePartParts - info = head ranges <> last ranges - return $ (namePart, info) - where - namePartPartP = P.tokenPrim showTok posFromTok testNameTok +-- Hive supports property names and values contianing +-- equal signs and spaces. Here we stop on the first equal sign. +propertyNameP :: Parser (Text, Range) +propertyNameP = textUntilP ["=", ";"] --- a property name is composed of name-parts separated by dots -propertyNameP :: Parser ([Text], Range) -propertyNameP = do - firstPart <- propertyNamePartP - subsequentParts <- P.many $ dotP >> propertyNamePartP - let allParts = firstPart:subsequentParts - s = snd $ head allParts - e = snd $ last allParts - nameParts = map fst allParts - return $ (nameParts, s <> e) +-- Parses all tokens until a token equal to a given string is found +-- returns parsed text. +textUntilP :: [Text] -> Parser (Text, Range) +textUntilP x = do + res <- P.many anyTokenExceptX + let name = Data.Text.Lazy.concat $ fst <$> res + s = snd $ head res + e = snd $ last res + pure (name, s <> e) + where + anyTokenExceptX :: Parser (Text, Range) + anyTokenExceptX = P.tokenPrim showTok posFromTok $ + \ (tok, s, e) -> case tok of + TokSymbol t | not (t `elem` x) -> + Just (t, Range s e) + TokWord _ t | not (t `elem` x) -> + Just (t, Range s e) + TokNumber t | not (t `elem` x) -> + Just (t, Range s e) + _ -> Nothing keywordP :: Text -> Parser Range diff --git a/src/Database/Sql/Hive/Scanner.hs b/src/Database/Sql/Hive/Scanner.hs --- a/src/Database/Sql/Hive/Scanner.hs +++ b/src/Database/Sql/Hive/Scanner.hs @@ -49,6 +49,7 @@ operators :: [Text] operators = sortBy (flip compare) [ "+", "-", "*", "/", "%" + , "||" , "&", "|", "^", "~" , "!" , ":" diff --git a/src/Database/Sql/Hive/Type.hs b/src/Database/Sql/Hive/Type.hs --- a/src/Database/Sql/Hive/Type.hs +++ b/src/Database/Sql/Hive/Type.hs @@ -27,6 +27,7 @@ {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE RecordWildCards #-} module Database.Sql.Hive.Type where @@ -47,6 +48,7 @@ import Data.Aeson import qualified Data.Map as M import qualified Data.Set as S +import Data.Text.Lazy (Text) import qualified Data.Text.Lazy.Encoding as TL import Data.ByteString.Lazy (ByteString) import qualified Data.ByteString.Lazy as BL @@ -64,10 +66,12 @@ dialectProxy = Proxy instance Dialect Hive where + type DialectCreateTableExtra Hive r = HiveCreateTableExtra r shouldCTEsShadowTables _ = False - resolveCreateTableExtra _ Unused = pure Unused + -- Nothing to resolve in the table extra + resolveCreateTableExtra _ HiveCreateTableExtra{..} = pure HiveCreateTableExtra{..} getSelectScope _ fromColumns selectionAliases = SelectScope { bindForWhere = bindFromColumns fromColumns @@ -87,6 +91,7 @@ | HiveTruncatePartitionStmt (TruncatePartition r a) | HiveAlterTableSetLocationStmt (AlterTableSetLocation r a) | HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation r a) + | HiveSetPropertyStmt (SetProperty a) | HiveUnhandledStatement a deriving instance (ConstrainSNames Data r a, Data r) => Data (HiveStatement r a) @@ -97,6 +102,40 @@ deriving instance ConstrainSASNames Foldable r => Foldable (HiveStatement r) deriving instance ConstrainSASNames Traversable r => Traversable (HiveStatement r) +data SetProperty a = SetProperty (SetPropertyDetails a) + | PrintProperties a Text + deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable) + +data SetPropertyDetails a = SetPropertyDetails + { setPropertyDetailsInfo :: a + , setPropertyDetailsName :: Text + , setPropertyDetailsValue :: Text + } deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable) + +data HiveCreateTableExtra r a = HiveCreateTableExtra + { hiveCreateTableExtraInfo :: a + , hiveCreateTableExtraTableProperties :: Maybe (HiveMetadataProperties a) + } + +deriving instance (ConstrainSNames Data r a, Data r) => Data (HiveCreateTableExtra r a) +deriving instance Generic (HiveCreateTableExtra r a) +deriving instance ConstrainSNames Eq r a => Eq (HiveCreateTableExtra r a) +deriving instance ConstrainSNames Show r a => Show (HiveCreateTableExtra r a) +deriving instance ConstrainSASNames Functor r => Functor (HiveCreateTableExtra r) +deriving instance ConstrainSASNames Foldable r => Foldable (HiveCreateTableExtra r) +deriving instance ConstrainSASNames Traversable r => Traversable (HiveCreateTableExtra r) + +data HiveMetadataProperties a = HiveMetadataProperties + { hiveMetadataPropertiesInfo :: a + , hiveMetadataPropertiesProperties :: [HiveMetadataProperty a] + } deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable) + +data HiveMetadataProperty a = HiveMetadataProperty + { hiveMetadataPropertyInfo :: a + , hiveMetadataPropertyKey :: ByteString + , hiveMetadataPropertyValue :: ByteString + } deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable) + -- Important terminology note: -- Hive "databases" are schemas. -- https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Create/Drop/Alter/UseDatabase @@ -215,6 +254,7 @@ getJoins (HiveTruncatePartitionStmt _) = S.empty getJoins (HiveAlterTableSetLocationStmt _) = S.empty getJoins (HiveAlterPartitionSetLocationStmt _) = S.empty + getJoins (HiveSetPropertyStmt _) = S.empty getJoins (HiveUnhandledStatement _) = S.empty instance HasTableLineage (HiveStatement ResolvedNames a) where @@ -227,6 +267,7 @@ in M.singleton table $ S.singleton table getTableLineage (HiveAlterTableSetLocationStmt _) = M.empty getTableLineage (HiveAlterPartitionSetLocationStmt _) = M.empty + getTableLineage (HiveSetPropertyStmt _) = M.empty getTableLineage (HiveUnhandledStatement _) = M.empty instance HasColumnLineage (HiveStatement ResolvedNames Range) where @@ -240,6 +281,7 @@ $ M.fromList $ map ((\ fqcn -> (Right fqcn, singleColumnSet (getInfo t) fqcn)) . fqcnToFQCN . qualifyColumnName t) columnsList getColumnLineage (HiveAlterTableSetLocationStmt _) = returnNothing M.empty getColumnLineage (HiveAlterPartitionSetLocationStmt _) = returnNothing M.empty + getColumnLineage (HiveSetPropertyStmt _) = returnNothing M.empty getColumnLineage (HiveUnhandledStatement _) = returnNothing M.empty resolveHiveStatement :: HiveStatement RawNames a -> Resolver (HiveStatement ResolvedNames) a @@ -250,6 +292,7 @@ resolveHiveStatement (HiveTruncatePartitionStmt stmt) = HiveTruncatePartitionStmt <$> resolveTruncatePartition stmt resolveHiveStatement (HiveAlterTableSetLocationStmt stmt) = HiveAlterTableSetLocationStmt <$> resolveAlterTableSetLocation stmt resolveHiveStatement (HiveAlterPartitionSetLocationStmt stmt) = HiveAlterPartitionSetLocationStmt <$> resolveAlterPartitionSetLocation stmt +resolveHiveStatement (HiveSetPropertyStmt stmt) = pure $ HiveSetPropertyStmt stmt resolveHiveStatement (HiveUnhandledStatement stmt) = pure $ HiveUnhandledStatement stmt resolveAnalyze :: Analyze RawNames a -> Resolver (Analyze ResolvedNames) a @@ -314,6 +357,7 @@ getSchemaChange (HiveAlterTableSetLocationStmt _) = [] -- In fact, it may very well have a schema change but -- we can't know without hitting the metastore. getSchemaChange (HiveAlterPartitionSetLocationStmt _) = [] + getSchemaChange (HiveSetPropertyStmt _) = [] getSchemaChange (HiveUnhandledStatement _) = [] instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (HiveStatement r a) where @@ -342,6 +386,10 @@ [ "tag" .= String "HiveAlterPartitionSetLocationStmt" , "statement" .= alterPartitionSetLocation ] + toJSON (HiveSetPropertyStmt stmt) = object + [ "tag" .= String "HiveSetPropertyStmt" + , "info" .= stmt + ] toJSON (HiveUnhandledStatement info) = object [ "tag" .= String "HiveUnhandledStatement" , "info" .= info @@ -350,6 +398,33 @@ typeExample :: () typeExample = const () $ toJSON (undefined :: HiveStatement ResolvedNames Range) +instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (HiveCreateTableExtra r a) where + toJSON HiveCreateTableExtra{..} = object + [ "tag" .= String "HiveCreateTableExtra" + , "info" .= hiveCreateTableExtraInfo + , "properties" .= hiveCreateTableExtraTableProperties + ] + +instance ToJSON a => ToJSON (HiveMetadataProperties a) where + toJSON HiveMetadataProperties{..} = object + [ "tag" .= String "HiveMetadataProperties" + , "info" .= hiveMetadataPropertiesInfo + , "properties" .= hiveMetadataPropertiesProperties + ] + +instance ToJSON a => ToJSON (HiveMetadataProperty a) where + toJSON HiveMetadataProperty{..} = object + [ "tag" .= String "MetadataProperty" + , "info" .= hiveMetadataPropertyInfo + , "key" .= bsToJSON hiveMetadataPropertyKey + , "value" .= bsToJSON hiveMetadataPropertyValue + ] + +bsToJSON :: ByteString -> Value +bsToJSON bs = case TL.decodeUtf8' bs of + Left _ -> toJSON $ BL.unpack bs + Right str -> toJSON str + instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (InsertDirectory r a) where toJSON stmt = object [ "tag" .= String "InsertDirectory" @@ -360,10 +435,7 @@ ] instance ToJSON a => ToJSON (Location a) where - toJSON (HDFSPath _ p) = - case TL.decodeUtf8' p of - Left _ -> toJSON $ BL.unpack p - Right str -> toJSON str + toJSON (HDFSPath _ p) = bsToJSON p instance ToJSON a => ToJSON (InsertDirectoryLocale a) where toJSON (InsertDirectoryLocal _) = String "Local" @@ -373,6 +445,22 @@ toJSON (UseDatabase dbn) = toJSON dbn toJSON (UseDefault _) = String "Default" +instance ToJSON a => ToJSON (SetProperty a) where + toJSON (SetProperty details) = toJSON details + toJSON (PrintProperties info t) = object + [ "tag" .= String "Set" + , "info" .= info + , "property" .= t + ] + +instance ToJSON a => ToJSON (SetPropertyDetails a) where + toJSON d@SetPropertyDetails{} = object + [ "tag" .= String "Set" + , "info" .= setPropertyDetailsInfo d + , "property" .= setPropertyDetailsName d + , "value" .= setPropertyDetailsValue d + ] + instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (Analyze r a) where toJSON (Analyze info tbn) = object [ "tag" .= String "Analyze" @@ -429,6 +517,7 @@ goTables (HiveTruncatePartitionStmt s) = goTables s goTables (HiveAlterTableSetLocationStmt s) = goTables s goTables (HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation _ (RTableName fqtn _) _ _)) = tell $ S.singleton $ TableUse WriteMeta $ fqtnToFQTN fqtn + goTables (HiveSetPropertyStmt _) = return () goTables (HiveUnhandledStatement _) = return () instance HasTables (InsertDirectory ResolvedNames a) where @@ -452,6 +541,7 @@ goColumns (HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation _ _ items _)) = forM_ items $ \ (StaticPartitionSpecItem _ column _) -> tell [clauseObservation (void column) "PARTITION"] + goColumns (HiveSetPropertyStmt _) = return () goColumns (HiveUnhandledStatement _) = return () instance HasColumns (InsertDirectory ResolvedNames a) where