queryparser-hive (empty) → 0.1.0.0
raw patch · 8 files changed
+4274/−0 lines, 8 filesdep +QuickCheckdep +aesondep +base
Dependencies added: QuickCheck, aeson, base, bytestring, containers, fixed-list, hashable, mtl, parsec, predicate-class, pretty, queryparser, regex-tdfa, semigroups, text, unordered-containers, yaml
Files
- LICENSE +21/−0
- queryparser-hive.cabal +103/−0
- src/Database/Sql/Hive/Parser.hs +2302/−0
- src/Database/Sql/Hive/Parser/Internal.hs +41/−0
- src/Database/Sql/Hive/Parser/Token.hs +915/−0
- src/Database/Sql/Hive/Scanner.hs +317/−0
- src/Database/Sql/Hive/Token.hs +117/−0
- src/Database/Sql/Hive/Type.hs +458/−0
+ LICENSE view
@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2017 Uber Technologies, Inc.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
+ queryparser-hive.cabal view
@@ -0,0 +1,103 @@+name: queryparser-hive++-- The package version. See the Haskell package versioning policy (PVP)+-- for standards guiding when and how versions should be incremented.+-- http://www.haskell.org/haskellwiki/Package_versioning_policy+-- PVP summary: +-+------- breaking API changes+-- | | +----- non-breaking API additions+-- | | | +--- code changes with no API change+version: 0.1.0.0++-- A short (one-line) description of the package.+synopsis: Parsing for Hive SQL queries++-- A longer description of the package.+description:+ A library for parsing Hive SQL queries into analyzable ASTs.+ .+ This library is to be used with the queryparser library, which+ provides the common type definitions and analyses across the+ different SQL dialects.++-- The license under which the package is released.+license: MIT++-- The file containing the license text.+license-file: LICENSE++-- The package author(s).+author: Heli Wang, David Thomas, Matt Halverson++-- An email address to which users can send suggestions, bug reports, and+-- patches.+maintainer: heli@uber.com++-- A copyright notice.+-- copyright:++category: Database++build-type: Simple++-- Extra files to be distributed with the package, such as examples or a+-- README.+-- extra-source-files:++-- Constraint on the version of Cabal needed to build this package.+cabal-version: >=1.10++flag development {+ description: Enable development level of strictness+ default: False+ manual: True+}++library+ -- Modules exported by the library.+ exposed-modules: Database.Sql.Hive.Parser+ , Database.Sql.Hive.Parser.Token+ , Database.Sql.Hive.Scanner+ , Database.Sql.Hive.Token+ , Database.Sql.Hive.Type+ , Database.Sql.Hive.Parser.Internal++ -- Modules included in this library but not exported.+ -- other-modules: ++ default-extensions: OverloadedStrings+ , LambdaCase+ , RecordWildCards+ , TupleSections+ , ConstraintKinds+ , FlexibleInstances++ -- Other library packages from which modules are imported.+ build-depends: base >=4.8 && <4.9+ , text >=1.2 && <1.3+ , bytestring+ , queryparser+ , containers+ , semigroups >= 0.16+ , mtl >= 2.2 && < 2.3+ , parsec >= 3.1 && < 3.2+ , pretty >= 1.1 && < 1.2+ , aeson >= 0.8+ , yaml >= 0.8 && < 0.9+ , unordered-containers+ , hashable+ , QuickCheck+ , regex-tdfa+ , fixed-list+ , predicate-class++ -- Directories containing source files.+ hs-source-dirs: src+++ ghc-options: -Wall++ if flag(development)+ ghc-options: -Werror++ -- Base language which the package is written in.+ default-language: Haskell2010
+ src/Database/Sql/Hive/Parser.hs view
@@ -0,0 +1,2302 @@+-- Copyright (c) 2017 Uber Technologies, Inc.+--+-- Permission is hereby granted, free of charge, to any person obtaining a copy+-- of this software and associated documentation files (the "Software"), to deal+-- in the Software without restriction, including without limitation the rights+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+-- copies of the Software, and to permit persons to whom the Software is+-- furnished to do so, subject to the following conditions:+--+-- The above copyright notice and this permission notice shall be included in+-- all copies or substantial portions of the Software.+--+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+-- THE SOFTWARE.++{-# LANGUAGE DeriveFunctor #-}+{-# LANGUAGE DeriveTraversable #-}+{-# LANGUAGE StandaloneDeriving #-}+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE UndecidableInstances #-}++module Database.Sql.Hive.Parser where++import Database.Sql.Type+import Database.Sql.Info+import Database.Sql.Helpers+import Database.Sql.Hive.Type as HT++import Database.Sql.Hive.Scanner+import Database.Sql.Hive.Parser.Internal+import Database.Sql.Position++import qualified Database.Sql.Hive.Parser.Token as Tok++import Control.Monad (void)+import Control.Monad.Reader (runReader, local, asks)+import Data.Char (isDigit)+import Data.Text.Lazy (Text)+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Encoding as TL+import Data.Set (Set)+import qualified Data.Set as S+import qualified Data.List as L++import Data.Maybe (fromMaybe)+import Data.Monoid (Endo (..))+import qualified Text.Parsec as P+import Text.Parsec ( chainl1, choice, many+ , option, optional, optionMaybe+ , sepBy, sepBy1, try, (<|>), (<?>))+++import Data.Semigroup (Semigroup (..), sconcat)+import Data.List.NonEmpty (NonEmpty ((:|)))+import Data.Foldable (fold)++statementParser :: Parser (HiveStatement RawNames Range)+statementParser = do+ maybeStmt <- optionMaybe $ choice+ [ HiveUseStmt <$> useP+ , HiveAnalyzeStmt <$> analyzeP+ , do+ let options =+ -- this list is hive-specific statement types that may be+ -- preceded by an optional `WITH` and an optional inverted+ -- `FROM`+ [ (void insertDirectoryPrefixP, fmap HiveInsertDirectoryStmt . insertDirectoryP)+ ]+ prefixes = map fst options+ baseParsers = map snd options+ _ <- try $ P.lookAhead $ optional withP >> invertedFromP >> choice prefixes+ with <- option id withP+ invertedFrom <- invertedFromP+ let parsers = map ($ (with, invertedFrom)) baseParsers+ choice $ parsers+ , try $ HiveTruncatePartitionStmt <$> truncatePartitionStatementP+ , HiveUnhandledStatement <$> describeP+ , HiveUnhandledStatement <$> showP+ , do+ _ <- try $ P.lookAhead createFunctionPrefixP+ HiveUnhandledStatement <$> createFunctionP+ , do+ _ <- try $ P.lookAhead dropFunctionPrefixP+ HiveUnhandledStatement <$> dropFunctionP+ , HiveStandardSqlStatement <$> statementP+ , try $ HiveAlterTableSetLocationStmt <$> alterTableSetLocationP+ , try $ HiveUnhandledStatement <$> alterTableSetTblPropertiesP+ , alterPartitionP+ , HiveUnhandledStatement <$> setP+ , HiveUnhandledStatement <$> reloadFunctionP+ ]+ case maybeStmt of+ Just stmt -> terminator >> return stmt+ Nothing -> HiveStandardSqlStatement <$> emptyStatementP+ where+ terminator = (Tok.semicolonP <|> eof) -- normal statements may be terminated by `;` or eof+ emptyStatementP = EmptyStmt <$> Tok.semicolonP -- but we don't allow eof here. `;` is the+ -- only way to write the empty statement, i.e. `` (empty string) is not allowed.+++emptyParserScope :: ParserScope+emptyParserScope = ParserScope+ { selectTableAliases = Nothing }++-- | parse consumes a statement, or fails+parse :: Text -> Either P.ParseError (HiveStatement RawNames Range)+parse text = runReader (P.runParserT statementParser 0 "-" . tokenize $ text) emptyParserScope++-- | parseAll consumes all input as a single statement, or fails+parseAll :: Text -> Either P.ParseError (HiveStatement RawNames Range)+parseAll text = runReader (P.runParserT (statementParser <* P.eof) 0 "-" . tokenize $ text) emptyParserScope++-- | parseMany consumes multiple statements, or fails+parseMany :: Text -> Either P.ParseError [HiveStatement RawNames Range]+parseMany text = runReader (P.runParserT (P.many1 statementParser) 0 "-" . tokenize $ text) emptyParserScope++-- | parseManyAll consumes all input multiple statements, or fails+parseManyAll :: Text -> Either P.ParseError [HiveStatement RawNames Range]+parseManyAll text = runReader (P.runParserT (P.many1 statementParser <* P.eof) 0 "-" . tokenize $ text) emptyParserScope++-- | parseManyEithers consumes all input as multiple (statements or failures)+-- it should never fail+parseManyEithers :: Text -> Either P.ParseError [Either (Unparsed Range) (HiveStatement RawNames Range)]+parseManyEithers text = runReader (P.runParserT parser 0 "-" . tokenize $ text) emptyParserScope+ where+ parser = do+ statements <- P.many1 $ P.setState 0 >> choice+ [ try $ Right <$> statementParser+ , try $ Left <$> do+ ss <- many Tok.notSemicolonP+ e <- Tok.semicolonP+ pure $ case ss of+ [] -> Unparsed e+ s:_ -> Unparsed (s <> e)+ ]++ locs <- many Tok.notSemicolonP+ P.eof+ pure $ case locs of+ [] -> statements+ s:es -> statements ++ [Left $ Unparsed $ sconcat (s:|es)]++optionBool :: Parser a -> Parser Bool+optionBool p = option False $ p >> pure True++statementP :: Parser (Statement Hive RawNames Range)+statementP = choice+ [ do+ let options =+ -- this list is universal statement types that may be preceded+ -- by an optional `WITH` and an optional inverted `FROM`+ [ (void Tok.insertP, fmap InsertStmt . insertP)+ , (void Tok.selectP, fmap QueryStmt . queryP )+ ]+ prefixes = map fst options+ baseParsers = map snd options+ _ <- try $ P.lookAhead $ optional withP >> invertedFromP >> choice prefixes+ with <- option id withP+ invertedFrom <- invertedFromP+ let parsers = map ($ (with, invertedFrom)) baseParsers+ choice $ parsers+ , InsertStmt <$> loadDataInPathP+ , DeleteStmt <$> deleteP+ , explainP+ , TruncateStmt <$> truncateP+ , do+ _ <- try $ P.lookAhead createSchemaPrefixP+ CreateSchemaStmt <$> createSchemaP+ , do+ _ <- try $ P.lookAhead createViewPrefixP+ CreateViewStmt <$> createViewP+ , CreateTableStmt <$> createTableP+ , DropTableStmt <$> dropTableP+ , do+ _ <- try $ P.lookAhead alterTableRenameTablePrefixP+ AlterTableStmt <$> alterTableRenameTableP+ , do+ _ <- try $ P.lookAhead alterTableRenameColumnPrefixP+ AlterTableStmt <$> alterTableRenameColumnP+ , do+ _ <- try $ P.lookAhead alterTableAddColumnsPrefixP+ AlterTableStmt <$> alterTableAddColumnsP+ , GrantStmt <$> grantP+ , RevokeStmt <$> revokeP+ , CommitStmt <$> Tok.commitP+ , RollbackStmt <$> Tok.rollbackP+ ]++useP :: Parser (Use Range)+useP = do+ r <- Tok.useP++ use <- choice+ [ UseDefault <$> Tok.defaultP+ , UseDatabase . uncurry mkNormalSchema <$> Tok.schemaNameP+ ]++ return $ (r<>) <$> use+++analyzeP :: Parser (Analyze RawNames Range)+analyzeP = do+ r <- Tok.analyzeP+ _ <- Tok.tableP+ tn <- tableNameP+ optional $ do+ _ <- Tok.partitionP+ partitionSpecP++ _ <- Tok.computeP+ e <- Tok.statisticsP+ e' <- consumeOrderedOptions e $+ [ do+ _ <- Tok.forP+ Tok.columnsP+ , do+ _ <- Tok.cacheP+ Tok.metadataP+ , Tok.noScanP+ ]+ return $ Analyze (r<>e') tn+++insertDirectoryPrefixP :: Parser (Range, InsertDirectoryLocale Range, Location Range)+insertDirectoryPrefixP = do+ s <- Tok.insertP+ _ <- Tok.overwriteP+ insertDirectoryLocale <- insertDirectoryLocaleP+ insertDirectoryPath <- insertDirectoryPathP++ return (s, insertDirectoryLocale, insertDirectoryPath)++insertDirectoryP :: (QueryPrefix, InvertedFrom) -> Parser (InsertDirectory RawNames Range)+insertDirectoryP (with, farInvertedFrom) = do+ r <- Tok.insertP+ _ <- Tok.overwriteP+ insertDirectoryLocale <- insertDirectoryLocaleP+ insertDirectoryPath <- insertDirectoryPathP++ case farInvertedFrom of+ Just _ -> pure ()+ Nothing -> optional rowFormatP >> optional storedAsP++ insertDirectoryQuery <- case farInvertedFrom of+ Just _ -> querySelectP (with, farInvertedFrom)+ Nothing -> do+ nearInvertedFrom <- invertedFromP+ queryP (with, nearInvertedFrom)+ let insertDirectoryInfo = r <> (getInfo insertDirectoryQuery)+ return InsertDirectory{..}+ where+ rowFormatP :: Parser Range+ rowFormatP = do+ s <- Tok.rowP+ _ <- Tok.formatP+ e <- delimitedP+ return $ s <> e++insertDirectoryLocaleP :: Parser (InsertDirectoryLocale Range)+insertDirectoryLocaleP = do+ localToken <- optionMaybe Tok.localP+ let locale = case localToken of+ Just a -> InsertDirectoryLocal a+ Nothing -> InsertDirectoryHDFS+ return locale++insertDirectoryPathP :: Parser (Location Range)+insertDirectoryPathP = do+ r <- Tok.directoryP+ (path, r') <- Tok.stringP+ return $ HDFSPath (r <> r') path++staticPartitionSpecItemP :: Parser (StaticPartitionSpecItem RawNames Range)+staticPartitionSpecItemP = do+ col <- columnNameP+ _ <- Tok.equalP+ val <- constantP+ return $ StaticPartitionSpecItem (getInfo col <> getInfo val) col val++staticPartitionSpecP :: Parser ([StaticPartitionSpecItem RawNames Range], Range)+staticPartitionSpecP = do+ s <- Tok.openP+ items <- staticPartitionSpecItemP `sepBy1` Tok.commaP+ e <- Tok.closeP+ return (items, s <> e)++type PartitionDecider = (Either+ (StaticPartitionSpecItem RawNames Range)+ (DynamicPartitionSpecItem RawNames Range))++dynamicPartitionSpecItemP :: Parser (DynamicPartitionSpecItem RawNames Range)+dynamicPartitionSpecItemP = do+ col <- columnNameP+ return $ DynamicPartitionSpecItem (getInfo col) col++partitionSpecDeciderP :: Parser PartitionDecider+partitionSpecDeciderP = do+ item <- choice+ [ do+ sp <- try $ staticPartitionSpecItemP+ return $ Left sp+ , do+ dp <- dynamicPartitionSpecItemP+ return $ Right dp+ ]+ return item++partitionSpecP :: Parser ()+partitionSpecP = do+ -- partitionSpecP currently does not tie down to any specific datatype.+ -- The datatype implementation is being deferred.+ _ <- Tok.openP+ items <- partitionSpecDeciderP `sepBy1` Tok.commaP+ _ <- Tok.closeP+ let dpSpec = L.foldl' specHelper dpSpecBase $ L.reverse items+ case dpSpec of+ Right _ -> return ()+ Left err -> fail err+ where+ specHelper :: (Either String ([StaticPartitionSpecItem RawNames Range],+ [DynamicPartitionSpecItem RawNames Range])) ->+ PartitionDecider ->+ (Either String ([StaticPartitionSpecItem RawNames Range],+ [DynamicPartitionSpecItem RawNames Range]))+ -- Note that specHelper reads partition cols from right to left+ -- This allows for list insertions to output a non-reversed list+ specHelper (Right (spItems, dpItems)) (Left spItem) =+ Right (spItem:spItems, dpItems)++ specHelper (Right (spItems, dpItems)) (Right dpItem) =+ case spItems of+ [] -> Right $ (spItems, dpItem:dpItems)+ _ -> Left $ "Failed to parse partition \"" ++ show dpItem ++ "\": dynamic partition found preceding static partition"++ specHelper (Left s) _ = Left s++ dpSpecBase :: (Either String ([StaticPartitionSpecItem RawNames Range],+ [DynamicPartitionSpecItem RawNames Range]))+ dpSpecBase = Right ([], [])++truncatePartitionStatementP :: Parser (TruncatePartition RawNames Range)+truncatePartitionStatementP = do+ s <- Tok.truncateP+ _ <- Tok.tableP+ table <- tableNameP+ _ <- Tok.partitionP+ (_, e) <- staticPartitionSpecP++ let truncate' = Truncate (s <> getInfo table) table++ return $ TruncatePartition (s <> e) truncate'+++describeP :: Parser Range+describeP = do+ s <- Tok.describeP+ e <- P.many1 Tok.notSemicolonP+ return $ s <> last e+++showP :: Parser Range+showP = do+ s <- Tok.showP+ e <- P.many1 Tok.notSemicolonP+ return $ s <> last e+++createFunctionPrefixP :: Parser Range+createFunctionPrefixP = do+ s <- Tok.createP+ optional Tok.temporaryP+ e <- Tok.functionP+ return $ s <> e++createFunctionP :: Parser Range+createFunctionP = do+ s <- createFunctionPrefixP+ e <- P.many1 Tok.notSemicolonP+ return $ s <> last e+++dropFunctionPrefixP :: Parser Range+dropFunctionPrefixP = do+ s <- Tok.dropP+ optional Tok.temporaryP+ e <- Tok.functionP+ return $ s <> e++dropFunctionP :: Parser Range+dropFunctionP = do+ s <- dropFunctionPrefixP+ e <- P.many1 Tok.notSemicolonP+ return $ s <> last e++alterTableSetLocationP :: Parser (AlterTableSetLocation RawNames Range)+alterTableSetLocationP = do+ s <- Tok.alterP+ _ <- Tok.tableP+ table <- tableNameP+ _ <- Tok.setP+ loc <- locationP++ let alterTableSetLocationInfo = s <> getInfo loc+ alterTableSetLocationTable = table+ alterTableSetLocationLocation = loc++ return AlterTableSetLocation{..}++alterTableSetTblPropertiesP :: Parser Range+alterTableSetTblPropertiesP = do+ s <- Tok.alterP+ _ <- Tok.tableP+ _ <- tableNameP+ _ <- Tok.setP+ _ <- Tok.tblPropertiesP+ _ <- Tok.openP+ _ <- (Tok.stringP >> Tok.equalP >> Tok.stringP) `sepBy1` Tok.commaP+ e <- Tok.closeP++ return $ s <> e++alterPartitionP :: Parser (HiveStatement RawNames Range)+alterPartitionP = do+ s <- Tok.alterP+ _ <- Tok.tableP+ tableName <- tableNameP+ choice $+ [ do+ _ <- Tok.partitionP+ (items, _) <- staticPartitionSpecP+ _ <- Tok.setP+ location <- locationP+ pure $ HiveAlterPartitionSetLocationStmt $ AlterPartitionSetLocation (s <> getInfo location) tableName items location+ , HiveUnhandledStatement . (s <>) <$> (addP <|> dropP)+ ]++ where+ addP :: Parser Range+ addP = do+ _ <- Tok.addP+ _ <- ifNotExistsP++ let partitionLocationP = do+ _ <- Tok.partitionP+ (_, e) <- staticPartitionSpecP+ option e (getInfo <$> locationP)+ last <$> P.many1 partitionLocationP++ dropP :: Parser Range+ dropP = do+ _ <- Tok.dropP+ optional $ ifExistsP+ (_, e) <- last <$> P.many1 (Tok.partitionP >> staticPartitionSpecP)+ consumeOrderedOptions e $+ [ Tok.ignoreP >> Tok.protectionP+ , Tok.purgeP+ ]+++setP :: Parser Range+setP = do+ s <- Tok.setP+ e <- option s $ choice+ [ Tok.propertyNameP >> Tok.equalP >> last <$> propertyValueP+ , Tok.symbolP "-" >> Tok.keywordP "v"+ ]+ return $ s <> e+ where+ propertyValueP = P.many1 Tok.notSemicolonP+++reloadFunctionP :: Parser Range+reloadFunctionP = do+ s <- Tok.reloadP+ e <- Tok.functionP+ return $ s <> e+++insertBehaviorHelper :: InsertBehavior Range ->+ Maybe (TablePartition) ->+ InsertBehavior Range+insertBehaviorHelper (InsertOverwrite a) (Just partition) = InsertOverwritePartition a partition+insertBehaviorHelper (InsertAppend a) (Just partition) = InsertAppendPartition a partition+insertBehaviorHelper ib _ = ib+++insertP :: (QueryPrefix, InvertedFrom) -> Parser (Insert RawNames Range)+insertP (with, farInvertedFrom) = do+ r <- Tok.insertP+ insertBehaviorTok <- choice+ -- T407432 Overhaul Overwrite with Partition support+ [ do+ overwrite <- Tok.overwriteP+ return $ InsertOverwrite overwrite+ , do+ into <- Tok.intoP+ return $ InsertAppend into+ ]++ optional Tok.tableP+ insertTable <- tableNameP++ tablePartition <- optionMaybe $ do+ _ <- Tok.partitionP+ partitionSpecP++ let insertBehavior = insertBehaviorHelper insertBehaviorTok tablePartition++ insertColumns <- optionMaybe $ try $ do+ _ <- Tok.openP+ let oqColumnNameP = (\ (c, r') -> QColumnName r' Nothing c) <$> Tok.columnNameP+ c:cs <- oqColumnNameP `sepBy1` Tok.commaP+ _ <- Tok.closeP+ pure (c :| cs)++ insertValues <- choice+ [ do+ s <- Tok.valuesP+ (e, rows) <- rowsOfValuesP+ pure $ InsertExprValues (s <> e) rows+ , do+ isv <- case farInvertedFrom of+ Just _ -> InsertSelectValues <$> querySelectP (with, farInvertedFrom)+ Nothing -> InsertSelectValues <$> queryP (with, noInversion) -- INSERT .. FROM .. SELECT is not permitted+ pure $ isv+ ]++ let insertInfo = r <> (getInfo insertValues)++ pure Insert{..}+ where+ valueP :: Parser (DefaultExpr RawNames Range)+ valueP = do+ value <- constantP+ let r = getInfo value+ pure $ ExprValue $ ConstantExpr r value++ rowOfValuesP = do+ s <- Tok.openP+ x:xs <- valueP `sepBy1` Tok.commaP+ e <- Tok.closeP+ pure $ (s <> e, x :| xs)++ rowsOfValuesP = do+ rows <- rowOfValuesP `sepBy1` Tok.commaP+ let infos = map fst rows+ r:rs = map snd rows+ pure $ (head infos <> last infos, r :| rs)+++loadDataInPathP :: Parser (Insert RawNames Range)+loadDataInPathP = do+ s <- Tok.loadP+ _ <- Tok.dataP+ optional Tok.localP+ _ <- Tok.inPathP+ (path, r) <- Tok.stringP+ maybeOverwrite <- optionMaybe Tok.overwriteP+ into <- Tok.intoP+ _ <- Tok.tableP+ table <- tableNameP+ partitions <- optionMaybe $ do+ _ <- Tok.partitionP+ snd <$> staticPartitionSpecP++ let e = maybe (getInfo table) id partitions+ insertInfo = s <> e+ behaviorTok = case maybeOverwrite of+ Nothing -> InsertAppend into+ Just overwrite -> InsertOverwrite overwrite+ insertBehavior = insertBehaviorHelper behaviorTok (void partitions)+ insertTable = table+ insertColumns = Nothing+ insertValues = InsertDataFromFile r path++ pure Insert{..}++deleteP :: Parser (Delete RawNames Range)+deleteP = do+ r <- Tok.deleteP++ _ <- Tok.fromP+ table <- tableNameP++ maybeExpr <- optionMaybe $ do+ _ <- Tok.whereP+ local (introduceAliases $ tableNameToTableAlias table) exprP++ let r' = case maybeExpr of+ Nothing -> getInfo table+ Just expr -> getInfo expr+ info = r <> r'++ pure $ Delete info table maybeExpr+++truncateP :: Parser (Truncate RawNames Range)+truncateP = do+ s <- Tok.truncateP+ _ <- Tok.tableP+ table <- tableNameP++ pure $ Truncate (s <> getInfo table) table+++type QueryPrefix = Query RawNames Range -> Query RawNames Range++emptyPrefix :: QueryPrefix+emptyPrefix = id++withP :: Parser QueryPrefix+withP = do+ r <- Tok.withP+ withs <- cteP `sepBy1` Tok.commaP++ return $ \ query ->+ let r' = sconcat $ r :| getInfo query : map cteInfo withs+ in QueryWith r' withs query+ where+ cteP = do+ alias <- tableAliasP+ columns <- option []+ $ P.between Tok.openP Tok.closeP $ columnAliasP `sepBy1` Tok.commaP++ _ <- Tok.asP++ (query, r') <- do+ _ <- Tok.openP+ invertedFrom <- invertedFromP+ q <- queryP (emptyPrefix, invertedFrom) -- a query may not have more than 1 WITH clause.+ r' <- Tok.closeP+ return (q, r')++ return $ CTE (getInfo alias <> r') alias columns query++-- parses just a SELECT, consuming no UNIONs+-- i.e. it returns only QuerySelects+querySelectP :: (QueryPrefix, InvertedFrom) -> Parser (Query RawNames Range)+querySelectP (with, invertedFrom) = queryPHelper with invertedFrom False++-- parses SELECTs and UNIONs+-- i.e. it returns QuerySelects and QueryUnions+queryP :: (QueryPrefix, InvertedFrom) -> Parser (Query RawNames Range)+queryP (with, invertedFrom) = queryPHelper with invertedFrom True++queryPHelper :: QueryPrefix -> InvertedFrom -> Bool -> Parser (Query RawNames Range)+queryPHelper with invertedFrom unionsPermitted = do+ -- The invertedFrom, if supplied, will only be applied to the first SELECT.+ -- If invertedFrom is Nothing, then it is assumed the first SELECT has no inversion.+ firstSelect <- onlySelectP invertedFrom++ query <- if unionsPermitted+ then do+ maybeUnion <- optionMaybe unionP+ case maybeUnion of+ Nothing -> return firstSelect+ Just union -> do+ let subsequentSelectP = do+ nextInvertedFrom <- invertedFromP+ onlySelectP nextInvertedFrom+ subsequentSelects <- subsequentSelectP `chainl1` unionP+ return $ union firstSelect subsequentSelects+ else return firstSelect++ order <- option id orderP+ optional selectClusterP+ limit <- option id limitP++ return $ with $ limit $ order query+ where+ onlySelectP invertedFrom' = do+ select <- selectP invertedFrom'+ return $ QuerySelect (selectInfo select) select++ unionP = do+ r <- Tok.unionP+ distinct <- option (Distinct True) distinctP+ return $ QueryUnion r distinct Unused++ orderP = do+ (r, orders) <- orderTopLevelP+ return $ \ query -> QueryOrder (getInfo query <> r) orders query++ limitP = do+ r <- Tok.limitP+ Tok.numberP >>= \ (v, r') ->+ let limit = Limit (r <> r') v+ in return $ \ query -> QueryLimit (getInfo query <> r') limit query+++distinctP :: Parser Distinct+distinctP = choice $+ [ Tok.allP >> return (Distinct False)+ , Tok.distinctP >> return (Distinct True)+ ]+++explainP :: Parser (Statement Hive RawNames Range)+explainP = do+ s <- Tok.explainP+ stmt <- choice+ [ InsertStmt <$> insertP (emptyPrefix, noInversion)+ , DeleteStmt <$> deleteP+ , QueryStmt <$> queryP (emptyPrefix, noInversion)+ ]++ pure $ ExplainStmt (s <> getInfo stmt) stmt+++tableAliasP :: Parser (TableAlias Range)+tableAliasP = do+ (name, r) <- Tok.tableNameP+ makeTableAlias r name+++columnAliasP :: Parser (ColumnAlias Range)+columnAliasP = do+ (name, r) <- Tok.columnNameP+ makeColumnAlias r name+++createSchemaPrefixP :: Parser Range+createSchemaPrefixP = do+ s <- Tok.createP+ e <- Tok.schemaP <|> Tok.databaseP+ return $ s <> e+++ifNotExistsP :: Parser (Maybe Range)+ifNotExistsP = optionMaybe $ do+ s' <- Tok.ifP+ _ <- Tok.notP+ e' <- Tok.existsP+ pure $ s' <> e'+++commentP :: Parser Range+commentP = do+ s <- Tok.commentP+ (_, e) <- Tok.stringP+ return $ s <> e+++locationP :: Parser (Location Range)+locationP = do+ s <- Tok.locationP+ (loc, e) <- Tok.stringP+ return $ HDFSPath (s <> e) loc+++createSchemaP :: Parser (CreateSchema RawNames Range)+createSchemaP = do+ s <- createSchemaPrefixP+ createSchemaIfNotExists <- ifNotExistsP++ (name, r) <- Tok.schemaNameP+ let createSchemaName = mkNormalSchema name r++ e <- consumeOrderedOptions r $+ [ commentP+ , getInfo <$> locationP+ , dbPropertiesP+ ]+ let createSchemaInfo = s <> e++ return $ CreateSchema{..}++ where+ dbPropertiesP = do+ s <- Tok.withP+ _ <-Tok.dbPropertiesP+ _ <- Tok.openP+ _ <- propertyP `sepBy1` Tok.commaP+ e <- Tok.closeP+ return $ s <> e+++createViewPrefixP :: Parser Range+createViewPrefixP = do+ s <- Tok.createP+ e <- Tok.viewP+ return $ s <> e++createViewP :: Parser (CreateView RawNames Range)+createViewP = do+ s <- createViewPrefixP++ let createViewPersistence = Persistent+ createViewIfNotExists <- ifNotExistsP++ createViewName <- tableNameP++ createViewColumns <- optionMaybe $ do+ _ <- Tok.openP+ c:cs <- flip sepBy1 Tok.commaP $ do+ col <- unqualifiedColumnNameP+ _ <- commentP+ return col+ _ <- Tok.closeP+ return (c:|cs)++ optional commentP+ optional $ do+ _ <- Tok.tblPropertiesP+ _ <- Tok.openP+ _ <- (Tok.stringP >> Tok.equalP >> Tok.stringP) `sepBy1` Tok.commaP+ Tok.closeP++ _ <- Tok.asP+ createViewQuery <- querySelectP (emptyPrefix, noInversion)++ let createViewInfo = s <> getInfo createViewQuery+ pure CreateView{..}+++data CreateTablePrefix r a = CreateTablePrefix+ { createTablePrefixInfo :: a+ , createTablePrefixPersistence :: Persistence a+ , createTablePrefixExternality :: Externality a+ , createTablePrefixIfNotExists :: Maybe a+ , createTablePrefixName :: CreateTableName r a+ }++deriving instance ConstrainSNames Eq r a => Eq (CreateTablePrefix r a)+deriving instance ConstrainSNames Show r a => Show (CreateTablePrefix r a)+deriving instance ConstrainSASNames Functor r => Functor (CreateTablePrefix r)+deriving instance ConstrainSASNames Foldable r => Foldable (CreateTablePrefix r)+deriving instance ConstrainSASNames Traversable r => Traversable (CreateTablePrefix r)+++createTablePrefixP :: Parser (CreateTablePrefix RawNames Range)+createTablePrefixP = do+ s <- Tok.createP+ createTablePrefixPersistence <- option Persistent $ Temporary <$> Tok.temporaryP+ createTablePrefixExternality <- option Internal (External <$> Tok.externalP)+ _ <- Tok.tableP+ createTablePrefixIfNotExists <- ifNotExistsP+ createTablePrefixName <- tableNameP++ let createTablePrefixInfo = s <> getInfo createTablePrefixName++ return CreateTablePrefix{..}+++createTableP :: Parser (CreateTable Hive RawNames Range)+createTableP = choice+ [ do+ _ <- try $ P.lookAhead $ createTablePrefixP >> Tok.likeP+ createTableLikeP+ , createTableStandardP+ ]+++createTableLikeP :: Parser (CreateTable Hive RawNames Range)+createTableLikeP = do+ CreateTablePrefix{..} <- createTablePrefixP+ let s = createTablePrefixInfo+ createTablePersistence = createTablePrefixPersistence+ createTableExternality = createTablePrefixExternality+ createTableIfNotExists = createTablePrefixIfNotExists+ createTableName = createTablePrefixName++ _ <- Tok.likeP+ table <- tableNameP++ let e = getInfo table+ e' <- option e $ choice $+ [ getInfo <$> locationP+ , storedAsP+ ]++ let createTableInfo = s <> e'+ createTableDefinition = TableLike (s <> e) table+ createTableExtra = Nothing++ return CreateTable{..}+++propertyP :: Parser Range+propertyP = do+ s <- snd <$> Tok.stringP+ _ <- Tok.equalP+ e <- snd <$> Tok.stringP+ return $ s <> e++storedAsP :: Parser Range+storedAsP = do+ s <- Tok.storedP+ _ <- Tok.asP+ e <- choice+ [ Tok.orcP+ , Tok.sequenceFileP+ , Tok.textFileP+ , Tok.rcFileP+ , Tok.parquetP+ , Tok.avroP+ , do+ s' <- Tok.inputFormatP+ _ <- Tok.stringP+ _ <- Tok.outputFormatP+ (_, e') <- Tok.stringP+ return (s' <> e')+ ]+ return $ s <> e+++createTableStandardP :: Parser (CreateTable Hive RawNames Range)+createTableStandardP = do+ CreateTablePrefix{..} <- createTablePrefixP+ let s = createTablePrefixInfo+ createTablePersistence = createTablePrefixPersistence+ createTableExternality = createTablePrefixExternality+ createTableIfNotExists = createTablePrefixIfNotExists+ createTableName = createTablePrefixName++ tableDefColumns <- optionMaybe createTableColumnsP+ let e1 = maybe s getInfo tableDefColumns++ e2 <- consumeOrderedOptions e1 $+ [ commentP+ , partitionedByP+ , clusteredByP+ , rowFormatP+ , storedAsP+ , getInfo <$> locationP+ , tblPropertiesP+ ]++ createTableDefinition <- case tableDefColumns of+ Just definition -> return definition+ Nothing -> choice+ [ createTableAsP+ , createTableNoColumnInfoP e2+ ]++ let createTableExtra = Nothing+ e3 = getInfo createTableDefinition+ createTableInfo = s <> e1 <> e2 <> e3++ pure CreateTable{..}++ where+ columnDefinitionP = do+ (name, s) <- Tok.columnNameP+ columnDefinitionType <- dataTypeP+ optional commentP++ let columnDefinitionInfo = s <> getInfo columnDefinitionType+ columnDefinitionExtra = Nothing -- TODO+ columnDefinitionNull = Nothing+ columnDefinitionDefault = Nothing+ columnDefinitionName = QColumnName s None name++ pure ColumnDefinition{..}++ partitionedByP = do+ _ <- Tok.partitionedP+ _ <- Tok.byP+ _ <- Tok.openP+ _ <- columnDefinitionP `sepBy1` Tok.commaP+ Tok.closeP++ clusteredByP = do+ _ <- Tok.clusteredP+ _ <- Tok.byP+ _ <- Tok.openP+ _ <- Tok.columnNameP `sepBy1` Tok.commaP+ _ <- Tok.closeP+ optional $ do+ _ <- Tok.sortedP+ _ <- Tok.byP+ _ <- Tok.openP+ _ <- (Tok.columnNameP >> optional directionP) `sepBy1` Tok.commaP+ Tok.closeP+ _ <- Tok.intoP+ _ <- Tok.numberP+ Tok.bucketsP++ serdeP = do+ _ <- Tok.serdeP+ e <- snd <$> Tok.stringP+ option e $ do+ _ <- Tok.withP+ _ <- Tok.serdePropertiesP+ _ <- Tok.openP+ _ <- propertyP `sepBy1` Tok.commaP+ Tok.closeP++ rowFormatP = do+ _ <- Tok.rowP+ _ <- Tok.formatP+ delimitedP <|> serdeP++ tblPropertiesP = do+ _ <- Tok.tblPropertiesP+ _ <- Tok.openP+ _ <- propertyP `sepBy1` Tok.commaP+ Tok.closeP++ createTableColumnsP = do+ s <- Tok.openP+ c:cs <- (ColumnOrConstraintColumn <$> columnDefinitionP) `sepBy1` Tok.commaP+ e <- Tok.closeP+ pure $ TableColumns (s <> e) (c:|cs)++ createTableAsP = do+ s <- Tok.asP+ with <- option id withP+ query <- queryP (with, noInversion) -- WITH *is* permitted in a CTAS. Inverted FROM is *not*.+ pure $ TableAs (s <> getInfo query) Nothing query++ createTableNoColumnInfoP r =+ -- r represents 'the point at which we decided there was no column info'+ pure $ TableNoColumnInfo r+++delimitedP :: Parser Range+delimitedP = do+ s <- Tok.delimitedP+ e <- consumeOrderedOptions s $+ [ do+ _ <- Tok.fieldsP+ e' <- terminatedByCharP+ option e' $ do+ _ <- Tok.escapedP+ _ <- Tok.byP+ snd <$> Tok.stringP+ , do+ _ <- Tok.collectionP+ _ <- Tok.itemsP+ terminatedByCharP+ , do+ _ <- Tok.mapP+ _ <- Tok.keysP+ terminatedByCharP+ , do+ _ <- Tok.linesP+ terminatedByCharP+ , do+ _ <- Tok.nullP+ _ <- Tok.definedP+ _ <- Tok.asP+ snd <$> Tok.stringP+ ]+ return $ s <> e+ where+ terminatedByCharP = do+ _ <- Tok.terminatedP+ _ <- Tok.byP+ snd <$> Tok.stringP++++ifExistsP :: Parser Range+ifExistsP = do+ s <- Tok.ifP+ e <- Tok.existsP+ pure $ s <> e+++dropTableP :: Parser (DropTable RawNames Range)+dropTableP = do+ s <- Tok.dropP+ _ <- Tok.tableP+ dropTableIfExists <- optionMaybe ifExistsP+ dropTableName <- tableNameP+ purge <- optionMaybe Tok.purgeP++ let dropTableInfo = s <> (fromMaybe (getInfo dropTableName) purge)+ dropTableNames = dropTableName :| []+ pure DropTable{..}+++alterTableRenameTablePrefixP :: Parser (Range, TableName RawNames Range)+alterTableRenameTablePrefixP = do+ s <- Tok.alterP+ _ <- Tok.tableP+ from <- tableNameP+ _ <- Tok.renameP+ pure $ (s, from)+++alterTableRenameTableP :: Parser (AlterTable RawNames Range)+alterTableRenameTableP = do+ (s, from) <- alterTableRenameTablePrefixP+ _ <- Tok.toP+ to <- tableNameP++ pure $ AlterTableRenameTable (s <> getInfo to) from to+++alterTableRenameColumnPrefixP :: Parser (Range, TableName RawNames Range)+alterTableRenameColumnPrefixP = do+ s <- Tok.alterP+ _ <- Tok.tableP+ table <- tableNameP+ optional $ Tok.partitionP >> staticPartitionSpecP+ _ <- Tok.changeP+ pure (s, table)++unqualifiedColumnNameP :: Parser (UQColumnName Range)+unqualifiedColumnNameP = do+ (col, r) <- Tok.columnNameP+ pure $ QColumnName r None col++alterTableRenameColumnP :: Parser (AlterTable RawNames Range)+alterTableRenameColumnP = do+ (s, table) <- alterTableRenameColumnPrefixP+ optional Tok.columnP+ from <- unqualifiedColumnNameP+ to <- unqualifiedColumnNameP+ e <- getInfo <$> dataTypeP+ e' <- consumeOrderedOptions e $+ [ commentP+ , choice [ Tok.firstP+ , Tok.afterP >> snd <$> Tok.columnNameP+ ]+ , Tok.cascadeP <|> Tok.restrictP+ ]+ pure $ AlterTableRenameColumn (s <> e') table from to++alterTableAddColumnsPrefixP :: Parser (Range, TableName RawNames Range)+alterTableAddColumnsPrefixP = do+ s <- Tok.alterP+ _ <- Tok.tableP+ table <- tableNameP+ -- Note that Hive has ALTER TABLE ADD COLUMN for single partitions+ -- (!). Current behavior is to treat it the same as a top level ALTER TABLE+ -- ADD COLUMN. That seems fine initially, but hard to say if it makes sense+ -- in the long term.+ optional $ Tok.partitionP >> partitionSpecP+ _ <- Tok.addP+ e <- Tok.columnsP+ pure (s <> e, table)++alterTableAddColumnsP :: Parser (AlterTable RawNames Range)+alterTableAddColumnsP = do+ (s, table) <- alterTableAddColumnsPrefixP+ _ <- Tok.openP+ c:cs <- (colP `sepBy1` Tok.commaP)+ e <- Tok.closeP+ e' <- option e (Tok.cascadeP <|> Tok.restrictP)+ pure $ AlterTableAddColumns (s <> e') table (c:|cs)+ where+ colP :: Parser (UQColumnName Range)+ colP = do+ col <- unqualifiedColumnNameP+ _ <- dataTypeP+ _ <- optional commentP+ return col++grantP :: Parser (Grant Range)+grantP = do+ s <- Tok.grantP+ e <- P.many1 Tok.notSemicolonP+ return $ Grant (s <> (last e))++revokeP :: Parser (Revoke Range)+revokeP = do+ s <- Tok.revokeP+ e <- P.many1 Tok.notSemicolonP+ return $ Revoke (s <> (last e))++integerP :: Parser (Int, Range)+integerP = do+ (n, e) <- Tok.numberP+ case reads $ TL.unpack n of+ [(n', "")] -> pure (n', e)+ _ -> fail $ unwords ["unable to parse", show n, "as integer"]+++countingSepBy1 :: (Integer -> Parser b) -> Parser c -> Parser [b]+countingSepBy1 f sep = do+ x <- f 0+ xs <- rest 1+ pure (x:xs)+ where+ rest n = choice+ [ do+ _ <- sep+ x <- f n+ xs <- rest (n + 1)+ pure (x:xs)+ , pure []+ ]+++introduceAliases :: Set Text -> ParserScope -> ParserScope+introduceAliases aliases = \ scope ->+ let unioned = case selectTableAliases scope of+ Nothing -> aliases+ Just existing -> S.union existing aliases+ in scope { selectTableAliases = Just unioned }++tablishToTableAlias :: Tablish RawNames Range -> Set Text+tablishToTableAlias (TablishTable _ aliases tableName) = case aliases of+ TablishAliasesNone -> tableNameToTableAlias tableName+ TablishAliasesT (TableAlias _ name _) -> S.singleton name+ TablishAliasesTC _ _ -> error "shouldn't happen in hive"+tablishToTableAlias (TablishSubQuery _ aliases _) = case aliases of+ TablishAliasesNone -> error "shouldn't happen in hive"+ TablishAliasesT (TableAlias _ name _) -> S.singleton name+ TablishAliasesTC _ _ -> error "shouldn't happen in hive"+tablishToTableAlias (TablishLateralView _ LateralView{..} _) = case lateralViewAliases of+ TablishAliasesNone -> error "shouldn't happen in hive"+ TablishAliasesT (TableAlias _ name _) -> S.singleton name+ TablishAliasesTC (TableAlias _ name _) _ -> S.singleton name+tablishToTableAlias (TablishJoin _ (JoinSemi _) _ lTablish _) =+ tablishToTableAlias lTablish+tablishToTableAlias (TablishJoin _ _ _ lTablish rTablish) =+ tablishToTableAlias lTablish `S.union` tablishToTableAlias rTablish++tableNameToTableAlias :: OQTableName Range -> Set Text+tableNameToTableAlias (QTableName _ _ name) = S.singleton name+++fromP :: Parser (SelectFrom RawNames Range)+fromP = do+ r <- Tok.fromP+ tablishes <- tablishP `sepBy1` Tok.commaP++ let r' = foldl (<>) r $ fmap getInfo tablishes+ return $ SelectFrom r' tablishes++type InvertedFrom = Maybe (SelectFrom RawNames Range)++noInversion :: InvertedFrom+noInversion = Nothing++invertedFromP :: Parser InvertedFrom+invertedFromP = optionMaybe fromP+++selectP :: InvertedFrom -> Parser (Select RawNames Range)+selectP invertedFrom = do+ r <- Tok.selectP++ selectDistinct <- option notDistinct distinctP++ aliases <- try $ selectScopeLookAhead invertedFrom++ selectCols <- do+ selections <- local (introduceAliases aliases) $ selectionP `countingSepBy1` Tok.commaP+ let r' = foldl1 (<>) $ map getInfo selections+ return $ SelectColumns r' selections++ selectFrom <- maybe (optionMaybe fromP) (return . Just) invertedFrom+ selectWhere <- optionMaybe $ local (introduceAliases aliases) whereP+ let selectTimeseries = Nothing+ selectGroup <- optionMaybe selectGroupP+ selectHaving <- optionMaybe havingP+ selectNamedWindow <- optionMaybe namedWindowP++ let (Just selectInfo) = sconcat $ Just r :|+ [ Just $ getInfo selectCols+ , getInfo <$> selectFrom+ , getInfo <$> selectWhere+ , getInfo <$> selectGroup+ , getInfo <$> selectHaving+ , getInfo <$> selectNamedWindow+ ]+ return Select{..}++ where+ selectScopeLookAhead :: InvertedFrom -> Parser (Set Text)+ selectScopeLookAhead invertedFrom' = P.lookAhead $ do+ _ <- selectionP (-1) `sepBy1` Tok.commaP+ from <- maybe (optionMaybe fromP) (return . Just) invertedFrom'++ let tablishes = case from of+ Just (SelectFrom _ ts) -> ts+ Nothing -> []+ aliases = L.foldl' S.union S.empty $ map tablishToTableAlias tablishes+ return aliases++ whereP = do+ r <- Tok.whereP+ condition <- exprP+ return $ SelectWhere (r <> getInfo condition) condition++ havingP = do+ r <- Tok.havingP+ conditions <- exprP `sepBy1` Tok.commaP++ let r' = foldl (<>) r $ fmap getInfo conditions+ return $ SelectHaving r' conditions++ namedWindowP =+ do+ r <- Tok.windowP+ windows <- (flip sepBy1) Tok.commaP $ do+ name <- windowNameP+ _ <- Tok.asP+ s <- Tok.openP+ window <- choice+ [ do+ partition <- optionMaybe partitionP+ order <- option [] orderInWindowClauseP+ frame <- optionMaybe frameP+ e <- Tok.closeP+ let info = s <> e+ return $ Left $ WindowExpr info partition order frame+ , do+ inherit <- windowNameP+ partition <- optionMaybe partitionP+ order <- option [] orderInWindowClauseP+ frame <- optionMaybe frameP+ e <- Tok.closeP+ let info = s <> e+ return $ Right $ PartialWindowExpr info inherit partition order frame+ ]++ let infof = (getInfo name <>)+ return $ case window of+ Left w -> NamedWindowExpr (infof $ getInfo w) name w+ Right pw -> NamedPartialWindowExpr (infof $ getInfo pw) name pw+ let info = L.foldl' (<>) r $ fmap getInfo windows+ return $ SelectNamedWindow info windows+++handlePositionalReferences :: Expr RawNames Range -> PositionOrExpr RawNames Range+handlePositionalReferences e = case e of+ ConstantExpr _ (NumericConstant _ n) | TL.all isDigit n -> PositionOrExprPosition (getInfo e) (read $ TL.unpack n) Unused+ _ -> PositionOrExprExpr e++selectGroupP :: Parser (SelectGroup RawNames Range)+selectGroupP = do+ r <- Tok.groupP+ _ <- Tok.byP++ rawExprs <- exprP `sepBy1` Tok.commaP+ let exprs = map (toGroupingElement . handlePositionalReferences) rawExprs++ sets <- option [] $ choice+ [ groupingSetsP+ , do+ _ <- try $ P.lookAhead $ Tok.withP >> Tok.cubeP+ cubeP rawExprs+ , do+ _ <- try $ P.lookAhead $ Tok.withP >> Tok.rollupP+ rollupP rawExprs+ ]++ let selectGroupGroupingElements = exprs ++ sets+ selectGroupInfo = foldl (<>) r $ fmap getInfo selectGroupGroupingElements++ return SelectGroup{..}+ where+ toGroupingElement :: PositionOrExpr RawNames Range -> GroupingElement RawNames Range+ toGroupingElement posOrExpr = GroupingElementExpr (getInfo posOrExpr) posOrExpr++ groupingSetP :: Parser (GroupingElement RawNames Range)+ groupingSetP = choice $+ [ do+ s <- Tok.openP+ sets <- exprP `sepBy` Tok.commaP+ e <- Tok.closeP+ return $ GroupingElementSet (s <> e) sets+ , do+ -- if no parens, it will be the singleton list.+ expr <- exprP+ return $ GroupingElementSet (getInfo expr) [expr]+ ]++ groupingSetsP :: Parser [GroupingElement RawNames Range]+ groupingSetsP = do+ _ <- Tok.groupingP+ _ <- Tok.setsP+ _ <- Tok.openP+ sets <- groupingSetP `sepBy1` Tok.commaP+ _ <- Tok.closeP+ return sets++ toGroupingSet :: Range -> [Expr RawNames Range] -> GroupingElement RawNames Range+ toGroupingSet r [] = GroupingElementSet r []+ toGroupingSet _ exprs =+ let s = getInfo $ head exprs+ e = getInfo $ last exprs+ in GroupingElementSet (s <> e) exprs++ cubeP :: [Expr RawNames Range] -> Parser [GroupingElement RawNames Range]+ cubeP exprs = do+ _ <- Tok.withP+ _ <- Tok.cubeP+ let dimensions = L.subsequences exprs+ defaultRange = (getInfo $ head exprs) <> (getInfo $ last exprs)+ return $ map (toGroupingSet defaultRange) dimensions++ rollupP :: [Expr RawNames Range] -> Parser [GroupingElement RawNames Range]+ rollupP exprs = do+ _ <- Tok.withP+ _ <- Tok.rollupP+ let dimensions = L.reverse $ L.inits exprs+ defaultRange = (getInfo $ head exprs) <> (getInfo $ last exprs)+ return $ map (toGroupingSet defaultRange) dimensions+++-- | 'selectClusterP' parses for either clusterby or distributeby/sortby+-- T478023 - implement clusterby in select datatype+selectClusterP :: Parser ()+selectClusterP = choice+ [ clusterP+ , distributeSortP+ ]++ where+ clusterP :: Parser ()+ clusterP =+ do+ _ <- Tok.clusterP+ _ <- Tok.byP+ _ <- sepBy1 exprP Tok.commaP+ return ()++ distributeSortP :: Parser ()+ distributeSortP =+ do+ optional distributeP+ optional sortP++ distributeP :: Parser ()+ distributeP =+ do+ _ <- Tok.distributeP+ _ <- Tok.byP+ _ <- sepBy1 exprP Tok.commaP+ return ()++ sortP :: Parser ()+ sortP =+ do+ _ <- Tok.sortP+ _ <- Tok.byP+ _ <- flip sepBy1 Tok.commaP $ do+ expr <- exprP+ direction <- option (OrderAsc Nothing) $ choice+ [ OrderAsc . Just <$> Tok.ascP+ , OrderDesc . Just <$> Tok.descP+ ]+ return (expr, direction)+ return ()+++qualifiedTableNameP :: Parser (Text, Text, Range, Range)+qualifiedTableNameP = do+ (s, r) <- Tok.schemaNameP+ _ <- Tok.dotP+ (t, r') <- Tok.tableNameP++ return (s, t, r, r')+++-- | The columnName parser has been overhauled with checks for table names. If+-- a scope is present (i.e. in a select statement), the table name must be+-- a member of the tableAlias list for the parser to succeed. Otherwise,+-- the table parser fails and execution tries the next parser choice.+--+-- Should the scope not be set, e.g. when selectP is performing lookahead+-- to build scope, this check is skipped.+checkTableNameInScopeP :: Text -> Parser ()+checkTableNameInScopeP name = do+ maybeScope <- asks selectTableAliases+ case maybeScope of+ Just scope ->+ case L.find (==name) scope of+ Just _ -> return ()+ Nothing -> fail $ "Table " ++ (show name) +++ " doesn't exist in table scope " ++ show maybeScope+ Nothing -> return ()+++selectStarP :: Parser (Selection RawNames Range)+selectStarP = choice+ [ do+ r <- Tok.starP+ return $ SelectStar r Nothing Unused++ , try $ do+ (t, r) <- Tok.tableNameP+ _ <- Tok.dotP+ r' <- Tok.starP++ return $ SelectStar (r <> r') (Just $ QTableName r Nothing t) Unused++ , try $ do+ (s, t, r, r') <- qualifiedTableNameP+ _ <- Tok.dotP+ r'' <- Tok.starP++ return $ SelectStar (r <> r'')+ (Just $ QTableName r' (Just $ mkNormalSchema s r) t) Unused+ ]+++tableNameP :: Parser (OQTableName Range)+tableNameP = choice+ [ try $ do+ (s, t, r, r') <- qualifiedTableNameP+ return $ QTableName r' (Just $ mkNormalSchema s r) t++ , do+ (t, r) <- Tok.tableNameP+ return $ QTableName r Nothing t+ ]+++arrayAccessP :: Parser (Expr RawNames Range -> Expr RawNames Range)+arrayAccessP = do+ _ <- Tok.openBracketP+ index <- exprP+ e <- Tok.closeBracketP+ return $ \ expr ->+ let exprR = getInfo expr <> e+ in ArrayAccessExpr exprR expr index+++structFieldNameP :: Parser (StructFieldName Range)+structFieldNameP = do+ (t, r) <- Tok.structFieldNameP+ return $ StructFieldName r t+++structAccessP :: Parser (Expr RawNames Range -> Expr RawNames Range)+structAccessP = do+ _ <- Tok.dotP+ field <- structFieldNameP+ return $ \ struct ->+ let r = getInfo struct <> getInfo field+ in FieldAccessExpr r struct field+++columnNameP :: Parser (OQColumnName Range)+columnNameP = choice+ -- Note that in hive, column names cannot lead with schema qualifiers+ [ try $ do+ (t, r) <- Tok.tableNameP+ _ <- Tok.dotP+ (c, r') <- Tok.columnNameP++ _ <- checkTableNameInScopeP t++ return $ QColumnName r' (Just $ QTableName r Nothing t) c++ , do+ (c, r) <- Tok.columnNameP++ return $ QColumnName r Nothing c+ ]+++selectionP :: Integer -> Parser (Selection RawNames Range)+selectionP idx = try selectStarP <|> do+ expr <- exprP+ aliases <- aliasesP expr idx+ let info = foldr (<>) (getInfo expr) (map getInfo aliases)++ return $ SelectExpr info aliases expr+ where+ aliasesP :: Expr RawNames Range -> Integer -> Parser [ColumnAlias Range]+ aliasesP expr idx' = choice+ [ try $ do+ optional Tok.asP+ (name, r) <- Tok.columnNameP -- SELECT 1 as foo FROM bar;+ pure <$> makeColumnAlias r name++ , try $ do+ _ <- Tok.asP+ P.between Tok.openP Tok.closeP $ flip sepBy1 Tok.commaP $ do+ (name, r) <- Tok.columnNameP -- SELECT <expr> as (foo1, foo2) FROM bar;+ makeColumnAlias r name++ , do+ r <- Tok.asP+ pure <$> makeColumnAlias r "as" -- SELECT 1 as FROM bar; i.e. `as` is the alias!++ , pure <$> makeExprAlias expr idx'+ ]+++makeColumnAlias :: Range -> Text -> Parser (ColumnAlias Range)+makeColumnAlias r alias = ColumnAlias r alias . ColumnAliasId <$> getNextCounter++makeTableAlias :: Range -> Text -> Parser (TableAlias Range)+makeTableAlias r alias = TableAlias r alias . TableAliasId <$> getNextCounter++makeDummyAlias :: Range -> Integer -> Parser (ColumnAlias Range)+makeDummyAlias r idx = makeColumnAlias r $ TL.pack $ "_c" ++ show idx++makeExprAlias :: Expr RawNames Range -> Integer -> Parser (ColumnAlias Range)+makeExprAlias (ColumnExpr info (QColumnName _ _ name)) _ = makeColumnAlias info name+makeExprAlias expr idx = makeDummyAlias (getInfo expr) idx+++exprP :: Parser (Expr RawNames Range)+exprP = orExprP++parenExprP :: Parser (Expr RawNames Range)+parenExprP = P.between Tok.openP Tok.closeP exprP++caseExprP :: Parser (Expr RawNames Range)+caseExprP = do+ r <- Tok.caseP+ whens <- choice+ [ P.many1 $ do+ _ <- Tok.whenP+ condition <- exprP+ _ <- Tok.thenP+ result <- exprP+ return (condition, result)++ , do+ expr <- exprP+ P.many1 $ do+ whenr <- Tok.whenP+ condition <- BinOpExpr whenr "=" expr <$> exprP+ _ <- Tok.thenP+ result <- exprP+ return (condition, result)+ ]++ melse <- optionMaybe $ do+ _ <- Tok.elseP+ exprP++ r' <- Tok.endP++ return $ CaseExpr (r <> r') whens melse+++fieldTypeP :: Parser (Expr RawNames Range)+fieldTypeP = do+ (ftype, r) <- Tok.fieldTypeP+ return $ ConstantExpr r $ StringConstant r $ TL.encodeUtf8 ftype++functionExprP :: Parser (Expr RawNames Range)+functionExprP = choice+ [ castFuncP+ , dateDiffFuncP+ , extractFuncP+ , try regularFuncP+ , bareFuncP+ ]+ where+ castFuncP = do+ r <- Tok.castP+ _ <- Tok.openP+ e <- exprP+ _ <- Tok.asP+ t <- dataTypeP+ r' <- Tok.closeP++ return $ TypeCastExpr (r <> r') CastFailureError e t++ dateDiffFuncP = do+ r <- Tok.dateDiffP+ _ <- Tok.openP++ date1 <- exprP+ _ <- Tok.commaP+ date2 <- exprP+ r' <- Tok.closeP++ return $ FunctionExpr (r <> r') (QFunctionName r Nothing "datediff") notDistinct [date1, date2] [] Nothing Nothing++ extractFuncP = do+ r <- Tok.extractP+ _ <- Tok.openP+ ftype <- fieldTypeP+ _ <- Tok.fromP+ expr <- exprP+ r' <- Tok.closeP++ return $ FunctionExpr (r <> r') (QFunctionName r Nothing "extract") notDistinct [ftype, expr] [] Nothing Nothing++ regularFuncP = do+ name <- choice+ [ try $ do+ (s, r) <- Tok.schemaNameP+ _ <- Tok.dotP+ (f, r') <- Tok.functionNameP+ return $ QFunctionName (r <> r') (Just $ mkNormalSchema s r) f++ , do+ (f, r) <- Tok.functionNameP+ return $ QFunctionName r Nothing f+ ]++ (distinct, arguments, parameters, r') <- do+ _ <- Tok.openP+ (distinct, arguments) <- choice+ [ case name of+ QFunctionName _ Nothing "count" -> do+ r' <- Tok.starP+ return ( notDistinct+ , [ConstantExpr r' $ NumericConstant r' "1"]+ )++ _ -> fail "not count, can't star"++ , do+ isDistinct <- distinctP+ (isDistinct,) . (:[]) <$> exprP++ , (notDistinct,) <$> exprP `sepBy` Tok.commaP+ ]++ optional $ Tok.ignoreP >> Tok.nullsP++ r' <- Tok.closeP+ return (distinct, arguments, [], r')++ over <- optionMaybe $ try $ overP++ let r'' = maybe r' getInfo over <> getInfo name++ return $ FunctionExpr r'' name distinct arguments parameters Nothing over++ bareFuncP = do+ (v, r) <- choice+ [ Tok.currentDatabaseP+ , Tok.currentSchemaP+ , Tok.currentUserP+ , Tok.sessionUserP+ , Tok.currentDateP+ , Tok.currentTimeP+ , Tok.currentTimestampP+ , Tok.localTimeP+ , Tok.localTimestampP+ , Tok.sysDateP+ ]++ pure $ FunctionExpr r (QFunctionName r Nothing v) notDistinct [] [] Nothing Nothing++orderTopLevelP :: Parser (Range, [Order RawNames Range])+orderTopLevelP = orderExprP False True++orderInWindowClauseP :: Parser [Order RawNames Range]+orderInWindowClauseP = snd <$> orderExprP True False++orderExprP :: Bool -> Bool -> Parser (Range, [Order RawNames Range])+orderExprP nullsClausePermitted positionalReferencesPermitted = do+ r <- Tok.orderP+ _ <- Tok.byP+ orders <- helperP `sepBy1` Tok.commaP+ let r' = getInfo $ last orders+ return (r <> r', orders)+ where+ helperP :: Parser (Order RawNames Range)+ helperP = do+ expr <- exprP+ let posOrExpr = if positionalReferencesPermitted+ then handlePositionalReferences expr+ else PositionOrExprExpr expr+ dir <- directionP+ nulls <- case (nullsClausePermitted, dir) of+ (False, _) -> return $ NullsAuto Nothing+ (True, OrderAsc _) -> option (NullsLast Nothing) nullsP+ (True, OrderDesc _) -> option (NullsFirst Nothing) nullsP+ let info = (getInfo expr) ?<> (getInfo dir) <> (getInfo nulls)+ return $ Order info posOrExpr dir nulls++directionP :: Parser (OrderDirection (Maybe Range))+directionP = option (OrderAsc Nothing) $ choice+ [ OrderAsc . Just <$> Tok.ascP+ , OrderDesc . Just <$> Tok.descP+ ]++nullsP :: Parser (NullPosition (Maybe Range))+nullsP = do+ r <- Tok.nullsP+ choice+ [ Tok.firstP >>= \ r' -> return $ NullsFirst $ Just $ r <> r'+ , Tok.lastP >>= \ r' -> return $ NullsLast $ Just $ r <> r'+ , Tok.autoP >>= \ r' -> return $ NullsAuto $ Just $ r <> r'+ ]++frameP :: Parser (Frame Range)+frameP = do+ ftype <- choice+ [ RowFrame <$> Tok.rowsP+ , RangeFrame <$> Tok.rangeP+ ]++ choice+ [ do+ _ <- Tok.betweenP+ start <- frameBoundP+ _ <- Tok.andP+ end <- frameBoundP++ let r = getInfo ftype <> getInfo end+ return $ Frame r ftype start (Just end)++ , do+ start <- frameBoundP++ let r = getInfo ftype <> getInfo start+ return $ Frame r ftype start Nothing+ ]++frameBoundP :: Parser (FrameBound Range)+frameBoundP = choice+ [ fmap Unbounded $ (<>)+ <$> Tok.unboundedP+ <*> choice [ Tok.precedingP, Tok.followingP ]++ , fmap CurrentRow $ (<>) <$> Tok.currentP <*> Tok.rowP+ , constantP >>= \ expr -> choice+ [ Tok.precedingP >>= \ r ->+ return $ Preceding (getInfo expr <> r) expr++ , Tok.followingP >>= \ r ->+ return $ Following (getInfo expr <> r) expr+ ]+ ]+++overP :: Parser (OverSubExpr RawNames Range)+overP = do+ start <- Tok.overP+ subExpr <- choice+ [ Left <$> windowP+ , Right <$> windowNameP+ ]+ return $ case subExpr of+ Left w -> mergeWindowInfo start w+ Right wn -> OverWindowName (start <> getInfo wn) wn+ where+ windowP :: Parser (OverSubExpr RawNames Range)+ windowP = do+ start' <- Tok.openP+ expr <- choice+ [ Left <$> windowExprP start'+ , Right <$> partialWindowExprP start'+ ]+ return $ case expr of+ Left w -> OverWindowExpr (start' <> getInfo w) w+ Right pw -> OverPartialWindowExpr (start' <> getInfo pw) pw++ mergeWindowInfo :: Range -> OverSubExpr RawNames Range -> OverSubExpr RawNames Range+ mergeWindowInfo r = \case+ OverWindowExpr r' WindowExpr{..} ->+ OverWindowExpr (r <> r') $ WindowExpr { windowExprInfo = windowExprInfo <> r , ..}+ OverWindowName r' n -> OverWindowName (r <> r') n+ OverPartialWindowExpr r' PartialWindowExpr{..} ->+ OverPartialWindowExpr (r <> r') $ PartialWindowExpr { partWindowExprInfo = partWindowExprInfo <> r , ..}++windowExprP :: Range -> Parser (WindowExpr RawNames Range)+windowExprP start =+ do+ partition <- optionMaybe partitionP+ order <- option [] orderInWindowClauseP+ frame <- optionMaybe frameP+ end <- Tok.closeP+ let info = start <> end+ return (WindowExpr info partition order frame)++partialWindowExprP :: Range -> Parser (PartialWindowExpr RawNames Range)+partialWindowExprP start =+ do+ inherit <- windowNameP+ partition <- optionMaybe partitionP+ order <- option [] orderInWindowClauseP+ frame <- optionMaybe frameP+ end <- Tok.closeP+ let info = start <> end+ return (PartialWindowExpr info inherit partition order frame)++windowNameP :: Parser (WindowName Range)+windowNameP =+ do+ (name, r) <- Tok.windowNameP+ return $ WindowName r name++partitionP :: Parser (Partition RawNames Range)+partitionP = do+ r <- Tok.partitionP+ choice+ [ do+ _ <- Tok.byP+ exprs <- optionalParensP $ exprP `sepBy1` Tok.commaP+ return $ PartitionBy (sconcat $ r :| map getInfo exprs) exprs++ , Tok.bestP >>= \ r' -> return $ PartitionBest (r <> r')+ , Tok.nodesP >>= \ r' -> return $ PartitionNodes (r <> r')+ ]+++dataTypeP :: Parser (DataType Range)+dataTypeP = choice+ [ arrayTypeP+ , mapTypeP+ , structTypeP+ , unionTypeP+ , primitiveTypeP+ ]+ where+ primitiveTypeP = do+ (name, r) <- Tok.typeNameP+ args <- option [] $ P.between Tok.openP Tok.closeP $ constantP `sepBy1` Tok.commaP+ return $ PrimitiveDataType r name $ map DataTypeParamConstant args++ arrayTypeP = do+ s <- Tok.arrayP+ _ <- Tok.openAngleP+ itemType <- dataTypeP+ e <- Tok.closeAngleP+ return $ ArrayDataType (s <> e) itemType++ mapTypeP = do+ s <- Tok.mapP+ _ <- Tok.openAngleP+ keyType <- primitiveTypeP+ _ <- Tok.commaP+ valueType <- dataTypeP+ e <- Tok.closeAngleP+ return $ MapDataType (s <> e) keyType valueType++ structTypeP = do+ s <- Tok.structP+ _ <- Tok.openAngleP+ let fieldP = do+ (name, _) <- Tok.structFieldNameP+ _ <- Tok.colonP+ type_ <- dataTypeP+ optional commentP+ return (name, type_)+ fields <- fieldP `sepBy1` Tok.commaP+ e <- Tok.closeAngleP+ return $ StructDataType (s <> e) fields++ unionTypeP = do+ s <- Tok.uniontypeP+ _ <- Tok.openAngleP+ types <- dataTypeP `sepBy1` Tok.commaP+ e <- Tok.closeAngleP+ return $ UnionDataType (s <> e) types+++existsExprP :: Parser (Expr RawNames Range)+existsExprP = do+ r <- Tok.existsP+ _ <- Tok.openP+ query <- querySelectP (emptyPrefix, noInversion)+ r' <- Tok.closeP++ return $ ExistsExpr (r <> r') query+++columnExprP :: Parser (Expr RawNames Range)+columnExprP = do+ col <- columnNameP+ return $ ColumnExpr (getInfo col) col+++variableSubstitutionP :: Parser (Expr RawNames Range)+variableSubstitutionP = do+ info <- Tok.variableSubstitutionP+ return $ VariableSubstitutionExpr info+++exprWithArrayOrStructAccessP :: Parser (Expr RawNames Range)+exprWithArrayOrStructAccessP = foldl (flip ($)) <$> baseP <*> many (structAccessP <|> arrayAccessP)+ where+ baseP :: Parser (Expr RawNames Range)+ baseP = choice+ [ try parenExprP+ , try existsExprP+ , try functionExprP+ , caseExprP+ , try $ do+ constant <- constantP+ return $ ConstantExpr (getInfo constant) constant++ , columnExprP+ , variableSubstitutionP+ ]+++unOpP :: Text -> Parser (Expr RawNames Range -> Expr RawNames Range)+unOpP op = do+ r <- Tok.symbolP op+ return $ \ expr -> UnOpExpr (r <> getInfo expr) (Operator op) expr+++unaryPrefixExprP :: Parser (Expr RawNames Range)+unaryPrefixExprP = do+ prefix <- option id $ choice $ map unOpP [ "+", "-", "~" ]+ expr <- exprWithArrayOrStructAccessP+ return $ prefix expr+++notOperatorP :: Parser (Expr RawNames Range -> Expr RawNames Range)+notOperatorP = (\ r -> UnOpExpr r "NOT") <$> Tok.notOperatorP++unarySuffixExprP :: Parser (Expr RawNames Range)+unarySuffixExprP = do+ expr <- unaryPrefixExprP+ is <- option id $ do+ _ <- Tok.isP+ not_ <- option id notOperatorP+ (not_ .) <$> (Tok.nullP >>= \ r -> return (UnOpExpr r "ISNULL"))++ return $ is expr+++binOpP :: Text -> Parser (Expr RawNames Range -> Expr RawNames Range -> Expr RawNames Range)+binOpP op = do+ r <- Tok.symbolP op++ let r' lhs rhs = sconcat $ r :| map getInfo [lhs, rhs]+ return $ \ lhs rhs -> BinOpExpr (r' lhs rhs) (Operator op) lhs rhs+++bitwiseXorExprP :: Parser (Expr RawNames Range)+bitwiseXorExprP = unarySuffixExprP `chainl1` binOpP "^"+++productExprP :: Parser (Expr RawNames Range)+productExprP = bitwiseXorExprP `chainl1` opP+ where+ opP = choice $ map binOpP [ "*", "/", "%" ]+++sumExprP :: Parser (Expr RawNames Range)+sumExprP = productExprP `chainl1` opP+ where+ opP = choice $ map binOpP [ "+", "-" ]+++bitwiseAndExprP :: Parser (Expr RawNames Range)+bitwiseAndExprP = sumExprP `chainl1` binOpP "&"+++bitwiseOrExprP :: Parser (Expr RawNames Range)+bitwiseOrExprP = bitwiseAndExprP `chainl1` binOpP "|"+++inExprP :: Parser (Expr RawNames Range)+inExprP = do+ expr <- bitwiseOrExprP+ not_ <- option id notOperatorP+ in_ <- foldl (.) id <$> many inP++ return $ not_ $ in_ expr++ where+ inP = do+ _ <- Tok.inP+ _ <- Tok.openP+ list <- choice+ [ Left <$> queryP (emptyPrefix, noInversion)+ , Right <$> exprP `sepBy1` Tok.commaP+ ]++ r <- Tok.closeP++ return $ case list of+ Left query ->+ \ expr -> InSubqueryExpr (getInfo expr <> r) query expr++ Right constants ->+ \ expr -> InListExpr (getInfo expr <> r) constants expr++betweenExprP :: Parser (Expr RawNames Range)+betweenExprP = do+ expr <- inExprP+ between <- foldl (.) id <$> many betweenP++ return $ between expr+ where+ betweenP = do+ _ <- Tok.betweenP+ start <- sumExprP+ _ <- Tok.andP+ end <- sumExprP++ let r expr = getInfo expr <> getInfo end+ return $ \ expr -> BetweenExpr (r expr) start end expr+++likeExprP :: Parser (Expr RawNames Range)+likeExprP = do+ expr <- betweenExprP+ like <- option id comparisonP+ return $ like expr+ where+ comparisonP :: Parser (Expr RawNames Range -> Expr RawNames Range)+ comparisonP = do+ comparison <- textComparisonP+ pattern <- Pattern <$> betweenExprP+ return $ comparison Nothing pattern++ textComparisonP :: Parser (Maybe (Escape RawNames Range) -> Pattern RawNames Range -> Expr RawNames Range -> Expr RawNames Range)+ textComparisonP = do+ not_ <- option id notOperatorP++ like <- choice+ [ Tok.likeP >>= \ r -> return $ LikeExpr r "LIKE"+ , Tok.rlikeP >>= \ r -> return $ LikeExpr r "RLIKE"+ , Tok.regexpP >>= \ r -> return $ LikeExpr r "REGEXP"+ ]++ return $ \ escape pattern expr -> not_ $ like escape pattern expr+++mkBinOp :: (Text, a) -> Expr r a -> Expr r a -> Expr r a+mkBinOp (op, r) = BinOpExpr r (Operator op)++inequalityExprP :: Parser (Expr RawNames Range)+inequalityExprP = likeExprP `chainl1` (mkBinOp <$> Tok.inequalityOpP)+++equalityExprP :: Parser (Expr RawNames Range)+equalityExprP = inequalityExprP `chainl1` (mkBinOp <$> Tok.equalityOpP)+++notExprP :: Parser (Expr RawNames Range)+notExprP = do+ not_ <- option id notOperatorP+ expr <- equalityExprP+ return $ not_ expr+++andExprP :: Parser (Expr RawNames Range)+andExprP = notExprP `chainl1`+ (Tok.andP >>= \ r -> return $ BinOpExpr r "AND")+++orExprP :: Parser (Expr RawNames Range)+orExprP = andExprP `chainl1` (Tok.orP >>= \ r -> return (BinOpExpr r "OR"))+++singleTableP :: Parser (Tablish RawNames Range)+singleTableP = try subqueryP <|> try tableP+ where+ subqueryP = do+ r <- Tok.openP+ invertedFrom <- invertedFromP+ query <- queryP (emptyPrefix, invertedFrom)+ _ <- Tok.closeP+ maybe_alias <- aliasP+ case maybe_alias of+ Nothing -> fail $ "in hive, tablish subquery must have alias"+ Just alias -> return $ TablishSubQuery (r <> getInfo alias) (TablishAliasesT alias) query++ tableP = do+ name <- tableNameP+ _ <- optional tableSampleP+ maybe_alias <- aliasP+ let r = case maybe_alias of+ Nothing -> getInfo name+ Just alias -> getInfo alias <> getInfo name+ aliases = maybe TablishAliasesNone TablishAliasesT maybe_alias++ return $ TablishTable r aliases name++ -- This is slightly complicated because `full` is permitted as a table+ -- alias, but `full` is also a reserved word that may follow a table alias.+ -- Strategy is lookahead... if the next tokens are `full` and `outer` and+ -- `join`, we definitely don't have an alias so return Nothing. Otherwise,+ -- there may or may not be an alias.+ aliasP :: Parser (Maybe (TableAlias Range))+ aliasP = choice+ [ do+ _ <- try $ P.lookAhead $ Tok.fullP >> optional Tok.outerP >> Tok.joinP+ return Nothing+ , optionMaybe $ (optional Tok.asP) >> tableAliasP+ ]++ tableSampleP :: Parser Range+ tableSampleP = do+ s <- Tok.tableSampleP+ _ <- Tok.openP+ _ <- choice $+ [ do+ s' <- Tok.bucketP+ _ <- Tok.numberP+ _ <- Tok.outP+ _ <- Tok.ofP+ _ <- Tok.numberP+ option s' $ do+ _ <- Tok.onP+ choice $ [ try $ Tok.randP >> Tok.openP >> Tok.closeP+ , snd <$> Tok.columnNameP+ ]+ , Tok.numberP >> (Tok.percentP <|> Tok.rowsP)+ , snd <$> Tok.byteAmountP+ ]+ e <- Tok.closeP+ return $ s <> e++singleTableWithViewsP :: Parser (Tablish RawNames Range)+singleTableWithViewsP = do+ table <- singleTableP+ views <- fmap (appEndo . fold . reverse) $ many $ Endo <$> lateralViewP+ return $ views table+++-- see https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView+lateralViewP :: Parser (Tablish RawNames Range -> Tablish RawNames Range)+lateralViewP = do+ s <- Tok.lateralP+ _ <- Tok.viewP+ lateralViewOuter <- optionMaybe Tok.outerP+ lateralViewExprs <- (:[]) <$> functionExprP+ let lateralViewWithOrdinality = False -- it's not a thing in Hive++ tAlias <- tableAliasP+ cAliases <- optionMaybe $ do+ _ <- Tok.asP+ columnAliasP `sepBy1` Tok.commaP+ let lateralViewAliases = case cAliases of+ Nothing -> TablishAliasesT tAlias+ Just cAliases' -> TablishAliasesTC tAlias cAliases'++ e = getInfo tAlias+ es = maybe [] (map getInfo) cAliases+ lateralViewInfo = s <> sconcat (e:|es)++ pure $ \ lhs -> TablishLateralView (getInfo lhs <> lateralViewInfo) LateralView{..} (Just lhs)+++optionalParensP :: Parser a -> Parser a+optionalParensP p = try p <|> P.between Tok.openP Tok.closeP p++manyParensP :: Parser a -> Parser a+manyParensP p = try p <|> P.between Tok.openP Tok.closeP (manyParensP p)++tablishP :: Parser (Tablish RawNames Range)+tablishP = do+ table <- singleTableWithViewsP+ joins <- fmap (appEndo . fold . reverse) $ many $ Endo <$> joinP+ return $ joins table++joinP :: Parser (Tablish RawNames Range -> Tablish RawNames Range)+joinP = do+ maybeJoinType <- optionMaybe $ innerJoinTypeP <|> crossJoinTypeP <|> try semiJoinTypeP <|> outerJoinTypeP+ joinType <- Tok.joinP >>= \ r -> return $ case maybeJoinType of+ Nothing -> JoinInner r+ Just joinType -> (<> r) <$> joinType++ rhs <- singleTableWithViewsP+ maybeCondition <- optionMaybe $ do+ _ <- Tok.onP <?> "condition in join clause"+ JoinOn <$> exprP++ let condition = case maybeCondition of+ Nothing -> let info = getInfo joinType <> getInfo rhs+ in JoinOn $ ConstantExpr info $ BooleanConstant info True+ Just c -> c+ joinType' = case (joinType, maybeCondition) of+ -- omitting the ON clause turns a LEFT SEMI JOIN into an INNER JOIN+ (JoinSemi r, Nothing) -> JoinInner r+ _ -> joinType++ let r lhs = getInfo lhs <> getInfo rhs <> getInfo condition+ return $ \ lhs ->+ TablishJoin (r lhs) joinType' condition lhs rhs++outerJoinTypeP :: Parser (JoinType Range)+outerJoinTypeP = do+ joinType <- choice+ [ Tok.leftP >>= \ r -> return $ JoinLeft r+ , Tok.rightP >>= \ r -> return $ JoinRight r+ , Tok.fullP >>= \ r -> return $ JoinFull r+ ]++ optional Tok.outerP++ return joinType++innerJoinTypeP :: Parser (JoinType Range)+innerJoinTypeP = Tok.innerP >>= \ r -> return $ JoinInner r++crossJoinTypeP :: Parser (JoinType Range)+crossJoinTypeP = Tok.crossP >>= \ r -> return $ JoinInner r++semiJoinTypeP :: Parser (JoinType Range)+semiJoinTypeP = do+ r <- Tok.leftP+ r' <- Tok.semiP+ return $ JoinSemi (r <> r')++constantP :: Parser (Constant Range)+constantP = choice+ [ uncurry (flip StringConstant)+ <$> (try (optional Tok.timestampP) >> Tok.stringP)++ , uncurry (flip NumericConstant) <$> Tok.numberP+ , NullConstant <$> Tok.nullP+ , uncurry (flip BooleanConstant) <$> choice+ [ Tok.trueP >>= \ r -> return (True, r)+ , Tok.falseP >>= \ r -> return (False, r)+ ]+ ]
+ src/Database/Sql/Hive/Parser/Internal.hs view
@@ -0,0 +1,41 @@+-- Copyright (c) 2017 Uber Technologies, Inc.+--+-- Permission is hereby granted, free of charge, to any person obtaining a copy+-- of this software and associated documentation files (the "Software"), to deal+-- in the Software without restriction, including without limitation the rights+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+-- copies of the Software, and to permit persons to whom the Software is+-- furnished to do so, subject to the following conditions:+--+-- The above copyright notice and this permission notice shall be included in+-- all copies or substantial portions of the Software.+--+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+-- THE SOFTWARE.++module Database.Sql.Hive.Parser.Internal where++import qualified Text.Parsec as P++import Database.Sql.Hive.Token+import Database.Sql.Position+import Control.Monad.Reader+import Data.Text.Lazy (Text)+import Data.Set (Set)+++type ScopeTableRef = Text++data ParserScope = ParserScope+ { selectTableAliases :: Maybe (Set ScopeTableRef) }+ deriving (Eq, Ord, Show)++type Parser = P.ParsecT [(Token, Position, Position)] Integer (Reader ParserScope)++getNextCounter :: Parser Integer+getNextCounter = P.modifyState (+1) >> P.getState
+ src/Database/Sql/Hive/Parser/Token.hs view
@@ -0,0 +1,915 @@+-- Copyright (c) 2017 Uber Technologies, Inc.+--+-- Permission is hereby granted, free of charge, to any person obtaining a copy+-- of this software and associated documentation files (the "Software"), to deal+-- in the Software without restriction, including without limitation the rights+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+-- copies of the Software, and to permit persons to whom the Software is+-- furnished to do so, subject to the following conditions:+--+-- The above copyright notice and this permission notice shall be included in+-- all copies or substantial portions of the Software.+--+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+-- THE SOFTWARE.++module Database.Sql.Hive.Parser.Token where+++import Database.Sql.Hive.Token+import Database.Sql.Hive.Parser.Internal++import Database.Sql.Position++import qualified Text.Parsec as P+import qualified Text.Parsec.Pos as P++import Data.Char (isDigit)+import Data.String+import Data.Text.Lazy hiding (foldl1, map, head, last, all, null, init)+import qualified Data.Text.Lazy.Encoding as TL+import Data.ByteString.Lazy (ByteString)+import qualified Data.ByteString.Lazy as BL+import Data.Semigroup ((<>))+++showTok :: (Token, Position, Position) -> String+showTok (t, _, _) = show t++posFromTok :: P.SourcePos ->+ (Token, Position, Position) ->+ [(Token, Position, Position)] ->+ P.SourcePos+posFromTok _ (_, pos, _) _ = flip P.setSourceLine (fromEnum $ positionLine pos)+ $ flip P.setSourceColumn (fromEnum $ positionColumn pos)+ $ P.initialPos "-"++tokEqualsP :: Token -> Parser Range+tokEqualsP tok = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok', s, e) =+ if tok == tok'+ then Just $ Range s e+ else Nothing++tokNotEqualsP :: Token -> Parser Range+tokNotEqualsP tok = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok', s, e) =+ if tok /= tok'+ then Just $ Range s e+ else Nothing++testNameTok :: (Token, Position, Position) -> Maybe (Text, Range)+testNameTok (tok, s, e) =+ case tok of+ TokWord _ name -> Just (name, Range s e)+ _ -> Nothing++variableSubstitutionP :: Parser Range+variableSubstitutionP = P.tokenPrim showTok posFromTok testVariableTok+ where+ testVariableTok (tok, s, e) =+ case tok of+ TokVariable _ _ -> Just (Range s e)+ _ -> Nothing++typeNameP :: Parser (Text, Range)+typeNameP = P.tokenPrim showTok posFromTok testNameTok++nodeNameP :: Parser (Text, Range)+nodeNameP = P.tokenPrim showTok posFromTok testNameTok++structFieldNameP :: Parser (Text, Range)+structFieldNameP = P.tokenPrim showTok posFromTok testNameTok++windowNameP :: Parser (Text, Range)+windowNameP = P.tokenPrim showTok posFromTok testNameTok++datePartP :: Parser (Text, Range)+datePartP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord _ name | toLower name `elem` parts -> Just (toLower name, Range s e)+ _ -> Nothing++ parts = [ "year", "yy", "yyyy"+ , "quarter", "qq", "q"+ , "month", "mm", "m"+ , "day", "dd", "d", "dy", "dayofyear", "y"+ , "week", "wk", "ww"+ , "hour", "hh"+ , "minute", "mi", "n"+ , "second", "ss", "s"+ , "millisecond", "ms"+ , "microsecond", "mcs", "us"+ ]+++schemaNameP :: Parser (Text, Range)+schemaNameP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord True name -> Just (name, Range s e)+ TokWord False name+ | wordCanBeSchemaName (wordInfo name) -> Just (name, Range s e)++ _ -> Nothing+++tableNameP :: Parser (Text, Range)+tableNameP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord True name -> Just (name, Range s e)+ TokWord False name+ | wordCanBeTableName (wordInfo name) -> Just (name, Range s e)++ _ -> Nothing+++projectionNameP :: Parser (Text, Range)+projectionNameP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord True name -> Just (name, Range s e)+ TokWord False name+ | wordCanBeTableName (wordInfo name) -> Just (name, Range s e)++ _ -> Nothing+++columnNameP :: Parser (Text, Range)+columnNameP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord True name -> Just (name, Range s e)+ TokWord False name+ | wordCanBeColumnName (wordInfo name) -> Just (name, Range s e)++ _ -> Nothing+++functionNameP :: Parser (Text, Range)+functionNameP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord True name -> Just (name, Range s e)+ TokWord False name+ | wordCanBeFunctionName (wordInfo name) ->+ Just (name, Range s e)++ _ -> Nothing+++-- a property name part may include dashes+propertyNamePartP :: Parser (Text, Range)+propertyNamePartP = do+ xs <- namePartPartP `P.sepBy1` minusP+ let namePartParts = map fst xs+ ranges = map snd xs+ namePart = intercalate "-" namePartParts+ info = head ranges <> last ranges+ return $ (namePart, info)+ where+ namePartPartP = P.tokenPrim showTok posFromTok testNameTok++-- a property name is composed of name-parts separated by dots+propertyNameP :: Parser ([Text], Range)+propertyNameP = do+ firstPart <- propertyNamePartP+ subsequentParts <- P.many $ dotP >> propertyNamePartP+ let allParts = firstPart:subsequentParts+ s = snd $ head allParts+ e = snd $ last allParts+ nameParts = map fst allParts+ return $ (nameParts, s <> e)+++keywordP :: Text -> Parser Range+keywordP keyword = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokWord False name+ | name == keyword -> Just (Range s e)++ _ -> Nothing++fieldTypeP :: Parser (Text, Range)+fieldTypeP = P.tokenPrim showTok posFromTok testTok+ where+ isField :: (Eq s, IsString s) => s -> Bool+ isField = flip elem+ [ "century", "day", "decade", "doq", "dow", "doy", "epoch"+ , "hour", "isodow", "isoweek", "isoyear", "microseconds"+ , "millennium", "milliseconds", "minute", "month", "quarter"+ , "second", "time zone", "timezone_hour", "timezone_minute"+ , "week", "year"+ ]++ testTok (tok, s, e) = case tok of+ TokWord _ field | isField field -> Just (field, Range s e)+ TokString field | isField field -> Just (TL.decodeUtf8 field, Range s e)+ _ -> Nothing++periodP :: Parser (Text, Range)+periodP = P.tokenPrim showTok posFromTok testTok+ where+ isPeriod :: (Eq s, IsString s) => s -> Bool+ isPeriod = flip elem [ "day", "hour", "minute", "month", "second", "year" ]++ testTok (tok, s, e) = case tok of+ TokWord _ period | isPeriod period -> Just (period, Range s e)+ TokString period | isPeriod period -> Just (TL.decodeUtf8 period, Range s e)+ _ -> Nothing++byteAmountP :: Parser (Text, Range)+byteAmountP = P.tokenPrim showTok posFromTok testTok+ where+ isByteAmount :: Text -> Bool+ isByteAmount text =+ let str = unpack $ toLower text+ in and [ not $ null str+ , last str `elem` ['b', 'k', 'm', 'g']+ , all isDigit $ init str+ ]++ testTok (tok, s, e) = case tok of+ TokWord False byteAmount | isByteAmount byteAmount -> Just (byteAmount, Range s e)+ _ -> Nothing++stringP :: Parser (ByteString, Range)+stringP = do+ tokens <- P.many1 singleStringTokenP+ let s = foldl1 BL.append $ map fst tokens+ r = foldl1 (<>) $ map snd tokens++ pure $ (s, r)+ where+ singleStringTokenP :: Parser (ByteString, Range)+ singleStringTokenP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokString string -> Just (string, Range s e)+ _ -> Nothing++numberP :: Parser (Text, Range)+numberP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (tok, s, e) = case tok of+ TokNumber number -> Just (number, Range s e)+ _ -> Nothing++dotP :: Parser Range+dotP = symbolP "."++equalP :: Parser Range+equalP = symbolP "="++colonP :: Parser Range+colonP = symbolP ":"++symbolP :: Text -> Parser Range+symbolP op = tokEqualsP $ TokSymbol op++starP :: Parser Range+starP = symbolP "*"++openP :: Parser Range+openP = symbolP "("++closeP :: Parser Range+closeP = symbolP ")"++openBracketP :: Parser Range+openBracketP = symbolP "["++closeBracketP :: Parser Range+closeBracketP = symbolP "]"++openAngleP :: Parser Range+openAngleP = symbolP "<"++closeAngleP :: Parser Range+closeAngleP = symbolP ">"++castP :: Parser Range+castP = keywordP "cast"++castOpP :: Parser Range+castOpP = symbolP "::"++minusP :: Parser Range+minusP = symbolP "-"++accessRankP :: Parser Range+accessRankP = keywordP "accessrank"++addP :: Parser Range+addP = keywordP "add"++afterP :: Parser Range+afterP = keywordP "after"++allP :: Parser Range+allP = keywordP "all"++alterP :: Parser Range+alterP = keywordP "alter"++analyzeP :: Parser Range+analyzeP = keywordP "analyze"++andP :: Parser Range+andP = keywordP "and"++arrayP :: Parser Range+arrayP = keywordP "array"++asP :: Parser Range+asP = keywordP "as"++ascP :: Parser Range+ascP = keywordP "asc"++atP :: Parser Range+atP = keywordP "at"++autoP :: Parser Range+autoP = keywordP "auto"++avroP :: Parser Range+avroP = keywordP "avro"++bestP :: Parser Range+bestP = keywordP "best"++betweenP :: Parser Range+betweenP = keywordP "between"++bucketP :: Parser Range+bucketP = keywordP "bucket"++bucketsP :: Parser Range+bucketsP = keywordP "buckets"++byP :: Parser Range+byP = keywordP "by"++cacheP :: Parser Range+cacheP = keywordP "cache"++cascadeP :: Parser Range+cascadeP = keywordP "cascade"++caseP :: Parser Range+caseP = keywordP "case"++changeP :: Parser Range+changeP = keywordP "change"++clusterP :: Parser Range+clusterP = keywordP "cluster"++clusteredP :: Parser Range+clusteredP = keywordP "clustered"++collectionP :: Parser Range+collectionP = keywordP "collection"++columnP :: Parser Range+columnP = keywordP "column"++columnsP :: Parser Range+columnsP = keywordP "columns"++commaP :: Parser Range+commaP = symbolP ","++commentP :: Parser Range+commentP = keywordP "comment"++commitP :: Parser Range+commitP = keywordP "commit"++computeP :: Parser Range+computeP = keywordP "compute"++createP :: Parser Range+createP = keywordP "create"++crossP :: Parser Range+crossP = keywordP "cross"++cubeP :: Parser Range+cubeP = keywordP "cube"++currentP :: Parser Range+currentP = keywordP "current"++currentDatabaseP :: Parser (Text, Range)+currentDatabaseP = ("current_database",) <$> keywordP "current_database"++currentDateP :: Parser (Text, Range)+currentDateP = ("current_date",) <$> keywordP "current_date"++currentSchemaP :: Parser (Text, Range)+currentSchemaP = ("current_schema",) <$> keywordP "current_schema"++currentTimeP :: Parser (Text, Range)+currentTimeP = ("current_time",) <$> keywordP "current_time"++currentTimestampP :: Parser (Text, Range)+currentTimestampP = ("current_timestamp",) <$> keywordP "current_timestamp"++currentUserP :: Parser (Text, Range)+currentUserP = ("current_user",) <$> keywordP "current_user"++dataP :: Parser Range+dataP = keywordP "data"++databaseP :: Parser Range+databaseP = keywordP "database"++dateDiffP :: Parser Range+dateDiffP = keywordP "datediff"++dbPropertiesP :: Parser Range+dbPropertiesP = keywordP "dbproperties"++defaultP :: Parser Range+defaultP = keywordP "default"++definedP :: Parser Range+definedP = keywordP "defined"++deleteP :: Parser Range+deleteP = keywordP "delete"++delimitedP :: Parser Range+delimitedP = keywordP "delimited"++descP :: Parser Range+descP = keywordP "desc"++describeP :: Parser Range+describeP = keywordP "describe" P.<|> keywordP "desc"++directoryP :: Parser Range+directoryP = keywordP "directory"++distinctP :: Parser Range+distinctP = keywordP "distinct"++distributeP :: Parser Range+distributeP = keywordP "distribute"++dropP :: Parser Range+dropP = keywordP "drop"++elseP :: Parser Range+elseP = keywordP "else"++encodingP :: Parser Range+encodingP = keywordP "encoding"++endP :: Parser Range+endP = keywordP "end"++escapeP :: Parser Range+escapeP = keywordP "escape"++escapedP :: Parser Range+escapedP = keywordP "escaped"++excludingP :: Parser Range+excludingP = keywordP "excluding"++existsP :: Parser Range+existsP = keywordP "exists"++explainP :: Parser Range+explainP = keywordP "explain"++externalP :: Parser Range+externalP = keywordP "external"++extractP :: Parser Range+extractP = keywordP "extract"++falseP :: Parser Range+falseP = keywordP "false"++fieldsP :: Parser Range+fieldsP = keywordP "fields"++firstP :: Parser Range+firstP = keywordP "first"++followingP :: Parser Range+followingP = keywordP "following"++forP :: Parser Range+forP = keywordP "for"++formatP :: Parser Range+formatP = keywordP "format"++fromP :: Parser Range+fromP = keywordP "from"++functionP :: Parser Range+functionP = keywordP "function"++fullP :: Parser Range+fullP = keywordP "full"++globalP :: Parser Range+globalP = keywordP "global"++grantP :: Parser Range+grantP = keywordP "grant"++groupP :: Parser Range+groupP = keywordP "group"++groupingP :: Parser Range+groupingP = keywordP "grouping"++havingP :: Parser Range+havingP = keywordP "having"++ifP :: Parser Range+ifP = keywordP "if"++ignoreP :: Parser Range+ignoreP = keywordP "ignore"++inP :: Parser Range+inP = keywordP "in"++includingP :: Parser Range+includingP = keywordP "including"++inPathP :: Parser Range+inPathP = keywordP "inpath"++innerP :: Parser Range+innerP = keywordP "inner"++inputFormatP :: Parser Range+inputFormatP = keywordP "inputformat"++insertP :: Parser Range+insertP = keywordP "insert"++intervalP :: Parser Range+intervalP = keywordP "interval"++intoP :: Parser Range+intoP = keywordP "into"++isP :: Parser Range+isP = keywordP "is"++itemsP :: Parser Range+itemsP = keywordP "items"++joinP :: Parser Range+joinP = keywordP "join"++keysP :: Parser Range+keysP = keywordP "keys"++ksafeP :: Parser Range+ksafeP = keywordP "ksafe"++lastP :: Parser Range+lastP = keywordP "last"++lateralP :: Parser Range+lateralP = keywordP "lateral"++leftP :: Parser Range+leftP = keywordP "left"++likeP :: Parser Range+likeP = keywordP "like"++limitP :: Parser Range+limitP = keywordP "limit"++linesP :: Parser Range+linesP = keywordP "lines"++loadP :: Parser Range+loadP = keywordP "load"++localP :: Parser Range+localP = keywordP "local"++localTimeP :: Parser (Text, Range)+localTimeP = ("localtime",) <$> keywordP "localtime"++localTimestampP :: Parser (Text, Range)+localTimestampP = ("localtimestamp",) <$> keywordP "localtimestamp"++locationP :: Parser Range+locationP = keywordP "location"++mapP :: Parser Range+mapP = keywordP "map"++metadataP :: Parser Range+metadataP = keywordP "metadata"++noP :: Parser Range+noP = keywordP "no"++nodeP :: Parser Range+nodeP = keywordP "node"++nodesP :: Parser Range+nodesP = keywordP "nodes"++noScanP :: Parser Range+noScanP = keywordP "noscan"++notP :: Parser Range+notP = keywordP "not"++notOperatorP :: Parser Range+notOperatorP = keywordP "not" P.<|> symbolP "!"++nullP :: Parser Range+nullP = keywordP "null"++nullsP :: Parser Range+nullsP = keywordP "nulls"++nullsequalP :: Parser Range+nullsequalP = keywordP "nullsequal"++ofP :: Parser Range+ofP = keywordP "of"++offsetP :: Parser Range+offsetP = keywordP "offset"++onP :: Parser Range+onP = keywordP "on"++orP :: Parser Range+orP = keywordP "or"++orcP :: Parser Range+orcP = keywordP "orc"++orderP :: Parser Range+orderP = keywordP "order"++overlapsP :: Parser Range+overlapsP = keywordP "overlaps"++overwriteP :: Parser Range+overwriteP = keywordP "overwrite"++outP :: Parser Range+outP = keywordP "out"++outerP :: Parser Range+outerP = keywordP "outer"++outputFormatP :: Parser Range+outputFormatP = keywordP "outputformat"++overP :: Parser Range+overP = keywordP "over"++parametersP :: Parser Range+parametersP = keywordP "parameters"++parquetP :: Parser Range+parquetP = keywordP "parquet"++partitionP :: Parser Range+partitionP = keywordP "partition"++partitionedP :: Parser Range+partitionedP = keywordP "partitioned"++percentP :: Parser Range+percentP = keywordP "percent"++precedingP :: Parser Range+precedingP = keywordP "preceding"++preserveP :: Parser Range+preserveP = keywordP "preserve"++projectionP :: Parser Range+projectionP = keywordP "projection"++projectionsP :: Parser Range+projectionsP = keywordP "projections"++protectionP :: Parser Range+protectionP = keywordP "protection"++purgeP :: Parser Range+purgeP = keywordP "purge"++randP :: Parser Range+randP = keywordP "rand"++rangeP :: Parser Range+rangeP = keywordP "range"++rcFileP :: Parser Range+rcFileP = keywordP "rcfile"++regexpP :: Parser Range+regexpP = keywordP "regexp"++reloadP :: Parser Range+reloadP = keywordP "reload"++renameP :: Parser Range+renameP = keywordP "rename"++restrictP :: Parser Range+restrictP = keywordP "restrict"++revokeP :: Parser Range+revokeP = keywordP "revoke"++rlikeP :: Parser Range+rlikeP = keywordP "rlike"++rightP :: Parser Range+rightP = keywordP "right"++rollbackP :: Parser Range+rollbackP = keywordP "rollback"++rollupP :: Parser Range+rollupP = keywordP "rollup"++rowP :: Parser Range+rowP = keywordP "row"++rowsP :: Parser Range+rowsP = keywordP "rows"++schemaP :: Parser Range+schemaP = keywordP "schema"++segmentedP :: Parser Range+segmentedP = keywordP "segmented"++selectP :: Parser Range+selectP = keywordP "select"++semicolonP :: Parser Range+semicolonP = symbolP ";"++notSemicolonP :: Parser Range+notSemicolonP = tokNotEqualsP $ TokSymbol ";"++semiP :: Parser Range+semiP = keywordP "semi"++sessionUserP :: Parser (Text, Range)+sessionUserP = ("session_user",) <$> keywordP "session_user"++sequenceFileP :: Parser Range+sequenceFileP = keywordP "sequencefile"++serdeP :: Parser Range+serdeP = keywordP "serde"++serdePropertiesP :: Parser Range+serdePropertiesP = keywordP "serdeproperties"++setP :: Parser Range+setP = keywordP "set"++setsP :: Parser Range+setsP = keywordP "sets"++showP :: Parser Range+showP = keywordP "show"++sortP :: Parser Range+sortP = keywordP "sort"++sortedP :: Parser Range+sortedP = keywordP "sorted"++statisticsP :: Parser Range+statisticsP = keywordP "statistics"++storedP :: Parser Range+storedP = keywordP "stored"++structP :: Parser Range+structP = keywordP "struct"++sysDateP :: Parser (Text, Range)+sysDateP = ("sysdate",) <$> keywordP "sysdate"++tableP :: Parser Range+tableP = keywordP "table"++tableSampleP :: Parser Range+tableSampleP = keywordP "tablesample"++tblPropertiesP :: Parser Range+tblPropertiesP = keywordP "tblproperties"++temporaryP :: Parser Range+temporaryP = keywordP "temporary" P.<|> keywordP "temp"++terminatedP :: Parser Range+terminatedP = keywordP "terminated"++textFileP :: Parser Range+textFileP = keywordP "textfile"++thenP :: Parser Range+thenP = keywordP "then"++timeseriesP :: Parser Range+timeseriesP = keywordP "timeseries"++timestampP :: Parser Range+timestampP = keywordP "timestamp"++toP :: Parser Range+toP = keywordP "to"++trueP :: Parser Range+trueP = keywordP "true"++truncateP :: Parser Range+truncateP = keywordP "truncate"++unboundedP :: Parser Range+unboundedP = keywordP "unbounded"++unionP :: Parser Range+unionP = keywordP "union"++uniontypeP :: Parser Range+uniontypeP = keywordP "uniontype"++unknownP :: Parser Range+unknownP = keywordP "unknown"++unsegmentedP :: Parser Range+unsegmentedP = keywordP "unsegmented"++useP :: Parser Range+useP = keywordP "use"++userP :: Parser (Text, Range)+userP = ("user",) <$> keywordP "user"++valuesP :: Parser Range+valuesP = keywordP "values"++viewP :: Parser Range+viewP = keywordP "view"++whenP :: Parser Range+whenP = keywordP "when"++whereP :: Parser Range+whereP = keywordP "where"++windowP :: Parser Range+windowP = keywordP "window"++withP :: Parser Range+withP = keywordP "with"++inequalityOpP :: Parser (Text, Range)+inequalityOpP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (TokSymbol op, s, e)+ | op `elem` ["<", ">", "<=", ">="] = Just (op, Range s e)++ testTok _ = Nothing++equalityOpP :: Parser (Text, Range)+equalityOpP = P.tokenPrim showTok posFromTok testTok+ where+ testTok (TokSymbol op, s, e)+ | op `elem` ["=", "==", "<=>", "<>", "!="] = Just (op, Range s e)++ testTok _ = Nothing
+ src/Database/Sql/Hive/Scanner.hs view
@@ -0,0 +1,317 @@+-- Copyright (c) 2017 Uber Technologies, Inc.+--+-- Permission is hereby granted, free of charge, to any person obtaining a copy+-- of this software and associated documentation files (the "Software"), to deal+-- in the Software without restriction, including without limitation the rights+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+-- copies of the Software, and to permit persons to whom the Software is+-- furnished to do so, subject to the following conditions:+--+-- The above copyright notice and this permission notice shall be included in+-- all copies or substantial portions of the Software.+--+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+-- THE SOFTWARE.++module Database.Sql.Hive.Scanner where++import Prelude hiding ((&&), (||), not)++import Data.Int (Int64)+import Data.Text.Lazy (Text)+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Encoding as TL+import Data.ByteString.Lazy (ByteString)++import Data.List (sortBy)+import Data.Foldable (asum)+import Data.Char (isAlphaNum, isAlpha, isSpace, isDigit)++import Control.Monad.State++import Database.Sql.Position+import Database.Sql.Hive.Token++import Data.Predicate.Class+++isWordBody :: Char -> Bool+isWordBody = isAlphaNum || (== '_') || (== '$')++isHSpace :: Char -> Bool+isHSpace = isSpace && not (== '\n')++operators :: [Text]+operators = sortBy (flip compare)+ [ "+", "-", "*", "/", "%"+ , "&", "|", "^", "~"+ , "!"+ , ":"+ , "!=", "<>", ">", "<", ">=", "<=", "<=>", "=", "=="+ , "(", ")", "[", "]", ",", ";"+ ]++isOperator :: Char -> Bool+isOperator c = elem c $ map TL.head operators++isPlusOrMinus :: Text -> Bool+isPlusOrMinus s = elem s ["+", "-"]++parseNumber :: Text -> ((Token, Int64), Text)+parseNumber = runState $ do+ ipart <- state $ TL.span isDigit+ gets (TL.take 1) >>= \case+ "" -> pure (TokNumber ipart, TL.length ipart)+ "." -> do+ modify $ TL.drop 1+ fpart <- state $ TL.span isDigit+ gets (TL.take 1) >>= \case+ "e" -> do+ modify $ TL.drop 1+ sign <- gets (TL.take 1) >>= \case+ s | isPlusOrMinus s -> modify (TL.drop 1) >> pure s+ _ -> pure ""+ state (TL.span isDigit) >>= \ epart ->+ let number = TL.concat [ipart, ".", fpart, "e", sign, epart]+ in pure ( if TL.null epart+ then TokError "..."+ else TokNumber number+ , TL.length number+ )+ _ ->+ let number = TL.concat [ipart, ".", fpart]+ in pure ( TokNumber number, TL.length number )+ "e" -> do+ modify $ TL.drop 1+ sign <- gets (TL.take 1) >>= \case+ s | isPlusOrMinus s -> modify (TL.drop 1) >> pure s+ _ -> pure ""+ epart <- state $ TL.span isDigit+ gets (TL.take 1) >>= \case+ c | (not . TL.null && isWordBody . TL.head) c || TL.null epart -> do+ rest <- state $ TL.span isWordBody+ let word = TL.concat [ipart, "e", sign, epart, rest]+ pure (TokWord False word, TL.length word)+ | otherwise ->+ let number = TL.concat [ipart, "e", sign, epart]+ in pure (TokNumber number, TL.length number)++ c | (isAlpha || (== '_')) (TL.head c) -> do+ rest <- state $ TL.span (isAlpha || (== '_'))+ let word = TL.concat [ipart, rest]+ pure (TokWord False word, TL.length word)++ _ -> pure (TokNumber ipart, TL.length ipart)++tokenize :: Text -> [(Token, Position, Position)]+tokenize = go (Position 1 0 0)+ where+ go :: Position -> Text -> [(Token, Position, Position)]+ go _ "" = []+ go p t = case TL.head t of+ c | isAlpha c ->+ case tokUnquotedWord p t of+ (name, rest, p') -> (TokWord False name, p, p') : go p' rest++ c | isDigit c ->+ let ((token, len), rest) = parseNumber t+ p' = advanceHorizontal len p+ in (token, p, p') : go p' rest++ '$' | "${" `TL.isPrefixOf` t ->+ let ((token, len), rest) = parseVariable t+ p' = advanceHorizontal len p+ in (token, p, p') : go p' rest++ '`' ->+ case tokQuotedWord p t of+ Left p' -> [(TokError "end of input inside name", p, p')]+ Right (name, rest, p') -> (TokWord True name, p, p') : go p' rest++ c | (== '\n') c -> let (newlines, rest) = TL.span (== '\n') t+ p' = advanceVertical (TL.length newlines) p+ in go p' rest++ c | isHSpace c -> let (spaces, rest) = TL.span isHSpace t+ p' = advanceHorizontal (TL.length spaces) p+ in go p' rest++ -- comments+ '-' | "--" `TL.isPrefixOf` t ->+ let (comment, rest) = TL.span (/= '\n') t+ p' = advanceVertical 1+ (advanceHorizontal (TL.length comment) p)++ in go p' (TL.drop 1 rest)++ -- Join hints+ -- T399601: restrict to context aware join hints, which should follow a select+ '/' | "/*+" `TL.isPrefixOf` t ->+ case TL.breakOn "*/" t of+ (comment, "") ->+ let p' = advance comment p+ in [(TokError "unterminated join hint", p, p')]+ (comment, rest) ->+ let p' = advance (TL.append comment "*/") p+ in go p' $ TL.drop 2 rest++ c | c == '\'' || c == '"' ->+ case tokString c p t of+ Left (tok, p') -> [(tok, p, p')]+ Right (string, rest, p') -> (TokString string, p, p') : go p' rest++ '.' ->+ let p' = advanceHorizontal 1 p+ in (TokSymbol ".", p, p') : go p' (TL.tail t)+++ c | isOperator c -> case readOperator t of++ Just (sym, rest) -> let p' = advanceHorizontal (TL.length sym) p+ in (TokSymbol sym, p, p') : go p' rest++ Nothing ->+ let opchars = TL.take 5 t+ p' = advance opchars p+ message = unwords+ [ "unrecognized operator starting with"+ , show opchars+ ]+ in [(TokError message, p, p')]++ c ->+ let message = unwords+ [ "unmatched character ('" ++ show c ++ "') at position"+ , show p+ ]++ in [(TokError message, p, advanceHorizontal 1 p)]++ readOperator t = asum+ $ map (\ op -> (op,) <$> TL.stripPrefix op t) operators++tokUnquotedWord :: Position -> Text -> (Text, Text, Position)+tokUnquotedWord pos input =+ case TL.span (isAlphaNum || (== '_')) input of+ (word, rest) -> (TL.toLower word, rest, advanceHorizontal (TL.length word) pos)++tokQuotedWord :: Position -> Text -> Either Position (Text, Text, Position)+tokQuotedWord pos = go (advanceHorizontal 1 pos) [] . TL.tail+ where+ go p _ "" = Left p+ go p ts input = case TL.head input of+ c | c == '`' ->+ let (quotes, rest) = TL.span (== '`') input+ len = TL.length quotes+ in if len `mod` 2 == 0+ then go (advanceHorizontal len p)+ (TL.take (len `div` 2) quotes :ts)+ rest++ else Right ( TL.concat $ reverse (TL.take (len `div` 2) quotes : ts)+ , rest+ , advanceHorizontal len p+ )++ _ -> let (t, rest) = TL.span (/= '`') input+ in go (advance t p) (t:ts) rest++tokString :: Char -> Position -> Text -> Either (Token, Position) (ByteString, Text, Position)+tokString quote pos = go (advanceHorizontal 1 pos) [] . TL.drop 1+ where+ boring = not . (`elem` [quote, '\\'])+ halve txt = TL.take (TL.length txt `div` 2) txt+ go p ts input = case TL.span boring input of+ (cs, "") -> Left (TokError "end of input inside string", advance cs p)+ ("", rest) -> handleSlashes p ts rest+ (cs, rest) -> handleSlashes (advance cs p) (cs:ts) rest++ handleSlashes p ts input = case TL.span (== '\\') input of+ (cs, "") -> Left (TokError "end of input inside string", advance cs p)+ ("", _) -> handleQuote p ts input+ (slashes, rest) ->+ let len = TL.length slashes+ in if len `mod` 2 == 0+ then go (advanceHorizontal len p) (halve slashes:ts) rest+ else case TL.splitAt 1 rest of+ (c, rest')+ | c == "a" -> go (advanceHorizontal (len + 1) p) ("\a":halve slashes:ts) rest'+ | c == "b" -> go (advanceHorizontal (len + 1) p) ("\BS":halve slashes:ts) rest'+ | c == "f" -> go (advanceHorizontal (len + 1) p) ("\FF":halve slashes:ts) rest'+ | c == "n" -> go (advanceHorizontal (len + 1) p) ("\n":halve slashes:ts) rest'+ | c == "r" -> go (advanceHorizontal (len + 1) p) ("\r":halve slashes:ts) rest'+ | c == "t" -> go (advanceHorizontal (len + 1) p) ("\t":halve slashes:ts) rest'+ | c == "v" -> go (advanceHorizontal (len + 1) p) ("\v":halve slashes:ts) rest'+ | c == "'" -> go (advanceHorizontal (len + 1) p) ("'":halve slashes:ts) rest'+ | c == "\"" -> go (advanceHorizontal (len + 1) p) ("\"":halve slashes:ts) rest'+ | otherwise -> go (advanceHorizontal (len + 1) p) (c:"\\":halve slashes:ts) rest'++ handleQuote p ts input = case TL.splitAt 1 input of+ (c, rest) | c == TL.singleton quote ->+ Right ( TL.encodeUtf8 $ TL.concat $ reverse ts+ , rest+ , advanceHorizontal 1 p+ )+ x -> error $ "this shouldn't happen: handleQuote splitInput got " ++ show x++parseVariable :: Text -> ((Token, Int64), Text)+parseVariable = runState $ do+ let endOfInput = "end of input inside variable substitution"+ missingNamespaceOrName = "variable substitutions must have a namespace and a name"+ colon = (== ':')+ closeCurly = (== '}')++ modify $ TL.drop 2 -- consume the initial ${+ namespace <- state (TL.break colon)++ gets (TL.take 1) >>= \case+ "" -> -- there was no : in the rest of the text+ let varLen = 2 + TL.length namespace+ in if (not $ TL.null namespace) && (TL.last namespace == '}')+ then pure (TokError missingNamespaceOrName, varLen)+ else pure (TokError endOfInput, varLen)++ _ -> do+ modify $ TL.drop 1 -- consume the :+ gets (TL.take 2) >>= \case+ "${" -> do -- RECURSE!+ ((subName, subLen), rest) <- gets parseVariable+ _ <- put rest+ gets (TL.take 1) >>= \case+ "}" -> do+ modify $ TL.drop 1 -- consume the }+ let varLen = 2 + TL.length namespace + 1 + subLen + 1+ varName = TokVariable namespace $ DynamicName subName+ pure $ liftInnerErrors $ enforceNamespaces $ (varName, varLen)+ _ -> do+ let varLen = 2 + TL.length namespace + 1 + subLen+ pure (TokError endOfInput, varLen)++ _ -> do+ name <- state (TL.break closeCurly)+ gets (TL.take 1) >>= \case+ "" -> -- there was no } in the rest of the text+ let varLen = 2 + TL.length namespace + 1 + TL.length name+ in pure (TokError endOfInput, varLen)+ _ -> do+ modify $ TL.drop 1 -- consume the }+ let varLen = 2 + TL.length namespace + 1 + TL.length name + 1+ if TL.null name+ then pure (TokError missingNamespaceOrName, varLen)+ else pure $ enforceNamespaces $ (TokVariable namespace $ StaticName name, varLen)+ where+ enforceNamespaces tok@(TokVariable ns _, len) =+ let allowedNamespaces = ["hiveconf", "system", "env", "define", "hivevar"]+ permitted = (`elem` allowedNamespaces)+ in if permitted ns+ then tok+ else (TokError $ "bad namespace in variable substitution: " ++ show ns, len)+ enforceNamespaces x = x++ liftInnerErrors (TokVariable _ (DynamicName (TokError msg)), len) = (TokError msg, len)+ liftInnerErrors x = x
+ src/Database/Sql/Hive/Token.hs view
@@ -0,0 +1,117 @@+-- Copyright (c) 2017 Uber Technologies, Inc.+--+-- Permission is hereby granted, free of charge, to any person obtaining a copy+-- of this software and associated documentation files (the "Software"), to deal+-- in the Software without restriction, including without limitation the rights+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+-- copies of the Software, and to permit persons to whom the Software is+-- furnished to do so, subject to the following conditions:+--+-- The above copyright notice and this permission notice shall be included in+-- all copies or substantial portions of the Software.+--+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+-- THE SOFTWARE.++module Database.Sql.Hive.Token where++import Data.Text.Lazy (Text)+import Data.ByteString.Lazy (ByteString)+import qualified Data.Map as M++data Token = TokWord !Bool !Text+ | TokString !ByteString+ | TokNumber !Text+ | TokSymbol !Text+ | TokVariable !Text VariableName -- the Text is for namespace, the+ -- Token is the param name which+ -- may be another TokVariable!+ | TokError !String+ deriving (Show, Eq)++data VariableName = StaticName !Text+ | DynamicName Token+ deriving (Show, Eq)++data WordInfo = WordInfo+ { wordCanBeSchemaName :: !Bool+ , wordCanBeTableName :: !Bool+ , wordCanBeColumnName :: !Bool+ , wordCanBeFunctionName :: !Bool+ }+++wordInfo :: Text -> WordInfo+wordInfo word = maybe (WordInfo True True True True) id $ M.lookup word $ M.fromList+ [ ("all", WordInfo False True False True)+ , ("and", WordInfo False False False False)+ , ("asc", WordInfo False False False False)+ , ("array", WordInfo False False False True)+ , ("as", WordInfo False False True False)+ , ("at", WordInfo False True True False)+ , ("between", WordInfo False False False False)+ , ("by", WordInfo False False False False)+ , ("case", WordInfo False False False False)+ , ("cluster", WordInfo False False True False)+ , ("comma", WordInfo False False False False)+ , ("cross", WordInfo False False False False)+ , ("current_database", WordInfo False False False True)+ , ("current_date", WordInfo False False False True)+ , ("current_schema", WordInfo False False False True)+ , ("current_timestamp", WordInfo False False False True)+ , ("current_time", WordInfo False False False False)+ , ("current_user", WordInfo False False False True)+ , ("desc", WordInfo False False False False)+ , ("distinct", WordInfo False False False False)+ , ("distribute", WordInfo False False False False)+ , ("else", WordInfo False False False False)+ , ("end", WordInfo False False False False)+ , ("false", WordInfo False False False False)+ , ("from", WordInfo False False False False)+ , ("full", WordInfo False True False False)+ , ("group", WordInfo False False True False)+ , ("having", WordInfo False False False False)+ , ("inner", WordInfo False False False False)+ , ("interval", WordInfo False False True False)+ , ("in", WordInfo False False False False)+ , ("insert", WordInfo False False False False)+ , ("is", WordInfo False False False False)+ , ("join", WordInfo False False False False)+ , ("lateral", WordInfo False False False True)+ , ("left", WordInfo False False False True)+ , ("limit", WordInfo False False False False)+ , ("localtimestamp", WordInfo False False False False)+ , ("localtime", WordInfo False True True False)+ , ("map", WordInfo False False False True)+ , ("not", WordInfo False False False False)+ , ("null", WordInfo False False False False)+ , ("on", WordInfo False False False False)+ , ("order", WordInfo False False True False)+ , ("or", WordInfo False False False False)+ , ("outer", WordInfo False False False False)+ , ("overlaps", WordInfo False False False False)+ , ("right", WordInfo False False False True)+ , ("segmented", WordInfo False False False False)+ , ("select", WordInfo False False False False)+ , ("semi", WordInfo False False False False)+ , ("session_user", WordInfo False False False True)+ , ("sort", WordInfo False False False False)+ , ("sysdate", WordInfo False False False True)+ , ("then", WordInfo False False False False)+ , ("time", WordInfo False False True False)+ , ("timezone", WordInfo False False True False)+ , ("true", WordInfo False False False False)+ , ("union", WordInfo False False False False)+ , ("unknown", WordInfo False False False False)+ , ("user", WordInfo False False True True)+ , ("when", WordInfo False False False False)+ , ("where", WordInfo False False False False)+ , ("window", WordInfo False False False False)+ , ("with", WordInfo False False False False)+ , ("zone", WordInfo False False False False)+ ]
+ src/Database/Sql/Hive/Type.hs view
@@ -0,0 +1,458 @@+-- Copyright (c) 2017 Uber Technologies, Inc.+--+-- Permission is hereby granted, free of charge, to any person obtaining a copy+-- of this software and associated documentation files (the "Software"), to deal+-- in the Software without restriction, including without limitation the rights+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+-- copies of the Software, and to permit persons to whom the Software is+-- furnished to do so, subject to the following conditions:+--+-- The above copyright notice and this permission notice shall be included in+-- all copies or substantial portions of the Software.+--+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+-- THE SOFTWARE.++{-# LANGUAGE DeriveDataTypeable #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DeriveFunctor #-}+{-# LANGUAGE DeriveTraversable #-}+{-# LANGUAGE StandaloneDeriving #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE UndecidableInstances #-}++module Database.Sql.Hive.Type where++import Database.Sql.Type hiding (insertValues, insertInfo)+import Database.Sql.Position+import Database.Sql.Info+import Database.Sql.Util.Columns+import Database.Sql.Util.Joins+import Database.Sql.Util.Lineage.Table+import Database.Sql.Util.Lineage.ColumnPlus+import Database.Sql.Util.Scope+import Database.Sql.Util.Schema+import Database.Sql.Util.Tables++import Control.Monad.Identity+import Control.Monad.Writer (tell)++import Data.Aeson+import qualified Data.Map as M+import qualified Data.Set as S+import qualified Data.Text.Lazy.Encoding as TL+import Data.ByteString.Lazy (ByteString)+import qualified Data.ByteString.Lazy as BL+import Data.Proxy (Proxy (..))++import GHC.Generics (Generic)+import Data.Data (Data)+++data Hive++deriving instance Data Hive++dialectProxy :: Proxy Hive+dialectProxy = Proxy++instance Dialect Hive where++ shouldCTEsShadowTables _ = False++ resolveCreateTableExtra _ Unused = pure Unused++ getSelectScope _ fromColumns selectionAliases = SelectScope+ { bindForWhere = bindFromColumns fromColumns+ , bindForGroup = bindFromColumns fromColumns+ , bindForHaving = bindBothColumns fromColumns selectionAliases+ , bindForOrder = bindAliasedColumns selectionAliases+ , bindForNamedWindow = bindFromColumns fromColumns+ }++ areLcolumnsVisibleInLateralViews _ = False+++data HiveStatement r a = HiveStandardSqlStatement (Statement Hive r a)+ | HiveUseStmt (Use a)+ | HiveAnalyzeStmt (Analyze r a)+ | HiveInsertDirectoryStmt (InsertDirectory r a)+ | HiveTruncatePartitionStmt (TruncatePartition r a)+ | HiveAlterTableSetLocationStmt (AlterTableSetLocation r a)+ | HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation r a)+ | HiveUnhandledStatement a++deriving instance (ConstrainSNames Data r a, Data r) => Data (HiveStatement r a)+deriving instance Generic (HiveStatement r a)+deriving instance ConstrainSNames Eq r a => Eq (HiveStatement r a)+deriving instance ConstrainSNames Show r a => Show (HiveStatement r a)+deriving instance ConstrainSASNames Functor r => Functor (HiveStatement r)+deriving instance ConstrainSASNames Foldable r => Foldable (HiveStatement r)+deriving instance ConstrainSASNames Traversable r => Traversable (HiveStatement r)++-- Important terminology note:+-- Hive "databases" are schemas.+-- https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Create/Drop/Alter/UseDatabase+data Use a = UseDatabase (UQSchemaName a)+ | UseDefault a+ deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)+++data Analyze r a = Analyze+ { analyzeInfo :: a+ , analyzeTable :: TableName r a+ }++deriving instance (ConstrainSNames Data r a, Data r) => Data (Analyze r a)+deriving instance Generic (Analyze r a)+deriving instance ConstrainSNames Eq r a => Eq (Analyze r a)+deriving instance ConstrainSNames Show r a => Show (Analyze r a)+deriving instance ConstrainSASNames Functor r => Functor (Analyze r)+deriving instance ConstrainSASNames Foldable r => Foldable (Analyze r)+deriving instance ConstrainSASNames Traversable r => Traversable (Analyze r)+++data InsertDirectory r a = InsertDirectory+ { insertDirectoryInfo :: a+ , insertDirectoryLocale :: InsertDirectoryLocale a+ , insertDirectoryPath :: Location a+ , insertDirectoryQuery :: Query r a+ }++deriving instance (ConstrainSNames Data r a, Data r) => Data (InsertDirectory r a)+deriving instance Generic (InsertDirectory r a)+deriving instance ConstrainSNames Eq r a => Eq (InsertDirectory r a)+deriving instance ConstrainSNames Show r a => Show (InsertDirectory r a)+deriving instance ConstrainSASNames Functor r => Functor (InsertDirectory r)+deriving instance ConstrainSASNames Foldable r => Foldable (InsertDirectory r)+deriving instance ConstrainSASNames Traversable r => Traversable (InsertDirectory r)+++data Location a = HDFSPath a ByteString+ deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)+++data InsertDirectoryLocale a = InsertDirectoryLocal a | InsertDirectoryHDFS+ deriving (Generic, Data, Eq, Show, Functor, Foldable, Traversable)++data StaticPartitionSpecItem r a = StaticPartitionSpecItem a (ColumnRef r a) (Constant a)++deriving instance (ConstrainSNames Data r a, Data r) => Data (StaticPartitionSpecItem r a)+deriving instance Generic (StaticPartitionSpecItem r a)+deriving instance ConstrainSNames Eq r a => Eq (StaticPartitionSpecItem r a)+deriving instance ConstrainSNames Show r a => Show (StaticPartitionSpecItem r a)+deriving instance ConstrainSASNames Functor r => Functor (StaticPartitionSpecItem r)+deriving instance ConstrainSASNames Foldable r => Foldable (StaticPartitionSpecItem r)+deriving instance ConstrainSASNames Traversable r => Traversable (StaticPartitionSpecItem r)++data DynamicPartitionSpecItem r a = DynamicPartitionSpecItem a (ColumnRef r a)++deriving instance (ConstrainSNames Data r a, Data r) => Data (DynamicPartitionSpecItem r a)+deriving instance Generic (DynamicPartitionSpecItem r a)+deriving instance ConstrainSNames Eq r a => Eq (DynamicPartitionSpecItem r a)+deriving instance ConstrainSNames Show r a => Show (DynamicPartitionSpecItem r a)+deriving instance ConstrainSASNames Functor r => Functor (DynamicPartitionSpecItem r)+deriving instance ConstrainSASNames Foldable r => Foldable (DynamicPartitionSpecItem r)+deriving instance ConstrainSASNames Traversable r => Traversable (DynamicPartitionSpecItem r)++-- Note: We don't track anything at the partition level (for now), so+-- we discard the info on which particular partition is being truncated.+data TruncatePartition r a = TruncatePartition+ { truncatePartitionInfo :: a+ , truncatePartitionTruncate :: Truncate r a+ }++deriving instance (ConstrainSNames Data r a, Data r) => Data (TruncatePartition r a)+deriving instance Generic (TruncatePartition r a)+deriving instance ConstrainSNames Eq r a => Eq (TruncatePartition r a)+deriving instance ConstrainSNames Show r a => Show (TruncatePartition r a)+deriving instance ConstrainSASNames Functor r => Functor (TruncatePartition r)+deriving instance ConstrainSASNames Foldable r => Foldable (TruncatePartition r)+deriving instance ConstrainSASNames Traversable r => Traversable (TruncatePartition r)++data AlterTableSetLocation r a = AlterTableSetLocation+ { alterTableSetLocationInfo :: a+ , alterTableSetLocationTable :: TableName r a+ , alterTableSetLocationLocation :: Location a+ }++deriving instance (ConstrainSNames Data r a, Data r) => Data (AlterTableSetLocation r a)+deriving instance ConstrainSNames Eq r a => Eq (AlterTableSetLocation r a)+deriving instance ConstrainSNames Show r a => Show (AlterTableSetLocation r a)+deriving instance ConstrainSASNames Functor r => Functor (AlterTableSetLocation r)+deriving instance ConstrainSASNames Foldable r => Foldable (AlterTableSetLocation r)+deriving instance ConstrainSASNames Traversable r => Traversable (AlterTableSetLocation r)+++data AlterPartitionSetLocation r a = AlterPartitionSetLocation+ { alterPartitionSetLocationInfo :: a+ , alterPartitionSetLocationTable :: TableName r a+ , alterPartitionSetLocationPartition :: [StaticPartitionSpecItem r a]+ , alterPartitionSetLocationLocation :: Location a+ }++deriving instance (ConstrainSNames Data r a, Data r) => Data (AlterPartitionSetLocation r a)+deriving instance ConstrainSNames Eq r a => Eq (AlterPartitionSetLocation r a)+deriving instance ConstrainSNames Show r a => Show (AlterPartitionSetLocation r a)+deriving instance ConstrainSASNames Functor r => Functor (AlterPartitionSetLocation r)+deriving instance ConstrainSASNames Foldable r => Foldable (AlterPartitionSetLocation r)+deriving instance ConstrainSASNames Traversable r => Traversable (AlterPartitionSetLocation r)+++instance HasJoins (HiveStatement ResolvedNames a) where+ getJoins (HiveStandardSqlStatement stmt) = getJoins stmt+ getJoins (HiveInsertDirectoryStmt stmt) =+ getJoins (QueryStmt $ insertDirectoryQuery stmt)+ getJoins (HiveUseStmt _) = S.empty+ getJoins (HiveAnalyzeStmt _) = S.empty+ getJoins (HiveTruncatePartitionStmt _) = S.empty+ getJoins (HiveAlterTableSetLocationStmt _) = S.empty+ getJoins (HiveAlterPartitionSetLocationStmt _) = S.empty+ getJoins (HiveUnhandledStatement _) = S.empty++instance HasTableLineage (HiveStatement ResolvedNames a) where+ getTableLineage (HiveStandardSqlStatement stmt) = tableLineage stmt+ getTableLineage (HiveUseStmt _) = M.empty+ getTableLineage (HiveAnalyzeStmt _) = M.empty+ getTableLineage (HiveInsertDirectoryStmt _) = M.empty+ getTableLineage (HiveTruncatePartitionStmt (TruncatePartition _ (Truncate _ (RTableName t _)))) =+ let table = mkFQTN t+ in M.singleton table $ S.singleton table+ getTableLineage (HiveAlterTableSetLocationStmt _) = M.empty+ getTableLineage (HiveAlterPartitionSetLocationStmt _) = M.empty+ getTableLineage (HiveUnhandledStatement _) = M.empty++instance HasColumnLineage (HiveStatement ResolvedNames Range) where+ getColumnLineage (HiveStandardSqlStatement stmt) = columnLineage stmt+ getColumnLineage (HiveUseStmt _) = returnNothing M.empty+ getColumnLineage (HiveAnalyzeStmt _) = returnNothing M.empty+ getColumnLineage (HiveInsertDirectoryStmt _) = returnNothing M.empty+ getColumnLineage (HiveTruncatePartitionStmt (TruncatePartition _ (Truncate _ (RTableName t SchemaMember{..})))) =+ returnNothing+ $ M.insert (Left $ fqtnToFQTN t) (singleTableSet (getInfo t) $ fqtnToFQTN t)+ $ M.fromList $ map ((\ fqcn -> (Right fqcn, singleColumnSet (getInfo t) fqcn)) . fqcnToFQCN . qualifyColumnName t) columnsList+ getColumnLineage (HiveAlterTableSetLocationStmt _) = returnNothing M.empty+ getColumnLineage (HiveAlterPartitionSetLocationStmt _) = returnNothing M.empty+ getColumnLineage (HiveUnhandledStatement _) = returnNothing M.empty++resolveHiveStatement :: HiveStatement RawNames a -> Resolver (HiveStatement ResolvedNames) a+resolveHiveStatement (HiveStandardSqlStatement stmt) = HiveStandardSqlStatement <$> resolveStatement stmt+resolveHiveStatement (HiveUseStmt stmt) = pure $ HiveUseStmt stmt+resolveHiveStatement (HiveAnalyzeStmt stmt) = HiveAnalyzeStmt <$> resolveAnalyze stmt+resolveHiveStatement (HiveInsertDirectoryStmt stmt) = HiveInsertDirectoryStmt <$> resolveInsertDirectory stmt+resolveHiveStatement (HiveTruncatePartitionStmt stmt) = HiveTruncatePartitionStmt <$> resolveTruncatePartition stmt+resolveHiveStatement (HiveAlterTableSetLocationStmt stmt) = HiveAlterTableSetLocationStmt <$> resolveAlterTableSetLocation stmt+resolveHiveStatement (HiveAlterPartitionSetLocationStmt stmt) = HiveAlterPartitionSetLocationStmt <$> resolveAlterPartitionSetLocation stmt+resolveHiveStatement (HiveUnhandledStatement stmt) = pure $ HiveUnhandledStatement stmt++resolveAnalyze :: Analyze RawNames a -> Resolver (Analyze ResolvedNames) a+resolveAnalyze Analyze{..} = do+ analyzeTable' <- resolveTableName analyzeTable+ pure Analyze+ { analyzeTable = analyzeTable'+ , ..+ }++resolveInsertDirectory :: InsertDirectory RawNames a -> Resolver (InsertDirectory ResolvedNames) a+resolveInsertDirectory InsertDirectory{..} = do+ insertDirectoryQuery' <- resolveQuery insertDirectoryQuery+ pure InsertDirectory+ { insertDirectoryQuery = insertDirectoryQuery'+ , ..+ }++resolveTruncatePartition :: TruncatePartition RawNames a -> Resolver (TruncatePartition ResolvedNames) a+resolveTruncatePartition TruncatePartition{..} = do+ truncatePartitionTruncate' <- resolveTruncate truncatePartitionTruncate+ pure TruncatePartition+ { truncatePartitionTruncate = truncatePartitionTruncate'+ , ..+ }++resolveAlterTableSetLocation :: AlterTableSetLocation RawNames a -> Resolver (AlterTableSetLocation ResolvedNames) a+resolveAlterTableSetLocation AlterTableSetLocation{..} = do+ alterTableSetLocationTable' <- resolveTableName alterTableSetLocationTable+ pure AlterTableSetLocation+ { alterTableSetLocationTable = alterTableSetLocationTable'+ , ..+ }++resolveAlterPartitionSetLocation :: AlterPartitionSetLocation RawNames a -> Resolver (AlterPartitionSetLocation ResolvedNames) a+resolveAlterPartitionSetLocation AlterPartitionSetLocation{..} = do+ alterPartitionSetLocationTable'@(RTableName fqtn table@SchemaMember{..}) <- resolveTableName alterPartitionSetLocationTable+ let tableInfo = tableNameInfo fqtn+ columnSet =+ [ ( Just $ RTableRef fqtn table+ , map (RColumnRef . (const tableInfo <$>) . qualifyColumnName fqtn) columnsList+ )+ ]+ alterPartitionSetLocationPartition' <- bindColumns columnSet+ $ forM alterPartitionSetLocationPartition+ $ \ (StaticPartitionSpecItem info column constant) -> do+ column' <- resolveColumnName column+ pure $ StaticPartitionSpecItem info column' constant+ pure AlterPartitionSetLocation+ { alterPartitionSetLocationTable = alterPartitionSetLocationTable'+ , alterPartitionSetLocationPartition = alterPartitionSetLocationPartition'+ , ..+ }++instance HasSchemaChange (HiveStatement ResolvedNames a) where+ getSchemaChange (HiveStandardSqlStatement stmt) = getSchemaChange stmt+ getSchemaChange (HiveUseStmt (UseDefault _)) = [UsePath [QSchemaName () None "default" NormalSchema]]+ getSchemaChange (HiveUseStmt (UseDatabase schema)) = [UsePath [void schema]]+ getSchemaChange (HiveAnalyzeStmt _) = []+ getSchemaChange (HiveInsertDirectoryStmt _) = []+ getSchemaChange (HiveTruncatePartitionStmt _) = []+ getSchemaChange (HiveAlterTableSetLocationStmt _) = [] -- In fact, it may very well have a schema change but+ -- we can't know without hitting the metastore.+ getSchemaChange (HiveAlterPartitionSetLocationStmt _) = []+ getSchemaChange (HiveUnhandledStatement _) = []++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (HiveStatement r a) where+ toJSON (HiveStandardSqlStatement stmt) = toJSON stmt+ toJSON (HiveInsertDirectoryStmt stmt) = object+ [ "tag" .= String "InsertDirectoryStmt"+ , "target" .= stmt+ ]+ toJSON (HiveUseStmt stmt) = object+ [ "tag" .= String "UseStmt"+ , "target" .= stmt+ ]+ toJSON (HiveAnalyzeStmt stmt) = object+ [ "tag" .= String "AnalyzeStmt"+ , "target" .= stmt+ ]+ toJSON (HiveTruncatePartitionStmt truncatePartition) = object+ [ "tag" .= String "HiveTruncatePartitionStmt"+ , "statement" .= truncatePartition+ ]+ toJSON (HiveAlterTableSetLocationStmt alterTableSetLocation) = object+ [ "tag" .= String "HiveAlterTableSetLocationStmt"+ , "statement" .= alterTableSetLocation+ ]+ toJSON (HiveAlterPartitionSetLocationStmt alterPartitionSetLocation) = object+ [ "tag" .= String "HiveAlterPartitionSetLocationStmt"+ , "statement" .= alterPartitionSetLocation+ ]+ toJSON (HiveUnhandledStatement info) = object+ [ "tag" .= String "HiveUnhandledStatement"+ , "info" .= info+ ]++typeExample :: ()+typeExample = const () $ toJSON (undefined :: HiveStatement ResolvedNames Range)++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (InsertDirectory r a) where+ toJSON stmt = object+ [ "tag" .= String "InsertDirectory"+ , "info" .= insertDirectoryInfo stmt+ , "directory" .= insertDirectoryPath stmt+ , "locale" .= insertDirectoryLocale stmt+ , "values" .= insertDirectoryQuery stmt+ ]++instance ToJSON a => ToJSON (Location a) where+ toJSON (HDFSPath _ p) =+ case TL.decodeUtf8' p of+ Left _ -> toJSON $ BL.unpack p+ Right str -> toJSON str++instance ToJSON a => ToJSON (InsertDirectoryLocale a) where+ toJSON (InsertDirectoryLocal _) = String "Local"+ toJSON InsertDirectoryHDFS = String "HDFS"++instance ToJSON a => ToJSON (Use a) where+ toJSON (UseDatabase dbn) = toJSON dbn+ toJSON (UseDefault _) = String "Default"++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (Analyze r a) where+ toJSON (Analyze info tbn) = object+ [ "tag" .= String "Analyze"+ , "info" .= info+ , "table" .= toJSON tbn+ ]++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (TruncatePartition r a) where+ toJSON (TruncatePartition info truncate') = object+ [ "tag" .= String "TruncatePartition"+ , "info" .= info+ , "truncate" .= truncate'+ ]++instance HasInfo (TruncatePartition r a) where+ type Info (TruncatePartition r a) = a+ getInfo (TruncatePartition info _) = info++instance HasInfo (Location a) where+ type Info (Location a) = a+ getInfo (HDFSPath info _) = info++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (AlterTableSetLocation r a) where+ toJSON (AlterTableSetLocation info table location) = object+ [ "tag" .= String "AlterTableSetLocation"+ , "info" .= info+ , "table" .= table+ , "location" .= location+ ]++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (AlterPartitionSetLocation r a) where+ toJSON (AlterPartitionSetLocation info table items location) = object+ [ "tag" .= String "AlterPartitionSetLocation"+ , "info" .= info+ , "table" .= table+ , "items" .= items+ , "location" .= location+ ]++instance (ConstrainSNames ToJSON r a, ToJSON a) => ToJSON (StaticPartitionSpecItem r a) where+ toJSON (StaticPartitionSpecItem info column constant) = object+ [ "tag" .= String "StaticPartitionSpecItem"+ , "info" .= info+ , "column" .= column+ , "constant" .= constant+ ]+++instance HasTables (HiveStatement ResolvedNames a) where+ goTables (HiveStandardSqlStatement s) = goTables s+ goTables (HiveUseStmt _) = return ()+ goTables (HiveAnalyzeStmt _) = return ()+ goTables (HiveInsertDirectoryStmt s) = goTables s+ goTables (HiveTruncatePartitionStmt s) = goTables s+ goTables (HiveAlterTableSetLocationStmt s) = goTables s+ goTables (HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation _ (RTableName fqtn _) _ _)) = tell $ S.singleton $ TableUse WriteMeta $ fqtnToFQTN fqtn+ goTables (HiveUnhandledStatement _) = return ()++instance HasTables (InsertDirectory ResolvedNames a) where+ goTables InsertDirectory{..} = goTables insertDirectoryQuery++instance HasTables (TruncatePartition ResolvedNames a) where+ goTables (TruncatePartition _ s) = goTables s++instance HasTables (AlterTableSetLocation ResolvedNames a) where+ goTables (AlterTableSetLocation _ table _) = do+ goTables table+++instance HasColumns (HiveStatement ResolvedNames a) where+ goColumns (HiveStandardSqlStatement s) = goColumns s+ goColumns (HiveUseStmt _) = return ()+ goColumns (HiveAnalyzeStmt _) = return ()+ goColumns (HiveInsertDirectoryStmt s) = goColumns s+ goColumns (HiveTruncatePartitionStmt _) = return ()+ goColumns (HiveAlterTableSetLocationStmt _) = return ()+ goColumns (HiveAlterPartitionSetLocationStmt (AlterPartitionSetLocation _ _ items _)) =+ forM_ items $ \ (StaticPartitionSpecItem _ column _) ->+ tell [clauseObservation (void column) "PARTITION"]+ goColumns (HiveUnhandledStatement _) = return ()++instance HasColumns (InsertDirectory ResolvedNames a) where+ goColumns InsertDirectory{..} = goColumns insertDirectoryQuery