diff --git a/hadoop-tools.cabal b/hadoop-tools.cabal --- a/hadoop-tools.cabal +++ b/hadoop-tools.cabal @@ -1,5 +1,5 @@ name: hadoop-tools -version: 0.7.2 +version: 1.0.0 synopsis: Fast command line tools for working with Hadoop. @@ -9,9 +9,6 @@ . These tools support v9 of the Hadoop RPC protocol (CDH 5.x and above). . - Earlier versions (< 0.6) can be installed using --constraint=hadoop-rpc==0.1.1.1 - if you need v7 support. - . > hh cat - Print the contents of a file to stdout > hh cd - Change working directory > hh chmod - Change permissions @@ -28,7 +25,7 @@ homepage: http://github.com/jystic/hadoop-tools license: Apache-2.0 license-file: LICENSE -author: Jacob Stanley, Conrad Parker +author: Jacob Stanley, Conrad Parker, Luke Clifton maintainer: Jacob Stanley category: Data build-type: Simple @@ -52,14 +49,15 @@ Paths_hadoop_tools build-depends: - base >= 4.8 && < 5 + base >= 4.7 && < 5 , attoparsec >= 0.12 , boxes >= 0.1 , bytestring >= 0.10 , configurator >= 0.3 + , clock == 0.5.* , exceptions >= 0.6 , filepath >= 1.3 - , hadoop-rpc >= 1.0.0.1 + , hadoop-rpc >= 1.1 , old-locale >= 1.0 , optparse-applicative >= 0.11 , protobuf >= 0.2.0.4 @@ -67,7 +65,7 @@ , split >= 0.2 , stm >= 2.4 , text >= 1.1 - , time >= 1.4 + , time >= 1.5 , transformers >= 0.4 , unix >= 2.7 , vector >= 0.10 @@ -81,11 +79,11 @@ main-is: test.hs build-depends: - base >= 4 && < 5 + base >= 4 && < 5 , attoparsec - , bytestring >= 0.10 - , hadoop-rpc >= 0.1.1.1 - , tasty >= 0.7 + , bytestring + , hadoop-rpc + , tasty >= 0.7 , tasty-hunit , tasty-quickcheck , vector diff --git a/src/Chmod.hs b/src/Chmod.hs --- a/src/Chmod.hs +++ b/src/Chmod.hs @@ -6,7 +6,9 @@ , applyChmod ) where +import Control.Applicative import Control.Monad (guard, msum) + import qualified Data.Attoparsec.ByteString.Char8 as Atto import Data.Bits import qualified Data.ByteString.Char8 as B @@ -15,6 +17,8 @@ import Data.Maybe (mapMaybe) import Data.Word (Word16) +import Prelude + import Data.Hadoop.Types (FileType(..)) ------------------------------------------------------------------------ @@ -129,9 +133,13 @@ b Chmod_r = 4 b Chmod_w = 2 b Chmod_x = 1 + b Chmod_X = error "applyChmod: can't set a bit for X" + b Chmod_s = error "applyChmod: can't set a bit for s" + b Chmod_t = error "applyChmod: can't set a bit for t" -- Number of bits to shift for who s :: ChmodWho -> Int s Chmod_u = 6 s Chmod_g = 3 s Chmod_o = 0 + s Chmod_a = error "applyChmod: can't shift for a" diff --git a/src/Glob.hs b/src/Glob.hs --- a/src/Glob.hs +++ b/src/Glob.hs @@ -6,13 +6,17 @@ , matches ) where ------------------------------------------------------------------------- +import Control.Applicative import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as B import Data.Monoid ((<>)) + import System.IO.Unsafe (unsafeDupablePerformIO) + import qualified Text.Regex.PCRE.ByteString as Regex + +import Prelude ------------------------------------------------------------------------ diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -9,6 +9,7 @@ import Control.Monad import Control.Monad.Catch (handle, throwM) import Control.Monad.IO.Class (MonadIO, liftIO) + import qualified Data.Attoparsec.ByteString.Char8 as Atto import Data.Bits ((.&.), shiftL, shiftR) import Data.ByteString (ByteString) @@ -17,15 +18,17 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T +import qualified Data.Foldable as Foldable import Data.Time import Data.Time.Clock.POSIX import qualified Data.Vector as V import Data.Version (showVersion) import Data.Word (Word16, Word64) - import qualified Data.Configurator as C import Data.Configurator.Types (Worth(..)) + import Options.Applicative hiding (Success) + import System.Environment (getEnv) import System.Exit (exitFailure) import qualified System.FilePath as FilePath @@ -33,9 +36,10 @@ import System.IO import System.IO.Unsafe (unsafePerformIO) import System.Posix.User (GroupEntry(..), getGroups, getGroupEntryForID) + import Text.PrettyPrint.Boxes hiding ((<>), (//)) -import Data.Hadoop.Configuration (getHadoopConfig, getHadoopUser) +import Data.Hadoop.Configuration (getHadoopConfig, getHadoopUser, readPrincipal) import Data.Hadoop.HdfsPath import Data.Hadoop.Types import Network.Hadoop.Hdfs hiding (runHdfs) @@ -101,9 +105,16 @@ return (home `FilePath.combine` ".hh") {-# NOINLINE configPath #-} -getHdfsUser :: IO (Maybe User) -getHdfsUser = C.load [Optional configPath] >>= flip C.lookup "hdfs.user" +getHdfsUser :: IO (Maybe UserDetails) +getHdfsUser = do + cfg <- C.load [Optional configPath] + udUser <- C.lookup cfg "hdfs.user" + udAuthUser <- C.lookup cfg "auth.user" + return $ UserDetails <$> udUser <*> pure udAuthUser + +-- getHdfsUser = C.load [Optional configPath] >>= flip C.lookup "hdfs.user" + getGroupNames :: IO [Group] getGroupNames = do groups <- getGroups @@ -112,10 +123,13 @@ getNameNode :: IO (Maybe NameNode) getNameNode = do - cfg <- C.load [Optional configPath] - host <- C.lookup cfg "namenode.host" - port <- C.lookupDefault 8020 cfg "namenode.port" - return (Endpoint <$> host <*> pure port) + cfg <- C.load [Optional configPath] + host <- C.lookup cfg "namenode.host" + port <- C.lookupDefault 8020 cfg "namenode.port" + prinStr <- C.lookup cfg "namenode.principal" + let endpoint = Endpoint <$> host <*> pure port + principal = join $ readPrincipal <$> prinStr <*> host + return $ flip NameNode principal <$> endpoint getSocksProxy :: IO (Maybe SocksProxy) getSocksProxy = do @@ -134,7 +148,7 @@ {-# NOINLINE workingDirConfigPath #-} getHomeDir :: MonadIO m => m HdfsPath -getHomeDir = liftIO $ (("/user" ) . T.encodeUtf8 . hcUser) <$> getConfig +getHomeDir = liftIO $ (("/user" ) . T.encodeUtf8 . (udUser . hcUser)) <$> getConfig getWorkingDir :: MonadIO m => m HdfsPath getWorkingDir = liftIO $ handle onError @@ -189,7 +203,7 @@ sub SubCommand{..} = command subName (info subMethod $ progDesc subDescription) options :: Parser SubMethod -options = subparser (foldMap sub allSubCommands) +options = subparser (Foldable.foldMap sub allSubCommands) allSubCommands :: [SubCommand] allSubCommands =