diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ ### Changed +## [0.7.4] - 2018-02-28 + +### Changed + +* Fixed dynamic linking of sparkle when library dependencies don't set + the `RPATH` to `$ORIGIN`. PR #139 +* Linker flags `-rpath` and `-z origin` are no longer necessary. +* Build with `jvm-batching`. + ## [0.7.3] - 2018-02-06 ### Added diff --git a/Setup.hs b/Setup.hs --- a/Setup.hs +++ b/Setup.hs @@ -1,4 +1,9 @@ import Distribution.Simple import Language.Java.Inline.Cabal +import qualified Language.Java.Streaming.Jars (getJars) -main = defaultMainWithHooks (gradleHooks simpleUserHooks) +main = do + jars <- Language.Java.Streaming.Jars.getJars + defaultMainWithHooks + $ addJarsToClasspath jars + $ gradleHooks simpleUserHooks diff --git a/Sparkle.hs b/Sparkle.hs --- a/Sparkle.hs +++ b/Sparkle.hs @@ -3,19 +3,29 @@ import Codec.Archive.Zip import Data.Text (pack, strip, unpack) import Data.List (isInfixOf) -import qualified Data.ByteString.Lazy as BS +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as LBS import Paths_sparkle import System.Environment (getArgs) +import System.Exit (ExitCode(..)) import System.FilePath ((), (<.>), takeBaseName, takeFileName) import System.Info (os) import System.IO (hPutStrLn, stderr) -import System.Process (readProcess) +import System.IO.Temp (withSystemTempDirectory) +import System.Posix.Files (createSymbolicLink) +import System.Process + ( CreateProcess(..) + , proc + , readProcess + , waitForProcess + , withCreateProcess + ) import Text.Regex.TDFA doPackage :: FilePath -> IO () doPackage cmd = do dir <- getDataDir - jarbytes <- BS.readFile (dir "sparkle.jar") + jarbytes <- LBS.readFile (dir "build/libs/sparkle.jar") cmdpath <- unpack . strip . pack <$> readProcess "which" [cmd] "" ldd <- case os of "darwin" -> do @@ -27,16 +37,49 @@ let libs = filter (\x -> not $ any (`isInfixOf` x) ["libc.so", "libpthread.so"]) $ map (!! 1) (ldd =~ " => ([[:graph:]]+) " :: [[String]]) - libentries <- mapM mkEntry libs - cmdentry <- toEntry "hsapp" 0 <$> BS.readFile cmdpath + libentries0 <- mapM mkEntry libs + libentries <- + if os == "darwin" then return libentries0 + else do + libhsapp <- makeHsTopLibrary cmdpath libs + return $ toEntry "libhsapp.so" 0 libhsapp : libentries0 + cmdentry <- toEntry "hsapp" 0 <$> LBS.readFile cmdpath let appzip = toEntry "sparkle-app.zip" 0 $ fromArchive $ foldr addEntryToArchive emptyArchive (cmdentry : libentries) newjarbytes = fromArchive $ addEntryToArchive appzip (toArchive jarbytes) - BS.writeFile ("." takeBaseName cmd <.> "jar") newjarbytes + LBS.writeFile ("." takeBaseName cmd <.> "jar") newjarbytes where - mkEntry file = toEntry (takeFileName file) 0 <$> BS.readFile file + mkEntry file = toEntry (takeFileName file) 0 <$> LBS.readFile file + +-- We make a library which depends on all the libraries that go into the jar. +-- This removes the need to fiddle with the rpaths of the various libraries +-- and the application executable. +makeHsTopLibrary :: FilePath -> [FilePath] -> IO LBS.ByteString +makeHsTopLibrary hsapp libs = withSystemTempDirectory "libhsapp" $ \d -> do + let f = d "libhsapp.so" + createSymbolicLink hsapp (d "hsapp") + -- Changing the directory is necessary for gcc to link hsapp with a + -- relative path. "-L d -l:hsapp" doesn't work in centos 6 where the + -- path to hsapp in the output library ends up being absolute. + callProcessCwd d "gcc" $ + [ "-shared", "-Wl,-z,origin", "-Wl,-rpath=$ORIGIN", "hsapp" + , "-o", f] ++ libs + LBS.fromStrict <$> BS.readFile f + +-- This is a variant of 'callProcess' which takes a working directory. +callProcessCwd :: FilePath -> FilePath -> [String] -> IO () +callProcessCwd wd cmd args = do + exit_code <- withCreateProcess + (proc cmd args) + { delegate_ctlc = True + , cwd = Just wd + } $ \_ _ _ p -> + waitForProcess p + case exit_code of + ExitSuccess -> return () + ExitFailure r -> error $ "callProcessCwd: " ++ show (cmd, args, r) main :: IO () main = do diff --git a/build/libs/sparkle.jar b/build/libs/sparkle.jar deleted file mode 100644 Binary files a/build/libs/sparkle.jar and /dev/null differ diff --git a/sparkle.cabal b/sparkle.cabal --- a/sparkle.cabal +++ b/sparkle.cabal @@ -1,5 +1,5 @@ name: sparkle -version: 0.7.3 +version: 0.7.4 synopsis: Distributed Apache Spark applications in Haskell description: See https://www.stackage.org/package/sparkle. homepage: http://github.com/tweag/sparkle#readme @@ -30,8 +30,6 @@ CHANGELOG.md README.md build.gradle -data-dir: build/libs -data-files: sparkle.jar source-repository head type: git @@ -42,7 +40,8 @@ setup-depends: base, Cabal >= 1.24, - inline-java >= 0.6.3 + inline-java >= 0.6.3, + jvm-streaming >= 0.3 library include-dirs: cbits @@ -52,6 +51,7 @@ Control.Distributed.Spark Control.Distributed.Spark.Closure Control.Distributed.Spark.Context + Control.Distributed.Spark.Jars Control.Distributed.Spark.ML.Feature.CountVectorizer Control.Distributed.Spark.ML.Feature.RegexTokenizer Control.Distributed.Spark.ML.Feature.StopWordsRemover @@ -69,6 +69,8 @@ Control.Distributed.Spark.SQL.SparkSession Control.Distributed.Spark.RDD Language.Scala.Tuple + other-modules: + Paths_sparkle build-depends: base >=4.10.1.0 && <5, binary >=0.7, @@ -76,8 +78,8 @@ choice >= 0.1, constraints, distributed-closure >=0.3, - inline-java >=0.7.0 && <0.8, - jni >=0.5.0 && <0.6, + inline-java >=0.7.0 && <0.9, + jni >=0.5.0 && <0.7, jvm >=0.4.0.1 && <0.5, jvm-streaming >= 0.2, singletons >= 2.0, @@ -97,7 +99,9 @@ process >= 1.2, regex-tdfa >= 1.2, sparkle, + temporary, text >= 1.2, + unix, zip-archive >= 0.2 default-language: Haskell2010 ghc-options: -threaded diff --git a/src/Control/Distributed/Spark/Jars.hs b/src/Control/Distributed/Spark/Jars.hs new file mode 100644 --- /dev/null +++ b/src/Control/Distributed/Spark/Jars.hs @@ -0,0 +1,9 @@ +-- | Provides the paths to the jars needed to build with sparkle. +module Control.Distributed.Spark.Jars where + +import qualified Language.Java.Streaming.Jars (getJars) +import Paths_sparkle (getDataFileName) + +getJars :: IO [String] +getJars = (:) <$> getDataFileName "build/libs/sparkle.jar" + <*> Language.Java.Streaming.Jars.getJars diff --git a/src/Control/Distributed/Spark/RDD.hs b/src/Control/Distributed/Spark/RDD.hs --- a/src/Control/Distributed/Spark/RDD.hs +++ b/src/Control/Distributed/Spark/RDD.hs @@ -90,7 +90,11 @@ [java| $rdd.map($f) |] mapPartitions - :: (Static (Reify a), Static (Reflect b), Typeable a, Typeable b) + :: ( Static (Reify (Stream (Of a) IO ())) + , Static (Reflect (Stream (Of b) IO ())) + , Typeable a + , Typeable b + ) => Choice "preservePartitions" -> Closure (Stream (Of a) IO () -> Stream (Of b) IO ()) -> RDD a @@ -99,7 +103,11 @@ mapPartitionsWithIndex preservePartitions (closure (static const) `cap` clos) rdd mapPartitionsWithIndex - :: (Static (Reify a), Static (Reflect b), Typeable a, Typeable b) + :: ( Static (Reify (Stream (Of a) IO ())) + , Static (Reflect (Stream (Of b) IO ())) + , Typeable a + , Typeable b + ) => Choice "preservePartitions" -> Closure (Int32 -> Stream (Of a) IO () -> Stream (Of b) IO ()) -> RDD a diff --git a/src/main/java/io/tweag/sparkle/SparkleBase.java b/src/main/java/io/tweag/sparkle/SparkleBase.java --- a/src/main/java/io/tweag/sparkle/SparkleBase.java +++ b/src/main/java/io/tweag/sparkle/SparkleBase.java @@ -25,7 +25,10 @@ StandardCopyOption.REPLACE_EXISTING); in.close(); try { - loadApplication(sparkleAppZipFile, "hsapp"); + if (System.getProperty("os.name").startsWith("Mac ")) + loadApplication(sparkleAppZipFile, "hsapp"); + else + loadApplication(sparkleAppZipFile, "libhsapp.so"); } finally { sparkleAppZipFile.delete(); }