sparkle 0.7.3 → 0.7.4
raw patch · 8 files changed
+98/−17 lines, 8 filesdep +temporarydep +unixdep ~inline-javadep ~jnisetup-changed
Dependencies added: temporary, unix
Dependency ranges changed: inline-java, jni
Files
- CHANGELOG.md +9/−0
- Setup.hs +6/−1
- Sparkle.hs +50/−7
- build/libs/sparkle.jar binary
- sparkle.cabal +10/−6
- src/Control/Distributed/Spark/Jars.hs +9/−0
- src/Control/Distributed/Spark/RDD.hs +10/−2
- src/main/java/io/tweag/sparkle/SparkleBase.java +4/−1
CHANGELOG.md view
@@ -10,6 +10,15 @@ ### Changed +## [0.7.4] - 2018-02-28++### Changed++* Fixed dynamic linking of sparkle when library dependencies don't set+ the `RPATH` to `$ORIGIN`. PR #139+* Linker flags `-rpath` and `-z origin` are no longer necessary.+* Build with `jvm-batching`.+ ## [0.7.3] - 2018-02-06 ### Added
Setup.hs view
@@ -1,4 +1,9 @@ import Distribution.Simple import Language.Java.Inline.Cabal+import qualified Language.Java.Streaming.Jars (getJars) -main = defaultMainWithHooks (gradleHooks simpleUserHooks)+main = do+ jars <- Language.Java.Streaming.Jars.getJars+ defaultMainWithHooks+ $ addJarsToClasspath jars+ $ gradleHooks simpleUserHooks
Sparkle.hs view
@@ -3,19 +3,29 @@ import Codec.Archive.Zip import Data.Text (pack, strip, unpack) import Data.List (isInfixOf)-import qualified Data.ByteString.Lazy as BS+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as LBS import Paths_sparkle import System.Environment (getArgs)+import System.Exit (ExitCode(..)) import System.FilePath ((</>), (<.>), takeBaseName, takeFileName) import System.Info (os) import System.IO (hPutStrLn, stderr)-import System.Process (readProcess)+import System.IO.Temp (withSystemTempDirectory)+import System.Posix.Files (createSymbolicLink)+import System.Process+ ( CreateProcess(..)+ , proc+ , readProcess+ , waitForProcess+ , withCreateProcess+ ) import Text.Regex.TDFA doPackage :: FilePath -> IO () doPackage cmd = do dir <- getDataDir- jarbytes <- BS.readFile (dir </> "sparkle.jar")+ jarbytes <- LBS.readFile (dir </> "build/libs/sparkle.jar") cmdpath <- unpack . strip . pack <$> readProcess "which" [cmd] "" ldd <- case os of "darwin" -> do@@ -27,16 +37,49 @@ let libs = filter (\x -> not $ any (`isInfixOf` x) ["libc.so", "libpthread.so"]) $ map (!! 1) (ldd =~ " => ([[:graph:]]+) " :: [[String]])- libentries <- mapM mkEntry libs- cmdentry <- toEntry "hsapp" 0 <$> BS.readFile cmdpath+ libentries0 <- mapM mkEntry libs+ libentries <-+ if os == "darwin" then return libentries0+ else do+ libhsapp <- makeHsTopLibrary cmdpath libs+ return $ toEntry "libhsapp.so" 0 libhsapp : libentries0+ cmdentry <- toEntry "hsapp" 0 <$> LBS.readFile cmdpath let appzip = toEntry "sparkle-app.zip" 0 $ fromArchive $ foldr addEntryToArchive emptyArchive (cmdentry : libentries) newjarbytes = fromArchive $ addEntryToArchive appzip (toArchive jarbytes)- BS.writeFile ("." </> takeBaseName cmd <.> "jar") newjarbytes+ LBS.writeFile ("." </> takeBaseName cmd <.> "jar") newjarbytes where- mkEntry file = toEntry (takeFileName file) 0 <$> BS.readFile file+ mkEntry file = toEntry (takeFileName file) 0 <$> LBS.readFile file++-- We make a library which depends on all the libraries that go into the jar.+-- This removes the need to fiddle with the rpaths of the various libraries+-- and the application executable.+makeHsTopLibrary :: FilePath -> [FilePath] -> IO LBS.ByteString+makeHsTopLibrary hsapp libs = withSystemTempDirectory "libhsapp" $ \d -> do+ let f = d </> "libhsapp.so"+ createSymbolicLink hsapp (d </> "hsapp")+ -- Changing the directory is necessary for gcc to link hsapp with a+ -- relative path. "-L d -l:hsapp" doesn't work in centos 6 where the+ -- path to hsapp in the output library ends up being absolute.+ callProcessCwd d "gcc" $+ [ "-shared", "-Wl,-z,origin", "-Wl,-rpath=$ORIGIN", "hsapp"+ , "-o", f] ++ libs+ LBS.fromStrict <$> BS.readFile f++-- This is a variant of 'callProcess' which takes a working directory.+callProcessCwd :: FilePath -> FilePath -> [String] -> IO ()+callProcessCwd wd cmd args = do+ exit_code <- withCreateProcess+ (proc cmd args)+ { delegate_ctlc = True+ , cwd = Just wd+ } $ \_ _ _ p ->+ waitForProcess p+ case exit_code of+ ExitSuccess -> return ()+ ExitFailure r -> error $ "callProcessCwd: " ++ show (cmd, args, r) main :: IO () main = do
− build/libs/sparkle.jar
binary file changed (14334 → absent bytes)
sparkle.cabal view
@@ -1,5 +1,5 @@ name: sparkle-version: 0.7.3+version: 0.7.4 synopsis: Distributed Apache Spark applications in Haskell description: See https://www.stackage.org/package/sparkle. homepage: http://github.com/tweag/sparkle#readme@@ -30,8 +30,6 @@ CHANGELOG.md README.md build.gradle-data-dir: build/libs-data-files: sparkle.jar source-repository head type: git@@ -42,7 +40,8 @@ setup-depends: base, Cabal >= 1.24,- inline-java >= 0.6.3+ inline-java >= 0.6.3,+ jvm-streaming >= 0.3 library include-dirs: cbits@@ -52,6 +51,7 @@ Control.Distributed.Spark Control.Distributed.Spark.Closure Control.Distributed.Spark.Context+ Control.Distributed.Spark.Jars Control.Distributed.Spark.ML.Feature.CountVectorizer Control.Distributed.Spark.ML.Feature.RegexTokenizer Control.Distributed.Spark.ML.Feature.StopWordsRemover@@ -69,6 +69,8 @@ Control.Distributed.Spark.SQL.SparkSession Control.Distributed.Spark.RDD Language.Scala.Tuple+ other-modules:+ Paths_sparkle build-depends: base >=4.10.1.0 && <5, binary >=0.7,@@ -76,8 +78,8 @@ choice >= 0.1, constraints, distributed-closure >=0.3,- inline-java >=0.7.0 && <0.8,- jni >=0.5.0 && <0.6,+ inline-java >=0.7.0 && <0.9,+ jni >=0.5.0 && <0.7, jvm >=0.4.0.1 && <0.5, jvm-streaming >= 0.2, singletons >= 2.0,@@ -97,7 +99,9 @@ process >= 1.2, regex-tdfa >= 1.2, sparkle,+ temporary, text >= 1.2,+ unix, zip-archive >= 0.2 default-language: Haskell2010 ghc-options: -threaded
+ src/Control/Distributed/Spark/Jars.hs view
@@ -0,0 +1,9 @@+-- | Provides the paths to the jars needed to build with sparkle.+module Control.Distributed.Spark.Jars where++import qualified Language.Java.Streaming.Jars (getJars)+import Paths_sparkle (getDataFileName)++getJars :: IO [String]+getJars = (:) <$> getDataFileName "build/libs/sparkle.jar"+ <*> Language.Java.Streaming.Jars.getJars
src/Control/Distributed/Spark/RDD.hs view
@@ -90,7 +90,11 @@ [java| $rdd.map($f) |] mapPartitions- :: (Static (Reify a), Static (Reflect b), Typeable a, Typeable b)+ :: ( Static (Reify (Stream (Of a) IO ()))+ , Static (Reflect (Stream (Of b) IO ()))+ , Typeable a+ , Typeable b+ ) => Choice "preservePartitions" -> Closure (Stream (Of a) IO () -> Stream (Of b) IO ()) -> RDD a@@ -99,7 +103,11 @@ mapPartitionsWithIndex preservePartitions (closure (static const) `cap` clos) rdd mapPartitionsWithIndex- :: (Static (Reify a), Static (Reflect b), Typeable a, Typeable b)+ :: ( Static (Reify (Stream (Of a) IO ()))+ , Static (Reflect (Stream (Of b) IO ()))+ , Typeable a+ , Typeable b+ ) => Choice "preservePartitions" -> Closure (Int32 -> Stream (Of a) IO () -> Stream (Of b) IO ()) -> RDD a
src/main/java/io/tweag/sparkle/SparkleBase.java view
@@ -25,7 +25,10 @@ StandardCopyOption.REPLACE_EXISTING); in.close(); try {- loadApplication(sparkleAppZipFile, "hsapp");+ if (System.getProperty("os.name").startsWith("Mac "))+ loadApplication(sparkleAppZipFile, "hsapp");+ else+ loadApplication(sparkleAppZipFile, "libhsapp.so"); } finally { sparkleAppZipFile.delete(); }