hoogle 4.2.26 → 4.2.27

raw patch · 22 files changed

+469/−363 lines, 22 filesdep +shake

Dependencies added: shake

Files

• CHANGES.txt +173/−0
• hoogle.cabal +8/−6
• src/CmdLine/All.hs +1/−0
• src/CmdLine/Load.hs +1/−0
• src/CmdLine/Type.hs +1/−1
• src/Console/All.hs +14/−1
• src/Console/Test.hs +1/−0
• src/General/Base.hs +0/−1
• src/General/System.hs +4/−1
• src/General/Web.hs +5/−0
• src/Hoogle/Language/Haskell.hs +1/−0
• src/Recipe/All.hs +146/−60
• src/Recipe/Cabal.hs +2/−2
• src/Recipe/Command.hs +62/−0
• src/Recipe/Download.hs +0/−80
• src/Recipe/General.hs +0/−44
• src/Recipe/Hackage.hs +10/−80
• src/Recipe/Keyword.hs +3/−9
• src/Recipe/Type.hs +0/−76
• src/Recipe/Warning.hs +25/−0
• src/Test.hs +7/−2
• src/Web/Server.hs +5/−0

@@ -0,0 +1,173 @@+Changelog for Hoogle++4.2.27+    Rewrite how "hoogle data" works+    #45, if you are building all, also depend on default+    #47, switch to using Shake to build the recipes+    Add a dependency on Shake+4.2.26+    Fix omission of the Test file+4.2.25+    Fix timestamps in .tar.gz dist file+4.2.24+    Allow WAI 2.0+    Fix the Keyword generator+4.2.23+    Improvements to Haddock conversion+4.2.22+    Download Hoogle data from the old Hackage server+4.2.21+    #25, make the tarball pass the tests+    #26, GHC 7.8 compatibility+    Ensure alias resolution is bottom-up and recursive+4.2.20+    #619, do not depend on unix on Windows+4.2.19+    Fix version bounds+4.2.18+    Allow bytestring-0.9+4.2.17+    Upgrade to haskell-src-exts-1.14+    Visual redesign+    Require bytestring-0.10 or above, due to NFData instances+    Add NFData instance to Database+4.2.16+    Remove lots of explicit upper bounds+4.2.15+    Allow http-types-0.8+    Allow case-insensitive-1.0+    Support --local for GHC docs+    #568, rewrite Haddock links in --local+4.2.14+    Switch license from GPL to BSD3+    GHC 7.6 fixes+    ##4, Allow Cabal-1.16.*+4.2.13+    Allow cmdargs-0.10.*+4.2.12+    Allow conduit-0.5, wai-1.3 and warp-1.3+    Upgrade to http-types-0.7.*+    In embedded mode, don't hijack the URL bar or title+    Upgrade to jQuery 1.7.2+    #89, add a format=json output mode+    #476, update the URL as you type+    Improve iPhone support+    Update the copyright year in the footer+    Double the speed of hoogle server responses+4.2.11+    Allow transformers-0.3+    Allow conduit-0.4, wai-1.2 and warp-1.2+    Allow haskell-src-exts-1.12 and 1.13+4.2.10+    #533, be paranoid when parsing Cabal files+4.2.9+    Upgrade to WAI-1.1 and Warp-1.1+    #525, don't rely on a redirect to find files+    Allow Cabal-1.14+4.2.8+    Allow cmdargs-0.9+    Fix setting the global read flags and unsetting them properly+    Allow case-insensitive-0.4.*+    Make data --local have a good default on Linux+    Make server --local work on Windows+    #407, rewrite OpenSearch plugin to work regardless of the server+4.2.7+    Allow cmdargs-0.8+4.2.6+    Permit GHC 7.2+    Allow case-insensitive-0.3.*+    Fix bug when searching for toString in the embed mode+    Make --redownload work on the tarballs+4.2.5+    Fix hyperlinks to symbols (i.e. &&) in Haddock+4.2.4+    #434, use either wget or curl to download the files+    Allow haskell-src-exts 1.11.*+    Upgrade to cmdargs==0.7.*+4.2.3+    Generate database for ghc internals+    Give an error if the database is the wrong version+    Add -fno-cse, to make cmdargs work (was losing args annotations)+4.2.2+    Upgrade to WAI==0.4.* and Warp==0.4.*+    Allow parsec 3, since various people have reported it works+    Default module URL's correctly when not on Hackage+    #417, use proper Haddock parsing for package descriptions+    Make sure https: links work through the web mode+    Rewrite the binary defer layer, cleaner and slightly faster+    Eliminate escaping problems when entering "'s in the search+    Some work on #375, add links to the library page+    Add --template to override the templates at runtime+    Add --dynamic flag to server mode, include update stamps+    Upgrade wai and warp, adds an exception handler+    #82, rewrite showing document snippets, better expand/collapse+    Support <em> tags in Haddock output (as well as <i>)+    Make package matching case insensitive+    #372, add features to improve module name search+    Make module name searching case insensitive+4.2.1+    Fix for pattern match failure when searching for "to"+    Fix for not sending text/html when serving local files+    Eliminate the --nostdin flag, now unnecessary+    Change to use WAI/Warp instead of HTTP and custom server+    Fix a bug, didn't ever show Waiting... on the real website+    Make embed work in IE8+4.2+    Make the CGI interface send the right number of newlines+    Accept prefix and suffix web parameters+    Eliminate isBlankQuery, make Query a Monoid+    Make Query abstract, add queryPackages/querySetPackage+    Rename querySuggestions/queryCompletions by dropping the query+    Add mode=embed support+    Add some log analysis features+    Ignore some whitespace in the input file+    Work around a cabal bug, include jquery.cookie.js+    Add ?version=xxx to .css and .js, to make it auto refresh+4.1.5+    #399, rehabilitate mode=suggest+    Stop mode=suggest failing entirely, now cleanly gives no answers+    Combine --web and --webmode+    Complete the command line flag help+    #327, ensure utf8 is found+    #187, ensure LT is found+    Use the Hoogle tarball direct from Hackage+    #54, rework packages, entries and keywords+    #87, make all database files lowercase+    Various javascript/ajax enhancements+4.1.4+    Running data on Posix sets all files to global read/execute+    Relax haskell-src-exts to allow 1.10.1+    Rework the web page display further+4.1.3+    Read and write all database files in UTF8+    #391, add tar -xzf when extracting from the tarball+    Change search results display, packages/modules on separate line+    Exclude haskellN* from the platform, they are in base+    Add --nostdin to server mode, to support running with nohup+    Loosen dependencies for tagsoup (typo'd it before)+4.1.2+    Loosen dependencies for tagsoup and HTTP (but not parsec)+    Avoid self circular dependencies+4.1.1+    #195, include all the necessary resources to run as a server+    When in server mode, be more robust to errors+4.1+    #320, make sure system is found+    #146, preserve forall's in everywhere+    #186, switch to using haskell-src-exts to parse input files+    #249, make sure you find MonadWriter+    #235, make sure you find forall+    #309, make sure there are links for keywords+    #78, add --link flag+    #59, operators look nicer, and have blue brackets round them+    #352, can now parse ( # ) as a function name in the input+    #280, searching for "~ +keyword" should find keyword+    #116, fix links for types/classes+    #83, all searching for package-name+    #94, allow textbases to be used automatically+    #92, short flags are now handled by cmdargs, so work+    #51, test mode now does what test and testfile did+    #66, turn Hoogle into a library+    Add --server flag, to run as a server+4.0.7+    Start of changelog

@@ -1,7 +1,7 @@ cabal-version:      >= 1.10 build-type:         Simple name:               hoogle-version:            4.2.26+version:            4.2.27 license:            BSD3 license-file:       docs/LICENSE category:           Development@@ -16,8 +16,10 @@ homepage:           http://www.haskell.org/hoogle/ bug-reports:        https://github.com/ndmitchell/hoogle/issues tested-with:        GHC==7.6.3, GHC==7.4.2-extra-source-files: README.txt-                    datadir/*.txt+extra-source-files:+    README.txt+    CHANGES.txt+    datadir/*.txt  data-dir: datadir data-files:@@ -137,6 +139,7 @@         wai >= 1.1,         warp >= 1.1,         Cabal >= 1.8,+        shake >= 0.10.7,         haskell-src-exts >= 1.14 && < 1.15      if !os(mingw32)@@ -154,12 +157,11 @@         Paths_hoogle         Recipe.All         Recipe.Cabal-        Recipe.Download-        Recipe.General+        Recipe.Command         Recipe.Hackage         Recipe.Haddock         Recipe.Keyword-        Recipe.Type+        Recipe.Warning         Test.All         Test.Docs         Test.General

@@ -18,6 +18,7 @@  import General.Base import General.System+import System.FilePath import CmdLine.Type import CmdLine.Load import General.Web

@@ -5,6 +5,7 @@ import General.Base import General.Util import General.System+import System.FilePath   -- | Given a list of search directories, and a query, load the databases you

@@ -102,7 +102,7 @@ data_ = Data     {datadir = def &= typDir &= help "Database directory"     ,redownload = def &= help "Redownload all files from the web"-    ,threads = def &= typ "INT" &= name "j" &= help "Number of threads to use" &= ignore -- ignore until it works+    ,threads = 1 &= typ "INT" &= name "j" &= help "Number of threads to use"     ,actions = def &= args &= typ "RULE"     ,local = def &= opt "" &= typ "FILEPATH" &= help "Use local documentation if available"     } &= help "Generate Hoogle databases"

@@ -3,7 +3,6 @@  import CmdLine.All import Recipe.All-import Recipe.General import Recipe.Haddock import Console.Log import Console.Search@@ -12,6 +11,7 @@ import General.Base import General.System import General.Web+import System.FilePath import Hoogle  @@ -75,3 +75,16 @@  action q@Search{} = actionSearch q (fromRight $ queryParsed q) ++--- convert a single database+convert :: [FilePath] -> String -> FilePath -> String -> IO ()+convert deps x out src = do+    deps2 <- filterM doesFileExist deps+    when (deps /= deps2) $ putStrLn $ "Warning: " ++ x ++ " doesn't know about dependencies on " ++ unwords (deps \\ deps2)+    dbs <- mapM loadDatabase deps2+    let (err,db) = createDatabase Haskell dbs src+    unless (null err) $ putStrLn $ "Skipped " ++ show (length err) ++ " warnings in " ++ x+    putStr $ "Converting " ++ x ++ "... "+    performGC+    saveDatabase out db+    putStrLn "done"

@@ -7,6 +7,7 @@ import Hoogle import General.Base import General.System+import System.FilePath import Paths_hoogle import CmdLine.All import Test.All

@@ -20,7 +20,6 @@ import Data.Word as X import Debug.Trace as X (trace) import Numeric as X (readHex,showHex)-import System.FilePath as X hiding (combine) import qualified Data.ByteString.Char8 as BS import qualified Data.ByteString.Lazy.Char8 as LBS 

@@ -33,6 +33,9 @@ #endif  +removeFile_ x = removeFile x `E.catch` \(_ :: E.SomeException) -> return ()++ withDirectory dir cmd = E.bracket     (do x <- getCurrentDirectory; setCurrentDirectory dir; return x)     setCurrentDirectory@@ -53,7 +56,7 @@ #else captureOutput act = do     tmp <- getTemporaryDirectory-    (f,h) <- openTempFile tmp "hlint"+    (f,h) <- openTempFile tmp "hoogle"     sto <- hDuplicate stdout     ste <- hDuplicate stderr     hDuplicateTo h stdout

@@ -13,8 +13,13 @@     Args, cgiArgs, cgiResponse, parseHttpQueryArgs     ) where +#ifndef MIN_VERSION_wai+#define MIN_VERSION_wai(a,b,c) 0+#endif+ import General.System import General.Base+import System.FilePath import Network.Wai #if MIN_VERSION_wai(2, 0, 0) import Network.Wai.Internal

@@ -4,6 +4,7 @@  import General.Base import General.Util+import System.FilePath import Hoogle.Type.All import Language.Haskell.Exts.Annotated hiding (TypeSig,Type) import qualified Language.Haskell.Exts.Annotated as HSE

@@ -1,85 +1,171 @@--- Recipe actions:--- Download to foo.src in most cases, then extract to foo.txt, which can later be compiled to foo.hoo+{-# LANGUAGE RecordWildCards, ScopedTypeVariables #-}+ module Recipe.All(recipes) where -import General.Base-import General.System-import Control.Concurrent-import qualified Data.Map as Map+import General.Base hiding (readFile')+import General.System as Sys+import Control.Exception as E+import qualified Data.Set as Set+import Development.Shake+import Development.Shake.FilePath+import Recipe.Haddock -import Recipe.Type-import Recipe.Download+import Recipe.Warning+import Recipe.Command import Recipe.Keyword-import Recipe.General import Recipe.Hackage+import Recipe.Cabal+import Hoogle+import qualified Paths_hoogle as V+import Data.Version+import CmdLine.All   -- CmdLine is guaranteed to be a constructor of type Data recipes :: CmdLine -> IO ()-recipes opt = withModeGlobalRead $ do+recipes opt@Data{..} = withModeGlobalRead $ do     hSetBuffering stdout NoBuffering-    createDirectoryIfMissing True $ datadir opt-    withDirectory (datadir opt) $ do+    createDirectoryIfMissing True datadir+    withDirectory datadir $ do         resetWarnings-        download opt-        let ys = parseRules $ actions opt-        make opt (filter (not . null . snd) ys) (map fst ys)+        when redownload $ do+            forM_ urls $ \(file,_) -> removeFile_ $ "downloads" </> file+        shake shakeOptions{shakeVersion=showVersion V.version, shakeThreads=threads, shakeProgress=progressSimple} $ do+            want $ map (<.> "hoo") $ if null actions then ["default"] else actions+            rules opt         recapWarnings         putStrLn "Data generation complete"  --- If I switch to the parallel-io library then it segfaults, due to GHC bug:--- http://hackage.haskell.org/trac/ghc/ticket/4850--- import "parallel-io" Control.Concurrent.ParallelIO.Local-withPool i f = f ()-extraWorkerWhileBlocked _ = id-parallel_ _ = sequence_+rules :: CmdLine -> Rules ()+rules Data{..} = do+    "downloads/packages.txt" *> \out -> do+        need ["downloads/hoogle.untar","downloads/cabal.untar"]+        as <- liftIO $ listing "downloads/hoogle"+        bs <- liftIO $ listing "downloads/cabal"+        writeFileLines out $ Set.toList $ Set.fromList as `Set.intersection` Set.fromList bs +    packages <- do+        cache <- newCache $ fmap (Set.fromList . lines) . readFile+        return $ cache "downloads/packages.txt" -data Status = Built | Building (MVar ())+    phony "all.hoo" $ do+        pkgs <- packages+        need $ map (<.> "hoo") $ "default" : Set.toList pkgs -make :: CmdLine -> [(Name,[Name])] -> [Name] -> IO ()-make opt rules xs = withPool (threads opt) $ \pool -> do-    ref <- newMVar Map.empty-    fs ref pool [] xs-    where-        fs ref pool rec xs = parallel_ pool $ map (f ref pool rec) xs+    "keyword.txt" *> \out -> do+        let src = "downloads/keyword.htm"+        need [src]+        contents <- liftIO $ readFileUtf8' src+        liftIO $ writeFileUtf8 out $ translateKeywords contents -        f ref pool rec x-            | x `elem` rec = outStrLn $ "Warning: Package database appears to be recursive, " ++ x-            | otherwise = join $ modifyMVar ref $ \mp -> case Map.lookup x mp of-                Just Built -> return (mp, return ())-                Just (Building v) -> return $ (,) mp $-                    extraWorkerWhileBlocked pool $ readMVar v-                Nothing -> do-                    v <- newEmptyMVar-                    return $ (,) (Map.insert x (Building v) mp) $ do-                        build (fs ref pool $ x:rec) opt rules x-                        modifyMVar_ ref $ \mp -> return $ Map.insert x Built mp-                        putMVar v ()+    "default.txt" *> \out -> do+        writeFileLines out ["@combine keyword","@combine package","@combine platform"] +    "platform.txt" *> \out -> do+        let src = "downloads/platform.cabal"+        contents <- readFile' src+        writeFileLines out ["@combine " ++ x | x <- platformPackages contents] -build :: ([Name] -> IO ()) -> CmdLine -> [(Name,[Name])] -> Name -> IO ()-build makeRec opt rules x = do-    outStrLn $ "Starting " ++ x-    case lookup x rules of-        Just ys -> combine makeRec x ys True-        _ -> case x of-            "keyword" -> makeKeyword-            "default" -> combine makeRec x ["keyword","package","platform"] False-            "platform" -> makePlatform makeRec-            "package" -> makePackage-            "all" -> makeAll makeRec-            _ -> makeDefault makeRec (local opt) x-    outStrLn $ "Finished " ++ x+    "package.txt" *> \out -> do+        need ["downloads/cabal.untar"]+        xs <- liftIO $ listing "downloads/cabal"+        xs <- liftIO $ forM xs $ \name -> do+            ver <- version "downloads/cabal" name+            let file = "downloads/cabal" </> name </> ver </> name <.> "cabal"+            src <- readCabal file+            return $ case src of+                Nothing -> []+                Just src ->+                    [""] ++ zipWith (++) ("-- | " : repeat "--   ") (cabalDescription src) +++                    ["--","-- Version " ++ ver, "@url package/" ++ name, "@entry package " ++ name]+        liftIO $ writeFileUtf8 out $ unlines $ "@url http://hackage.haskell.org/" : "@package package" : concat xs +    (\x -> "*.txt" ?== x && takeBaseName x `notElem` ["keyword","default","platform","package"]) ?> \out -> do+        need ["downloads/hoogle.untar","downloads/cabal.untar"]+        let name = takeBaseName out+            base = name == "base"+        vc <- liftIO $ version "downloads/cabal" name+        vh <- liftIO $ if base then return vc else version "downloads/hoogle" name+        let hoo = if base then "downloads/base.txt" else "downloads/hoogle" </> name </> vh </> "doc" </> "html" </> name <.> "txt"+            cab = "downloads/cabal" </> name </> vc </> name <.> "cabal"+        need [hoo, cab]+        hoo <- liftIO $ readFileUtf8' hoo `E.catch` \(_ :: SomeException) -> readFile hoo+        deps <- liftIO $ fmap (maybe [] cabalDepends) $ readCabal cab+        let cleanDeps = deps \\ (name:avoid)+        loc <- liftIO $ findLocal local name+        liftIO $ writeFileUtf8 out $ unlines $+            ["@depends " ++ a | a <- cleanDeps] ++ haddockHacks loc (lines hoo) -parseRules :: [String] -> [(Name,[Name])]-parseRules [] = [("default",[])]-parseRules xs = map parseRule xs+    imported <- newCache $ \file -> do+        xs <- readFileUtf8' file+        return [x | x <- lines xs, takeWhile (not . isSpace) x `elem` ["type","data","newtype","class","instance","@depends"]]+    let listDeps = map (drop 9) . takeWhile ("@depends " `isPrefixOf`) +    let genImported pkgs seen [] = return []+        genImported pkgs seen (t:odo)+            | t `Set.member` seen || not (t `Set.member` pkgs) = genImported pkgs seen odo+            | otherwise = do+                i <- imported $ t <.> "txt"+                let deps = listDeps i+                fmap (i++) $ genImported pkgs (Set.insert t seen) (deps++odo) -parseRule :: String -> (Name,[Name])-parseRule x = (a, uncommas $ drop 1 b)-    where (a,b) = break (== '=') x-          uncommas = words . map (\x -> if x == ',' then ' ' else x)+    (\x -> "*.hoo" ?== x && x /= "all.hoo") ?> \out -> do+        let src = out -<.> "txt"+        need [src]+        contents <- liftIO $ fmap lines $ readFileUtf8' src+        if not (null contents) && "@combine " `isPrefixOf` head contents then do+            let deps = [x <.> "hoo" | x <- contents, Just x <- [stripPrefix "@combine " x]]+            need deps+            dbs <- liftIO $ mapM loadDatabase deps+            putNormal $ "Creating " ++ out ++ " from " ++ show (length deps) ++ " databases... "+            liftIO $ performGC+            liftIO $ saveDatabase out $ mconcat dbs+         else do+            pkgs <- packages+            deps <- genImported pkgs (Set.singleton $ takeBaseName out) $ listDeps contents+            let (err,db) = createDatabase Haskell [snd $ createDatabase Haskell [] $ unlines deps] $ unlines contents+            unless (null err) $ putNormal $ "Skipped " ++ show (length err) ++ " warnings in " ++ out+            putLoud $ unlines $ map show err+            putNormal $ "Creating " ++ out ++ "... "+            liftIO $ performGC+            liftIO $ saveDatabase out db++    "//*.tar" *> \out -> do+        let src = out <.> "gz"+        need [src]+        ungzip src out++    "//*.untar" *> \out -> do+        let src = out -<.> "tar"+        need [src]+        untar src+        writeFile' out ""++    (\x -> "downloads/*" ?== x && isJust (lookup (takeFileName x) urls)) ?> \out -> do+        let Just url = lookup (takeFileName out) urls+        -- liftIO $ copyFile ("C:/spacework/hoogle/cache" </> takeFileName out) out+        wget url out+++urls :: [(FilePath, URL)]+urls = let (*) = (,) in+    ["keyword.htm" * "http://www.haskell.org/haskellwiki/Keywords"+    ,"platform.cabal" * "http://code.galois.com/darcs/haskell-platform/haskell-platform.cabal"+    ,"base.txt" * "http://www.haskell.org/hoogle/base.txt"+    ,"ghc.txt" * "http://www.haskell.org/ghc/docs/latest/html/libraries/ghc/ghc.txt"+    ,"cabal.tar.gz" * "http://hackage.haskell.org/packages/index.tar.gz"+    ,"hoogle.tar.gz" * "http://hackage.haskell.org/packages/hoogle.tar.gz"]+++listing :: FilePath -> IO [String]+listing dir = do+    xs <- Sys.getDirectoryContents dir+    return $ sortBy (comparing $ map toLower) $ filter (`notElem` [".","..","preferred-versions"]) xs++version :: FilePath -> String -> IO String+version dir x = do+    ys <- Sys.getDirectoryContents $ dir </> x+    when (null ys) $ error $ "Couldn't find version for " ++ x ++ " in " ++ dir+    let f = map (read :: String -> Int) . words . map (\x -> if x == '.' then ' ' else x)+    return $ maximumBy (comparing f) $ filter (all (not . isAlpha)) ys

@@ -13,11 +13,11 @@ import Distribution.Verbosity import Distribution.Version import Recipe.Haddock-import Recipe.Type+import Recipe.Warning import Control.Exception  -ghcVersion = [7,4,1]+ghcVersion = [7,6,3]  data Cabal = Cabal     {cabalName :: String

@@ -0,0 +1,62 @@++module Recipe.Command(wget, ungzip, untar) where++import General.Base+import General.System+import System.FilePath+import Development.Shake(Action, command, CmdOption(..), liftIO)+++wget :: URL -> FilePath -> Action ()+wget from to = do+    dl <- liftIO findDownloader+    command [Shell] (dl to from) []+++ungzip :: FilePath -> FilePath -> Action ()+ungzip from to = do+    hasGzip <- liftIO $ check "gzip"+    when (isNothing hasGzip) $ error "Could not extract tarballs, could not find tar on the $PATH."+    command [Shell] ("gzip --decompress --stdout --force " ++ from ++ " > " ++ to) []++untar :: FilePath -> Action ()+untar from = do+    hasTar  <- liftIO $ check "tar"+    when (isNothing hasTar) $ error "Could not extract tarballs, could not find tar on the $PATH."+    liftIO $ createDirectoryIfMissing True $ dropExtension from+    command [Shell, Cwd $ dropExtension from] ("tar -xf ../" ++ takeFileName from) []+++++type Downloader = FilePath -> URL -> String++wget2 :: Downloader+wget2 fp url = "wget -nv " ++ url ++ " --output-document=" ++ fp+curl :: Downloader+curl fp url = "curl -sSL " ++ url ++ " --output " ++ fp++findDownloader :: IO Downloader+findDownloader = do+    dl <- check "wget"+    dl <- maybe (check "curl") (return . Just) dl+    when (isNothing dl) $ error "Could not find downloader, neither curl nor wget are on the $PATH."+    return $ matchDl (fromJust dl)+    where matchDl d | "wget" `isInfixOf` d = wget2+                    | "curl" `isInfixOf` d = curl+++check :: String -> IO (Maybe FilePath)+check name = do+    res <- findExecutable name+    when (isNothing res) $ do+        putStrLn $ "WARNING: Could not find command line program " ++ name ++ "."+        when isWindows $ putStrLn $ "  You may be able to install it from:\n  " ++ url+    return res+    where+        srcList = [ ("gzip", "http://gnuwin32.sourceforge.net/packages/gzip.htm")+                  , ("tar",  "http://gnuwin32.sourceforge.net/packages/gtar.htm")+                  , ("wget", "http://gnuwin32.sourceforge.net/packages/wget.htm")+                  , ("curl", "http://curl.haxx.se/download.html")+                  ]+        url = fromJust . lookup name $ srcList

@@ -1,80 +0,0 @@--module Recipe.Download(download) where--import General.Base-import General.System-import Recipe.Type--type Downloader = FilePath -> URL -> String--wget :: Downloader-wget fp url = "wget -nv " ++ url ++ " --output-document=" ++ fp-curl :: Downloader-curl fp url = "curl -sSL " ++ url ++ " --output " ++ fp--findDownloader :: IO Downloader-findDownloader = do-    dl <- liftM2 mplus (check "wget") (check "curl")-    when (isNothing dl) $ error "Could not find downloader, neither curl nor wget are on the $PATH."-    return $ matchDl (fromJust dl)-    where matchDl d | "wget" `isInfixOf` d = wget-                    | "curl" `isInfixOf` d = curl--withDownloader :: CmdLine -> Downloader -> [(FilePath, FilePath, URL)] -> IO ()-withDownloader opt downloader items =-    let sys = fmap (== ExitSuccess) . system-        download (f, f', u) = do-            b <- doesFileExist f-            when (not b || redownload opt) $ do-                res <- sys (downloader f' u)-                unless res $ do-                    b <- doesFileExist f'-                    when b $ removeFile f'-                    error $ "Failed to download: " ++ u-            doesFileExist f'-    in forM_ items download---- download everything required for the recipes-download :: CmdLine -> IO ()-download opt = do-    createDirectoryIfMissing True "download"-    downloader <- findDownloader-    let items = [ (keywords, keywords, "http://www.haskell.org/haskellwiki/Keywords")-                , (platform, platform, "http://code.galois.com/darcs/haskell-platform/haskell-platform.cabal")-                , ("download/base.txt", "download/base.txt", "http://www.haskell.org/hoogle/base.txt")-                , ("download/ghc.txt",  "download/ghc.txt", "http://www.haskell.org/ghc/docs/latest/html/libraries/ghc/ghc.txt")-                , (cabals <.> "txt", cabals <.> "tar.gz", "http://hackage.haskell.org/packages/archive/00-index.tar.gz")-                , (inputs <.> "txt", inputs <.> "tar.gz", "http://old.hackage.haskell.org/packages/archive/00-hoogle.tar.gz")-                ]-    withDownloader opt downloader items-    doesFileExist (cabals <.> "tar.gz") >>= \b -> when b $ extractTarball cabals-    doesFileExist (inputs <.> "tar.gz") >>= \b -> when b $ extractTarball inputs---check :: String -> IO (Maybe FilePath)-check name = do-    res <- findExecutable name-    when (isNothing res) $ do-        putStrLn $ "WARNING: Could not find command line program " ++ name ++ "."-        when isWindows $ putStrLn $ "  You may be able to install it from:\n  " ++ url-    return res-    where-        srcList = [ ("gzip", "http://gnuwin32.sourceforge.net/packages/gzip.htm")-                  , ("tar",  "http://gnuwin32.sourceforge.net/packages/gtar.htm")-                  , ("wget", "http://gnuwin32.sourceforge.net/packages/wget.htm")-                  , ("curl", "http://curl.haxx.se/download.html")-                  ]-        url = fromJust . lookup name $ srcList--extractTarball :: FilePath -> IO ()-extractTarball out = do-        createDirectoryIfMissing True out-        withDirectory out $ do-            hasGzip <- check "gzip"-            hasTar  <- check "tar"-            when (any isNothing [hasGzip, hasTar]) $ error "Could not extract tarball(s), could not find either gzip or tar on the $PATH."-            putStrLn "Extracting tarball... "-            system_ $ "gzip --decompress --stdout --force .." </> takeFileName out <.> "tar.gz > .." </> takeFileName out <.> "tar"-            system_ $ "tar -xf .." </> takeFileName out <.> "tar"-            putStrLn "Finished extracting tarball"-        writeFile (out <.> "txt") ""

@@ -1,44 +0,0 @@--module Recipe.General(convertSrc, convert, combine) where--import Recipe.Type-import Hoogle-import General.Base-import General.System-import System.Console.CmdArgs.Verbosity--txt x = map toLower x <.> "txt"-hoo x = map toLower x <.> "hoo"---convertSrc :: ([Name] -> IO ()) -> [Name] -> Name -> String -> IO ()-convertSrc make deps x src = do-    writeFileUtf8 (txt x) src-    make deps-    convert (map hoo deps) x (hoo x) src------- convert a single database-convert :: [FilePath] -> Name -> FilePath -> String -> IO ()-convert deps x out src = do-    deps2 <- filterM doesFileExist deps-    when (deps /= deps2) $ putWarning $ "Warning: " ++ x ++ " doesn't know about dependencies on " ++ unwords (deps \\ deps2)-    dbs <- mapM loadDatabase deps2-    let (err,db) = createDatabase Haskell dbs src-    unless (null err) $ outStrLn $ "Skipped " ++ show (length err) ++ " warnings in " ++ x-    whenLoud $ outStr $ unlines $ map show err-    outStr $ "Converting " ++ x ++ "... "-    performGC-    saveDatabase out db-    outStrLn "done"----- combine multiple databases-combine :: ([Name] -> IO ()) -> Name -> [Name] -> Bool -> IO ()-combine make x deps force = do-    make deps-    dbs <- mapM (loadDatabase . hoo) deps-    outStr $ "Creating " ++ x ++ " from " ++ show (length deps) ++ " databases... "-    performGC-    saveDatabase (hoo x) $ mconcat dbs-    outStrLn "done"

@@ -1,90 +1,19 @@ -module Recipe.Hackage(makePlatform, makeDefault, makePackage, makeAll) where+module Recipe.Hackage(platformPackages, avoid, findLocal) where -import Recipe.Type-import Recipe.Cabal-import Recipe.General-import Recipe.Haddock import General.Base import General.System import General.Util import General.Web-import Control.Exception+import System.FilePath hiding (combine)   -- FIXME: This is a list of hack avoid = words "ghc-prim integer integer-simple integer-gmp rts ghc Win32"  -makePlatform :: ([Name] -> IO ()) -> IO ()-makePlatform make = do-    xs <- listPlatform-    forM_ xs $ \(name,ver) -> do-        v2 <- version cabals name-        when (ver /= v2) $ putStrLn $ "Warning: Version mismatch for " ++ name ++ " (platform=" ++ ver ++ ", cabal=" ++ v2 ++ ")"-    combine make "platform" (map fst xs) False---makeAll :: ([Name] -> IO ()) -> IO ()-makeAll make = do-    xs <- listing inputs-    make xs----- create a database containing an entry for each package in hackage-makePackage :: IO ()-makePackage = do-    xs <- listing cabals-    xs <- forM xs $ \name -> do-        ver <- version cabals name-        let file = cabals </> name </> ver </> name <.> "cabal"-        src <- readCabal file-        return $ case src of-            Nothing -> []-            Just src ->-                [""] ++ zipWith (++) ("-- | " : repeat "--   ") (cabalDescription src) ++-                ["--","-- Version " ++ ver, "@url package/" ++ name, "@entry package " ++ name]-    convertSrc noDeps [] "package" $ unlines $-        "@url http://hackage.haskell.org/" : "@package package" : concat xs---makeDefault :: ([Name] -> IO ()) -> [FilePath] -> Name -> IO ()-makeDefault make local "ghc" = do-    had <- try $ readFileUtf8' "download/ghc.txt"-    case had of-        Left e -> putWarning $ "Warning: Exception when reading haddock for ghc, " ++ show (e :: SomeException)-        Right had -> do-	    loc <- findLocal local "ghc"-            convertSrc make ["base"] "ghc" $ unlines $ "@depends base" : haddockHacks (url loc) (lines had)-            where url loc = if isNothing loc-                              then Just "http://www.haskell.org/ghc/docs/latest/html/libraries/ghc/"-                              else loc--makeDefault make local name = do-    let base = name == "base"-    b1 <- doesDirectoryExist $ cabals </> name-    b2 <- doesDirectoryExist $ inputs </> name-    if not base && (not b1 || not b2) then-        putWarning $ "Warning: " ++ name ++ " couldn't find both Cabal and Haddock inputs"-     else do-        vc <- version cabals name-        vh <- if base then return vc else version inputs name-        when (vc /= vh) $ putStrLn $ "Warning: Version mismatch for " ++ name ++ " (cabal=" ++ vc ++ ", haddock=" ++ vh ++ ")"-        let had = if base then "download/base.txt" else inputs </> name </> vh </> "doc" </> "html" </> name <.> "txt"-            cab = cabals </> name </> vc </> name <.> "cabal"-        had <- try $ readFileUtf8' had-        case had of-            Left e -> putWarning $ "Warning: Exception when reading haddock for " ++ name ++ ", " ++ show (e :: SomeException)-            Right had -> do-                deps <- fmap (maybe [] cabalDepends) $ readCabal cab-                let cleanDeps = deps \\ (name:avoid)-                loc <- findLocal local name-                convertSrc make cleanDeps name $ unlines $-                    ["@depends " ++ a | a <- cleanDeps] ++ haddockHacks loc (lines had)-- -- try and find a local filepath-findLocal :: [FilePath] -> Name -> IO (Maybe URL)+findLocal :: [FilePath] -> String -> IO (Maybe URL) findLocal paths name = fmap (listToMaybe . concat . concat) $ forM paths $ \p -> do     xs <- getDirectoryContents p     xs <- return [p </> x | x <- reverse $ sort xs, name == fst (rbreak (== '-') x)] -- make sure highest version comes first@@ -98,15 +27,16 @@ --------------------------------------------------------------------- -- READ PLATFORM -listPlatform :: IO [(Name,String)]-listPlatform = do-    src <- readFile platform+platformPackages :: String -> [String]+platformPackages = map fst . parsePlatform++parsePlatform src =     let xs = takeWhile (not . isPrefixOf "build-tools:" . ltrim) $              dropWhile (not . isPrefixOf "build-depends:" . ltrim) $              lines src-    return [(name, takeWhile (\x -> x == '.' || isDigit x) $ drop 1 b)-           | x <- xs, (a,_:b) <- [break (== '=') x], let name = trim $ dropWhile (== '-') $ trim a-           , not $ avoid name]+    in [(name, takeWhile (\x -> x == '.' || isDigit x) $ drop 1 b)+       | x <- xs, (a,_:b) <- [break (== '=') x], let name = trim $ dropWhile (== '-') $ trim a+       , not $ avoid name]     where         avoid x = ("haskell" `isPrefixOf` x && all isDigit (drop 7 x)) ||                   (x `elem` words "Cabal hpc Win32")

@@ -1,18 +1,12 @@ -module Recipe.Keyword(makeKeyword) where+module Recipe.Keyword(translateKeywords) where  import General.Base import Text.HTML.TagSoup-import Recipe.Type-import Recipe.General  -makeKeyword :: IO ()-makeKeyword = convertSrc noDeps [] "keyword" . translate =<< readFileUtf8' keywords---translate :: String -> String-translate src = unlines $ keywordPrefix ++ items+translateKeywords :: String -> String+translateKeywords src = unlines $ keywordPrefix ++ items     where items = concatMap keywordFormat $ partitions (~== "<span class='mw-headline' id>") $                   takeWhile (~/= "<div class=printfooter>") $ parseTags src 

@@ -1,76 +0,0 @@--module Recipe.Type(-    CmdLine(..), Name, noDeps, safeEncoding,-    keywords, platform, cabals, inputs, listing, version,-    resetWarnings, putWarning, recapWarnings,-    outStr, outStrLn-    ) where--import CmdLine.All-import Control.Concurrent-import System.IO.Unsafe-import General.Base-import General.System---type Name = String---noDeps :: [Name] -> IO ()-noDeps [] = return ()-noDeps xs = error "Internal error: package with no dependencies had dependencies"----- | Lots of things go slightly wrong if you use characters > 127 in places, this just replaces them with ?-safeEncoding :: String -> String-safeEncoding = map (\x -> if x <= '\0' || x > '\127' then '?' else x)--------------------------------------------------------------------------- DOWNLOADED INFORMATION--keywords = "download/keyword.txt"-platform = "download/haskell-platform.cabal"-cabals = "download/hackage-cabal"-inputs = "download/hackage-hoogle"--listing :: FilePath -> IO [Name]-listing dir = do-    xs <- getDirectoryContents dir-    return $ sortBy (comparing $ map toLower) $ filter (`notElem` [".","..","preferred-versions"]) xs--version :: FilePath -> Name -> IO String-version dir x = do-    ys <- getDirectoryContents $ dir </> x-    when (null ys) $ error $ "Couldn't find version for " ++ x ++ " in " ++ dir-    let f = map (read :: String -> Int) . words . map (\x -> if x == '.' then ' ' else x)-    return $ maximumBy (comparing f) $ filter (all (not . isAlpha)) ys--------------------------------------------------------------------------- WARNING MESSAGES--{-# NOINLINE warnings #-}-warnings :: MVar [String]-warnings = unsafePerformIO $ newMVar []--putWarning :: String -> IO ()-putWarning x = do-    outStrLn x-    modifyMVar_ warnings $ return . (x:)--recapWarnings :: IO ()-recapWarnings = do-    xs <- readMVar warnings-    mapM_ outStrLn $ reverse xs--resetWarnings :: IO ()-resetWarnings = modifyMVar_ warnings $ const $ return []---outputLock :: MVar ()-outputLock = unsafePerformIO $ newMVar ()--outStr, outStrLn :: String -> IO ()-outStr x = withMVar outputLock $ \_ -> do putStr x; hFlush stdout-outStrLn x = outStr $ x ++ "\n"

@@ -0,0 +1,25 @@++module Recipe.Warning(+    resetWarnings, putWarning, recapWarnings+    ) where++import Control.Concurrent+import System.IO.Unsafe+++{-# NOINLINE warnings #-}+warnings :: MVar [String]+warnings = unsafePerformIO $ newMVar []++putWarning :: String -> IO ()+putWarning x = do+    putStrLn x+    modifyMVar_ warnings $ return . (x:)++recapWarnings :: IO ()+recapWarnings = do+    xs <- readMVar warnings+    mapM_ putStrLn $ reverse xs++resetWarnings :: IO ()+resetWarnings = modifyMVar_ warnings $ const $ return []

@@ -6,6 +6,7 @@ import System.Directory import System.Exit import System.FilePath+import System.Environment   main :: IO ()@@ -19,5 +20,9 @@             res <- system cmd             when (res /= ExitSuccess) $ error $ "Command: " ++ cmd ++ "\nFailed with: " ++ show res -    hoogle "data"-    hoogle "test --example"+    args <- getArgs+    if "--no-net" `elem` args then+        hoogle "test"+     else do+        hoogle "data"+        hoogle "test --example"

@@ -2,8 +2,13 @@  module Web.Server(server) where +#ifndef MIN_VERSION_wai+#define MIN_VERSION_wai(a,b,c) 0+#endif+ import General.Base import General.Web+import System.FilePath import CmdLine.All import Web.Response import Network.HTTP.Types