diff --git a/CHANGES.txt b/CHANGES.txt
new file mode 100644
--- /dev/null
+++ b/CHANGES.txt
@@ -0,0 +1,173 @@
+Changelog for Hoogle
+
+4.2.27
+    Rewrite how "hoogle data" works
+    #45, if you are building all, also depend on default
+    #47, switch to using Shake to build the recipes
+    Add a dependency on Shake
+4.2.26
+    Fix omission of the Test file
+4.2.25
+    Fix timestamps in .tar.gz dist file
+4.2.24
+    Allow WAI 2.0
+    Fix the Keyword generator
+4.2.23
+    Improvements to Haddock conversion
+4.2.22
+    Download Hoogle data from the old Hackage server
+4.2.21
+    #25, make the tarball pass the tests
+    #26, GHC 7.8 compatibility
+    Ensure alias resolution is bottom-up and recursive
+4.2.20
+    #619, do not depend on unix on Windows
+4.2.19
+    Fix version bounds
+4.2.18
+    Allow bytestring-0.9
+4.2.17
+    Upgrade to haskell-src-exts-1.14
+    Visual redesign
+    Require bytestring-0.10 or above, due to NFData instances
+    Add NFData instance to Database
+4.2.16
+    Remove lots of explicit upper bounds
+4.2.15
+    Allow http-types-0.8
+    Allow case-insensitive-1.0
+    Support --local for GHC docs
+    #568, rewrite Haddock links in --local
+4.2.14
+    Switch license from GPL to BSD3
+    GHC 7.6 fixes
+    ##4, Allow Cabal-1.16.*
+4.2.13
+    Allow cmdargs-0.10.*
+4.2.12
+    Allow conduit-0.5, wai-1.3 and warp-1.3
+    Upgrade to http-types-0.7.*
+    In embedded mode, don't hijack the URL bar or title
+    Upgrade to jQuery 1.7.2
+    #89, add a format=json output mode
+    #476, update the URL as you type
+    Improve iPhone support
+    Update the copyright year in the footer
+    Double the speed of hoogle server responses
+4.2.11
+    Allow transformers-0.3
+    Allow conduit-0.4, wai-1.2 and warp-1.2
+    Allow haskell-src-exts-1.12 and 1.13
+4.2.10
+    #533, be paranoid when parsing Cabal files
+4.2.9
+    Upgrade to WAI-1.1 and Warp-1.1
+    #525, don't rely on a redirect to find files
+    Allow Cabal-1.14
+4.2.8
+    Allow cmdargs-0.9
+    Fix setting the global read flags and unsetting them properly
+    Allow case-insensitive-0.4.*
+    Make data --local have a good default on Linux
+    Make server --local work on Windows
+    #407, rewrite OpenSearch plugin to work regardless of the server
+4.2.7
+    Allow cmdargs-0.8
+4.2.6
+    Permit GHC 7.2
+    Allow case-insensitive-0.3.*
+    Fix bug when searching for toString in the embed mode
+    Make --redownload work on the tarballs
+4.2.5
+    Fix hyperlinks to symbols (i.e. &&) in Haddock
+4.2.4
+    #434, use either wget or curl to download the files
+    Allow haskell-src-exts 1.11.*
+    Upgrade to cmdargs==0.7.*
+4.2.3
+    Generate database for ghc internals
+    Give an error if the database is the wrong version
+    Add -fno-cse, to make cmdargs work (was losing args annotations)
+4.2.2
+    Upgrade to WAI==0.4.* and Warp==0.4.*
+    Allow parsec 3, since various people have reported it works
+    Default module URL's correctly when not on Hackage
+    #417, use proper Haddock parsing for package descriptions
+    Make sure https: links work through the web mode
+    Rewrite the binary defer layer, cleaner and slightly faster
+    Eliminate escaping problems when entering "'s in the search
+    Some work on #375, add links to the library page
+    Add --template to override the templates at runtime
+    Add --dynamic flag to server mode, include update stamps
+    Upgrade wai and warp, adds an exception handler
+    #82, rewrite showing document snippets, better expand/collapse
+    Support <em> tags in Haddock output (as well as <i>)
+    Make package matching case insensitive
+    #372, add features to improve module name search
+    Make module name searching case insensitive
+4.2.1
+    Fix for pattern match failure when searching for "to"
+    Fix for not sending text/html when serving local files
+    Eliminate the --nostdin flag, now unnecessary
+    Change to use WAI/Warp instead of HTTP and custom server
+    Fix a bug, didn't ever show Waiting... on the real website
+    Make embed work in IE8
+4.2
+    Make the CGI interface send the right number of newlines
+    Accept prefix and suffix web parameters
+    Eliminate isBlankQuery, make Query a Monoid
+    Make Query abstract, add queryPackages/querySetPackage
+    Rename querySuggestions/queryCompletions by dropping the query
+    Add mode=embed support
+    Add some log analysis features
+    Ignore some whitespace in the input file
+    Work around a cabal bug, include jquery.cookie.js
+    Add ?version=xxx to .css and .js, to make it auto refresh
+4.1.5
+    #399, rehabilitate mode=suggest
+    Stop mode=suggest failing entirely, now cleanly gives no answers
+    Combine --web and --webmode
+    Complete the command line flag help
+    #327, ensure utf8 is found
+    #187, ensure LT is found
+    Use the Hoogle tarball direct from Hackage
+    #54, rework packages, entries and keywords
+    #87, make all database files lowercase
+    Various javascript/ajax enhancements
+4.1.4
+    Running data on Posix sets all files to global read/execute
+    Relax haskell-src-exts to allow 1.10.1
+    Rework the web page display further
+4.1.3
+    Read and write all database files in UTF8
+    #391, add tar -xzf when extracting from the tarball
+    Change search results display, packages/modules on separate line
+    Exclude haskellN* from the platform, they are in base
+    Add --nostdin to server mode, to support running with nohup
+    Loosen dependencies for tagsoup (typo'd it before)
+4.1.2
+    Loosen dependencies for tagsoup and HTTP (but not parsec)
+    Avoid self circular dependencies
+4.1.1
+    #195, include all the necessary resources to run as a server
+    When in server mode, be more robust to errors
+4.1
+    #320, make sure system is found
+    #146, preserve forall's in everywhere
+    #186, switch to using haskell-src-exts to parse input files
+    #249, make sure you find MonadWriter
+    #235, make sure you find forall
+    #309, make sure there are links for keywords
+    #78, add --link flag
+    #59, operators look nicer, and have blue brackets round them
+    #352, can now parse ( # ) as a function name in the input
+    #280, searching for "~ +keyword" should find keyword
+    #116, fix links for types/classes
+    #83, all searching for package-name
+    #94, allow textbases to be used automatically
+    #92, short flags are now handled by cmdargs, so work
+    #51, test mode now does what test and testfile did
+    #66, turn Hoogle into a library
+    Add --server flag, to run as a server
+4.0.7
+    Start of changelog
diff --git a/hoogle.cabal b/hoogle.cabal
--- a/hoogle.cabal
+++ b/hoogle.cabal
@@ -1,7 +1,7 @@
 cabal-version:      >= 1.10
 build-type:         Simple
 name:               hoogle
-version:            4.2.26
+version:            4.2.27
 license:            BSD3
 license-file:       docs/LICENSE
 category:           Development
@@ -16,8 +16,10 @@
 homepage:           http://www.haskell.org/hoogle/
 bug-reports:        https://github.com/ndmitchell/hoogle/issues
 tested-with:        GHC==7.6.3, GHC==7.4.2
-extra-source-files: README.txt
-                    datadir/*.txt
+extra-source-files:
+    README.txt
+    CHANGES.txt
+    datadir/*.txt
 
 data-dir: datadir
 data-files:
@@ -137,6 +139,7 @@
         wai >= 1.1,
         warp >= 1.1,
         Cabal >= 1.8,
+        shake >= 0.10.7,
         haskell-src-exts >= 1.14 && < 1.15
 
     if !os(mingw32)
@@ -154,12 +157,11 @@
         Paths_hoogle
         Recipe.All
         Recipe.Cabal
-        Recipe.Download
-        Recipe.General
+        Recipe.Command
         Recipe.Hackage
         Recipe.Haddock
         Recipe.Keyword
-        Recipe.Type
+        Recipe.Warning
         Test.All
         Test.Docs
         Test.General
diff --git a/src/CmdLine/All.hs b/src/CmdLine/All.hs
--- a/src/CmdLine/All.hs
+++ b/src/CmdLine/All.hs
@@ -18,6 +18,7 @@
 
 import General.Base
 import General.System
+import System.FilePath
 import CmdLine.Type
 import CmdLine.Load
 import General.Web
diff --git a/src/CmdLine/Load.hs b/src/CmdLine/Load.hs
--- a/src/CmdLine/Load.hs
+++ b/src/CmdLine/Load.hs
@@ -5,6 +5,7 @@
 import General.Base
 import General.Util
 import General.System
+import System.FilePath
 
 
 -- | Given a list of search directories, and a query, load the databases you
diff --git a/src/CmdLine/Type.hs b/src/CmdLine/Type.hs
--- a/src/CmdLine/Type.hs
+++ b/src/CmdLine/Type.hs
@@ -102,7 +102,7 @@
 data_ = Data
     {datadir = def &= typDir &= help "Database directory"
     ,redownload = def &= help "Redownload all files from the web"
-    ,threads = def &= typ "INT" &= name "j" &= help "Number of threads to use" &= ignore -- ignore until it works
+    ,threads = 1 &= typ "INT" &= name "j" &= help "Number of threads to use"
     ,actions = def &= args &= typ "RULE"
     ,local = def &= opt "" &= typ "FILEPATH" &= help "Use local documentation if available"
     } &= help "Generate Hoogle databases"
diff --git a/src/Console/All.hs b/src/Console/All.hs
--- a/src/Console/All.hs
+++ b/src/Console/All.hs
@@ -3,7 +3,6 @@
 
 import CmdLine.All
 import Recipe.All
-import Recipe.General
 import Recipe.Haddock
 import Console.Log
 import Console.Search
@@ -12,6 +11,7 @@
 import General.Base
 import General.System
 import General.Web
+import System.FilePath
 import Hoogle
 
 
@@ -75,3 +75,16 @@
 
 action q@Search{} = actionSearch q (fromRight $ queryParsed q)
 
+
+--- convert a single database
+convert :: [FilePath] -> String -> FilePath -> String -> IO ()
+convert deps x out src = do
+    deps2 <- filterM doesFileExist deps
+    when (deps /= deps2) $ putStrLn $ "Warning: " ++ x ++ " doesn't know about dependencies on " ++ unwords (deps \\ deps2)
+    dbs <- mapM loadDatabase deps2
+    let (err,db) = createDatabase Haskell dbs src
+    unless (null err) $ putStrLn $ "Skipped " ++ show (length err) ++ " warnings in " ++ x
+    putStr $ "Converting " ++ x ++ "... "
+    performGC
+    saveDatabase out db
+    putStrLn "done"
diff --git a/src/Console/Test.hs b/src/Console/Test.hs
--- a/src/Console/Test.hs
+++ b/src/Console/Test.hs
@@ -7,6 +7,7 @@
 import Hoogle
 import General.Base
 import General.System
+import System.FilePath
 import Paths_hoogle
 import CmdLine.All
 import Test.All
diff --git a/src/General/Base.hs b/src/General/Base.hs
--- a/src/General/Base.hs
+++ b/src/General/Base.hs
@@ -20,7 +20,6 @@
 import Data.Word as X
 import Debug.Trace as X (trace)
 import Numeric as X (readHex,showHex)
-import System.FilePath as X hiding (combine)
 import qualified Data.ByteString.Char8 as BS
 import qualified Data.ByteString.Lazy.Char8 as LBS
 
diff --git a/src/General/System.hs b/src/General/System.hs
--- a/src/General/System.hs
+++ b/src/General/System.hs
@@ -33,6 +33,9 @@
 #endif
 
 
+removeFile_ x = removeFile x `E.catch` \(_ :: E.SomeException) -> return ()
+
+
 withDirectory dir cmd = E.bracket
     (do x <- getCurrentDirectory; setCurrentDirectory dir; return x)
     setCurrentDirectory
@@ -53,7 +56,7 @@
 #else
 captureOutput act = do
     tmp <- getTemporaryDirectory
-    (f,h) <- openTempFile tmp "hlint"
+    (f,h) <- openTempFile tmp "hoogle"
     sto <- hDuplicate stdout
     ste <- hDuplicate stderr
     hDuplicateTo h stdout
diff --git a/src/General/Web.hs b/src/General/Web.hs
--- a/src/General/Web.hs
+++ b/src/General/Web.hs
@@ -13,8 +13,13 @@
     Args, cgiArgs, cgiResponse, parseHttpQueryArgs
     ) where
 
+#ifndef MIN_VERSION_wai
+#define MIN_VERSION_wai(a,b,c) 0
+#endif
+
 import General.System
 import General.Base
+import System.FilePath
 import Network.Wai
 #if MIN_VERSION_wai(2, 0, 0)
 import Network.Wai.Internal
diff --git a/src/Hoogle/Language/Haskell.hs b/src/Hoogle/Language/Haskell.hs
--- a/src/Hoogle/Language/Haskell.hs
+++ b/src/Hoogle/Language/Haskell.hs
@@ -4,6 +4,7 @@
 
 import General.Base
 import General.Util
+import System.FilePath
 import Hoogle.Type.All
 import Language.Haskell.Exts.Annotated hiding (TypeSig,Type)
 import qualified Language.Haskell.Exts.Annotated as HSE
diff --git a/src/Recipe/All.hs b/src/Recipe/All.hs
--- a/src/Recipe/All.hs
+++ b/src/Recipe/All.hs
@@ -1,85 +1,171 @@
--- Recipe actions:
--- Download to foo.src in most cases, then extract to foo.txt, which can later be compiled to foo.hoo
+{-# LANGUAGE RecordWildCards, ScopedTypeVariables #-}
+
 module Recipe.All(recipes) where
 
-import General.Base
-import General.System
-import Control.Concurrent
-import qualified Data.Map as Map
+import General.Base hiding (readFile')
+import General.System as Sys
+import Control.Exception as E
+import qualified Data.Set as Set
+import Development.Shake
+import Development.Shake.FilePath
+import Recipe.Haddock
 
-import Recipe.Type
-import Recipe.Download
+import Recipe.Warning
+import Recipe.Command
 import Recipe.Keyword
-import Recipe.General
 import Recipe.Hackage
+import Recipe.Cabal
+import Hoogle
+import qualified Paths_hoogle as V
+import Data.Version
+import CmdLine.All
 
 
 -- CmdLine is guaranteed to be a constructor of type Data
 recipes :: CmdLine -> IO ()
-recipes opt = withModeGlobalRead $ do
+recipes opt@Data{..} = withModeGlobalRead $ do
     hSetBuffering stdout NoBuffering
-    createDirectoryIfMissing True $ datadir opt
-    withDirectory (datadir opt) $ do
+    createDirectoryIfMissing True datadir
+    withDirectory datadir $ do
         resetWarnings
-        download opt
-        let ys = parseRules $ actions opt
-        make opt (filter (not . null . snd) ys) (map fst ys)
+        when redownload $ do
+            forM_ urls $ \(file,_) -> removeFile_ $ "downloads" </> file
+        shake shakeOptions{shakeVersion=showVersion V.version, shakeThreads=threads, shakeProgress=progressSimple} $ do
+            want $ map (<.> "hoo") $ if null actions then ["default"] else actions
+            rules opt
         recapWarnings
         putStrLn "Data generation complete"
 
 
--- If I switch to the parallel-io library then it segfaults, due to GHC bug:
--- http://hackage.haskell.org/trac/ghc/ticket/4850
--- import "parallel-io" Control.Concurrent.ParallelIO.Local
-withPool i f = f ()
-extraWorkerWhileBlocked _ = id
-parallel_ _ = sequence_
+rules :: CmdLine -> Rules ()
+rules Data{..} = do
+    "downloads/packages.txt" *> \out -> do
+        need ["downloads/hoogle.untar","downloads/cabal.untar"]
+        as <- liftIO $ listing "downloads/hoogle"
+        bs <- liftIO $ listing "downloads/cabal"
+        writeFileLines out $ Set.toList $ Set.fromList as `Set.intersection` Set.fromList bs
 
+    packages <- do
+        cache <- newCache $ fmap (Set.fromList . lines) . readFile
+        return $ cache "downloads/packages.txt"
 
-data Status = Built | Building (MVar ())
+    phony "all.hoo" $ do
+        pkgs <- packages
+        need $ map (<.> "hoo") $ "default" : Set.toList pkgs
 
-make :: CmdLine -> [(Name,[Name])] -> [Name] -> IO ()
-make opt rules xs = withPool (threads opt) $ \pool -> do
-    ref <- newMVar Map.empty
-    fs ref pool [] xs
-    where
-        fs ref pool rec xs = parallel_ pool $ map (f ref pool rec) xs
+    "keyword.txt" *> \out -> do
+        let src = "downloads/keyword.htm"
+        need [src]
+        contents <- liftIO $ readFileUtf8' src
+        liftIO $ writeFileUtf8 out $ translateKeywords contents
 
-        f ref pool rec x
-            | x `elem` rec = outStrLn $ "Warning: Package database appears to be recursive, " ++ x
-            | otherwise = join $ modifyMVar ref $ \mp -> case Map.lookup x mp of
-                Just Built -> return (mp, return ())
-                Just (Building v) -> return $ (,) mp $
-                    extraWorkerWhileBlocked pool $ readMVar v
-                Nothing -> do
-                    v <- newEmptyMVar
-                    return $ (,) (Map.insert x (Building v) mp) $ do
-                        build (fs ref pool $ x:rec) opt rules x
-                        modifyMVar_ ref $ \mp -> return $ Map.insert x Built mp
-                        putMVar v ()
+    "default.txt" *> \out -> do
+        writeFileLines out ["@combine keyword","@combine package","@combine platform"]
 
+    "platform.txt" *> \out -> do
+        let src = "downloads/platform.cabal"
+        contents <- readFile' src
+        writeFileLines out ["@combine " ++ x | x <- platformPackages contents]
 
-build :: ([Name] -> IO ()) -> CmdLine -> [(Name,[Name])] -> Name -> IO ()
-build makeRec opt rules x = do
-    outStrLn $ "Starting " ++ x
-    case lookup x rules of
-        Just ys -> combine makeRec x ys True
-        _ -> case x of
-            "keyword" -> makeKeyword
-            "default" -> combine makeRec x ["keyword","package","platform"] False
-            "platform" -> makePlatform makeRec
-            "package" -> makePackage
-            "all" -> makeAll makeRec
-            _ -> makeDefault makeRec (local opt) x
-    outStrLn $ "Finished " ++ x
+    "package.txt" *> \out -> do
+        need ["downloads/cabal.untar"]
+        xs <- liftIO $ listing "downloads/cabal"
+        xs <- liftIO $ forM xs $ \name -> do
+            ver <- version "downloads/cabal" name
+            let file = "downloads/cabal" </> name </> ver </> name <.> "cabal"
+            src <- readCabal file
+            return $ case src of
+                Nothing -> []
+                Just src ->
+                    [""] ++ zipWith (++) ("-- | " : repeat "--   ") (cabalDescription src) ++
+                    ["--","-- Version " ++ ver, "@url package/" ++ name, "@entry package " ++ name]
+        liftIO $ writeFileUtf8 out $ unlines $ "@url http://hackage.haskell.org/" : "@package package" : concat xs
 
+    (\x -> "*.txt" ?== x && takeBaseName x `notElem` ["keyword","default","platform","package"]) ?> \out -> do
+        need ["downloads/hoogle.untar","downloads/cabal.untar"]
+        let name = takeBaseName out
+            base = name == "base"
+        vc <- liftIO $ version "downloads/cabal" name
+        vh <- liftIO $ if base then return vc else version "downloads/hoogle" name
+        let hoo = if base then "downloads/base.txt" else "downloads/hoogle" </> name </> vh </> "doc" </> "html" </> name <.> "txt"
+            cab = "downloads/cabal" </> name </> vc </> name <.> "cabal"
+        need [hoo, cab]
+        hoo <- liftIO $ readFileUtf8' hoo `E.catch` \(_ :: SomeException) -> readFile hoo
+        deps <- liftIO $ fmap (maybe [] cabalDepends) $ readCabal cab
+        let cleanDeps = deps \\ (name:avoid)
+        loc <- liftIO $ findLocal local name
+        liftIO $ writeFileUtf8 out $ unlines $
+            ["@depends " ++ a | a <- cleanDeps] ++ haddockHacks loc (lines hoo)
 
-parseRules :: [String] -> [(Name,[Name])]
-parseRules [] = [("default",[])]
-parseRules xs = map parseRule xs
+    imported <- newCache $ \file -> do
+        xs <- readFileUtf8' file
+        return [x | x <- lines xs, takeWhile (not . isSpace) x `elem` ["type","data","newtype","class","instance","@depends"]]
+    let listDeps = map (drop 9) . takeWhile ("@depends " `isPrefixOf`)
 
+    let genImported pkgs seen [] = return []
+        genImported pkgs seen (t:odo)
+            | t `Set.member` seen || not (t `Set.member` pkgs) = genImported pkgs seen odo
+            | otherwise = do
+                i <- imported $ t <.> "txt"
+                let deps = listDeps i
+                fmap (i++) $ genImported pkgs (Set.insert t seen) (deps++odo)
 
-parseRule :: String -> (Name,[Name])
-parseRule x = (a, uncommas $ drop 1 b)
-    where (a,b) = break (== '=') x
-          uncommas = words . map (\x -> if x == ',' then ' ' else x)
+    (\x -> "*.hoo" ?== x && x /= "all.hoo") ?> \out -> do
+        let src = out -<.> "txt"
+        need [src]
+        contents <- liftIO $ fmap lines $ readFileUtf8' src
+        if not (null contents) && "@combine " `isPrefixOf` head contents then do
+            let deps = [x <.> "hoo" | x <- contents, Just x <- [stripPrefix "@combine " x]]
+            need deps
+            dbs <- liftIO $ mapM loadDatabase deps
+            putNormal $ "Creating " ++ out ++ " from " ++ show (length deps) ++ " databases... "
+            liftIO $ performGC
+            liftIO $ saveDatabase out $ mconcat dbs
+         else do
+            pkgs <- packages
+            deps <- genImported pkgs (Set.singleton $ takeBaseName out) $ listDeps contents
+            let (err,db) = createDatabase Haskell [snd $ createDatabase Haskell [] $ unlines deps] $ unlines contents
+            unless (null err) $ putNormal $ "Skipped " ++ show (length err) ++ " warnings in " ++ out
+            putLoud $ unlines $ map show err
+            putNormal $ "Creating " ++ out ++ "... "
+            liftIO $ performGC
+            liftIO $ saveDatabase out db
+
+    "//*.tar" *> \out -> do
+        let src = out <.> "gz"
+        need [src]
+        ungzip src out
+
+    "//*.untar" *> \out -> do
+        let src = out -<.> "tar"
+        need [src]
+        untar src
+        writeFile' out ""
+
+    (\x -> "downloads/*" ?== x && isJust (lookup (takeFileName x) urls)) ?> \out -> do
+        let Just url = lookup (takeFileName out) urls
+        -- liftIO $ copyFile ("C:/spacework/hoogle/cache" </> takeFileName out) out
+        wget url out
+
+
+urls :: [(FilePath, URL)]
+urls = let (*) = (,) in
+    ["keyword.htm" * "http://www.haskell.org/haskellwiki/Keywords"
+    ,"platform.cabal" * "http://code.galois.com/darcs/haskell-platform/haskell-platform.cabal"
+    ,"base.txt" * "http://www.haskell.org/hoogle/base.txt"
+    ,"ghc.txt" * "http://www.haskell.org/ghc/docs/latest/html/libraries/ghc/ghc.txt"
+    ,"cabal.tar.gz" * "http://hackage.haskell.org/packages/index.tar.gz"
+    ,"hoogle.tar.gz" * "http://hackage.haskell.org/packages/hoogle.tar.gz"]
+
+
+listing :: FilePath -> IO [String]
+listing dir = do
+    xs <- Sys.getDirectoryContents dir
+    return $ sortBy (comparing $ map toLower) $ filter (`notElem` [".","..","preferred-versions"]) xs
+
+version :: FilePath -> String -> IO String
+version dir x = do
+    ys <- Sys.getDirectoryContents $ dir </> x
+    when (null ys) $ error $ "Couldn't find version for " ++ x ++ " in " ++ dir
+    let f = map (read :: String -> Int) . words . map (\x -> if x == '.' then ' ' else x)
+    return $ maximumBy (comparing f) $ filter (all (not . isAlpha)) ys
diff --git a/src/Recipe/Cabal.hs b/src/Recipe/Cabal.hs
--- a/src/Recipe/Cabal.hs
+++ b/src/Recipe/Cabal.hs
@@ -13,11 +13,11 @@
 import Distribution.Verbosity
 import Distribution.Version
 import Recipe.Haddock
-import Recipe.Type
+import Recipe.Warning
 import Control.Exception
 
 
-ghcVersion = [7,4,1]
+ghcVersion = [7,6,3]
 
 data Cabal = Cabal
     {cabalName :: String
diff --git a/src/Recipe/Command.hs b/src/Recipe/Command.hs
new file mode 100644
--- /dev/null
+++ b/src/Recipe/Command.hs
@@ -0,0 +1,62 @@
+
+module Recipe.Command(wget, ungzip, untar) where
+
+import General.Base
+import General.System
+import System.FilePath
+import Development.Shake(Action, command, CmdOption(..), liftIO)
+
+
+wget :: URL -> FilePath -> Action ()
+wget from to = do
+    dl <- liftIO findDownloader
+    command [Shell] (dl to from) []
+
+
+ungzip :: FilePath -> FilePath -> Action ()
+ungzip from to = do
+    hasGzip <- liftIO $ check "gzip"
+    when (isNothing hasGzip) $ error "Could not extract tarballs, could not find tar on the $PATH."
+    command [Shell] ("gzip --decompress --stdout --force " ++ from ++ " > " ++ to) []
+
+untar :: FilePath -> Action ()
+untar from = do
+    hasTar  <- liftIO $ check "tar"
+    when (isNothing hasTar) $ error "Could not extract tarballs, could not find tar on the $PATH."
+    liftIO $ createDirectoryIfMissing True $ dropExtension from
+    command [Shell, Cwd $ dropExtension from] ("tar -xf ../" ++ takeFileName from) []
+
+
+
+
+type Downloader = FilePath -> URL -> String
+
+wget2 :: Downloader
+wget2 fp url = "wget -nv " ++ url ++ " --output-document=" ++ fp
+curl :: Downloader
+curl fp url = "curl -sSL " ++ url ++ " --output " ++ fp
+
+findDownloader :: IO Downloader
+findDownloader = do
+    dl <- check "wget"
+    dl <- maybe (check "curl") (return . Just) dl
+    when (isNothing dl) $ error "Could not find downloader, neither curl nor wget are on the $PATH."
+    return $ matchDl (fromJust dl)
+    where matchDl d | "wget" `isInfixOf` d = wget2
+                    | "curl" `isInfixOf` d = curl
+
+
+check :: String -> IO (Maybe FilePath)
+check name = do
+    res <- findExecutable name
+    when (isNothing res) $ do
+        putStrLn $ "WARNING: Could not find command line program " ++ name ++ "."
+        when isWindows $ putStrLn $ "  You may be able to install it from:\n  " ++ url
+    return res
+    where
+        srcList = [ ("gzip", "http://gnuwin32.sourceforge.net/packages/gzip.htm")
+                  , ("tar",  "http://gnuwin32.sourceforge.net/packages/gtar.htm")
+                  , ("wget", "http://gnuwin32.sourceforge.net/packages/wget.htm")
+                  , ("curl", "http://curl.haxx.se/download.html")
+                  ]
+        url = fromJust . lookup name $ srcList
diff --git a/src/Recipe/Download.hs b/src/Recipe/Download.hs
deleted file mode 100644
--- a/src/Recipe/Download.hs
+++ /dev/null
@@ -1,80 +0,0 @@
-
-module Recipe.Download(download) where
-
-import General.Base
-import General.System
-import Recipe.Type
-
-type Downloader = FilePath -> URL -> String
-
-wget :: Downloader
-wget fp url = "wget -nv " ++ url ++ " --output-document=" ++ fp
-curl :: Downloader
-curl fp url = "curl -sSL " ++ url ++ " --output " ++ fp
-
-findDownloader :: IO Downloader
-findDownloader = do
-    dl <- liftM2 mplus (check "wget") (check "curl")
-    when (isNothing dl) $ error "Could not find downloader, neither curl nor wget are on the $PATH."
-    return $ matchDl (fromJust dl)
-    where matchDl d | "wget" `isInfixOf` d = wget
-                    | "curl" `isInfixOf` d = curl
-
-withDownloader :: CmdLine -> Downloader -> [(FilePath, FilePath, URL)] -> IO ()
-withDownloader opt downloader items =
-    let sys = fmap (== ExitSuccess) . system
-        download (f, f', u) = do
-            b <- doesFileExist f
-            when (not b || redownload opt) $ do
-                res <- sys (downloader f' u)
-                unless res $ do
-                    b <- doesFileExist f'
-                    when b $ removeFile f'
-                    error $ "Failed to download: " ++ u
-            doesFileExist f'
-    in forM_ items download
-
--- download everything required for the recipes
-download :: CmdLine -> IO ()
-download opt = do
-    createDirectoryIfMissing True "download"
-    downloader <- findDownloader
-    let items = [ (keywords, keywords, "http://www.haskell.org/haskellwiki/Keywords")
-                , (platform, platform, "http://code.galois.com/darcs/haskell-platform/haskell-platform.cabal")
-                , ("download/base.txt", "download/base.txt", "http://www.haskell.org/hoogle/base.txt")
-                , ("download/ghc.txt",  "download/ghc.txt", "http://www.haskell.org/ghc/docs/latest/html/libraries/ghc/ghc.txt")
-                , (cabals <.> "txt", cabals <.> "tar.gz", "http://hackage.haskell.org/packages/archive/00-index.tar.gz")
-                , (inputs <.> "txt", inputs <.> "tar.gz", "http://old.hackage.haskell.org/packages/archive/00-hoogle.tar.gz")
-                ]
-    withDownloader opt downloader items
-    doesFileExist (cabals <.> "tar.gz") >>= \b -> when b $ extractTarball cabals
-    doesFileExist (inputs <.> "tar.gz") >>= \b -> when b $ extractTarball inputs
-
-
-check :: String -> IO (Maybe FilePath)
-check name = do
-    res <- findExecutable name
-    when (isNothing res) $ do
-        putStrLn $ "WARNING: Could not find command line program " ++ name ++ "."
-        when isWindows $ putStrLn $ "  You may be able to install it from:\n  " ++ url
-    return res
-    where
-        srcList = [ ("gzip", "http://gnuwin32.sourceforge.net/packages/gzip.htm")
-                  , ("tar",  "http://gnuwin32.sourceforge.net/packages/gtar.htm")
-                  , ("wget", "http://gnuwin32.sourceforge.net/packages/wget.htm")
-                  , ("curl", "http://curl.haxx.se/download.html")
-                  ]
-        url = fromJust . lookup name $ srcList
-
-extractTarball :: FilePath -> IO ()
-extractTarball out = do
-        createDirectoryIfMissing True out
-        withDirectory out $ do
-            hasGzip <- check "gzip"
-            hasTar  <- check "tar"
-            when (any isNothing [hasGzip, hasTar]) $ error "Could not extract tarball(s), could not find either gzip or tar on the $PATH."
-            putStrLn "Extracting tarball... "
-            system_ $ "gzip --decompress --stdout --force .." </> takeFileName out <.> "tar.gz > .." </> takeFileName out <.> "tar"
-            system_ $ "tar -xf .." </> takeFileName out <.> "tar"
-            putStrLn "Finished extracting tarball"
-        writeFile (out <.> "txt") ""
diff --git a/src/Recipe/General.hs b/src/Recipe/General.hs
deleted file mode 100644
--- a/src/Recipe/General.hs
+++ /dev/null
@@ -1,44 +0,0 @@
-
-module Recipe.General(convertSrc, convert, combine) where
-
-import Recipe.Type
-import Hoogle
-import General.Base
-import General.System
-import System.Console.CmdArgs.Verbosity
-
-txt x = map toLower x <.> "txt"
-hoo x = map toLower x <.> "hoo"
-
-
-convertSrc :: ([Name] -> IO ()) -> [Name] -> Name -> String -> IO ()
-convertSrc make deps x src = do
-    writeFileUtf8 (txt x) src
-    make deps
-    convert (map hoo deps) x (hoo x) src
-
-
----- convert a single database
-convert :: [FilePath] -> Name -> FilePath -> String -> IO ()
-convert deps x out src = do
-    deps2 <- filterM doesFileExist deps
-    when (deps /= deps2) $ putWarning $ "Warning: " ++ x ++ " doesn't know about dependencies on " ++ unwords (deps \\ deps2)
-    dbs <- mapM loadDatabase deps2
-    let (err,db) = createDatabase Haskell dbs src
-    unless (null err) $ outStrLn $ "Skipped " ++ show (length err) ++ " warnings in " ++ x
-    whenLoud $ outStr $ unlines $ map show err
-    outStr $ "Converting " ++ x ++ "... "
-    performGC
-    saveDatabase out db
-    outStrLn "done"
-
-
--- combine multiple databases
-combine :: ([Name] -> IO ()) -> Name -> [Name] -> Bool -> IO ()
-combine make x deps force = do
-    make deps
-    dbs <- mapM (loadDatabase . hoo) deps
-    outStr $ "Creating " ++ x ++ " from " ++ show (length deps) ++ " databases... "
-    performGC
-    saveDatabase (hoo x) $ mconcat dbs
-    outStrLn "done"
diff --git a/src/Recipe/Hackage.hs b/src/Recipe/Hackage.hs
--- a/src/Recipe/Hackage.hs
+++ b/src/Recipe/Hackage.hs
@@ -1,90 +1,19 @@
 
-module Recipe.Hackage(makePlatform, makeDefault, makePackage, makeAll) where
+module Recipe.Hackage(platformPackages, avoid, findLocal) where
 
-import Recipe.Type
-import Recipe.Cabal
-import Recipe.General
-import Recipe.Haddock
 import General.Base
 import General.System
 import General.Util
 import General.Web
-import Control.Exception
+import System.FilePath hiding (combine)
 
 
 -- FIXME: This is a list of hack
 avoid = words "ghc-prim integer integer-simple integer-gmp rts ghc Win32"
 
 
-makePlatform :: ([Name] -> IO ()) -> IO ()
-makePlatform make = do
-    xs <- listPlatform
-    forM_ xs $ \(name,ver) -> do
-        v2 <- version cabals name
-        when (ver /= v2) $ putStrLn $ "Warning: Version mismatch for " ++ name ++ " (platform=" ++ ver ++ ", cabal=" ++ v2 ++ ")"
-    combine make "platform" (map fst xs) False
-
-
-makeAll :: ([Name] -> IO ()) -> IO ()
-makeAll make = do
-    xs <- listing inputs
-    make xs
-
-
--- create a database containing an entry for each package in hackage
-makePackage :: IO ()
-makePackage = do
-    xs <- listing cabals
-    xs <- forM xs $ \name -> do
-        ver <- version cabals name
-        let file = cabals </> name </> ver </> name <.> "cabal"
-        src <- readCabal file
-        return $ case src of
-            Nothing -> []
-            Just src ->
-                [""] ++ zipWith (++) ("-- | " : repeat "--   ") (cabalDescription src) ++
-                ["--","-- Version " ++ ver, "@url package/" ++ name, "@entry package " ++ name]
-    convertSrc noDeps [] "package" $ unlines $
-        "@url http://hackage.haskell.org/" : "@package package" : concat xs
-
-
-makeDefault :: ([Name] -> IO ()) -> [FilePath] -> Name -> IO ()
-makeDefault make local "ghc" = do
-    had <- try $ readFileUtf8' "download/ghc.txt"
-    case had of
-        Left e -> putWarning $ "Warning: Exception when reading haddock for ghc, " ++ show (e :: SomeException)
-        Right had -> do
-	    loc <- findLocal local "ghc"
-            convertSrc make ["base"] "ghc" $ unlines $ "@depends base" : haddockHacks (url loc) (lines had)
-            where url loc = if isNothing loc
-                              then Just "http://www.haskell.org/ghc/docs/latest/html/libraries/ghc/"
-                              else loc
-
-makeDefault make local name = do
-    let base = name == "base"
-    b1 <- doesDirectoryExist $ cabals </> name
-    b2 <- doesDirectoryExist $ inputs </> name
-    if not base && (not b1 || not b2) then
-        putWarning $ "Warning: " ++ name ++ " couldn't find both Cabal and Haddock inputs"
-     else do
-        vc <- version cabals name
-        vh <- if base then return vc else version inputs name
-        when (vc /= vh) $ putStrLn $ "Warning: Version mismatch for " ++ name ++ " (cabal=" ++ vc ++ ", haddock=" ++ vh ++ ")"
-        let had = if base then "download/base.txt" else inputs </> name </> vh </> "doc" </> "html" </> name <.> "txt"
-            cab = cabals </> name </> vc </> name <.> "cabal"
-        had <- try $ readFileUtf8' had
-        case had of
-            Left e -> putWarning $ "Warning: Exception when reading haddock for " ++ name ++ ", " ++ show (e :: SomeException)
-            Right had -> do
-                deps <- fmap (maybe [] cabalDepends) $ readCabal cab
-                let cleanDeps = deps \\ (name:avoid)
-                loc <- findLocal local name
-                convertSrc make cleanDeps name $ unlines $
-                    ["@depends " ++ a | a <- cleanDeps] ++ haddockHacks loc (lines had)
-
-
 -- try and find a local filepath
-findLocal :: [FilePath] -> Name -> IO (Maybe URL)
+findLocal :: [FilePath] -> String -> IO (Maybe URL)
 findLocal paths name = fmap (listToMaybe . concat . concat) $ forM paths $ \p -> do
     xs <- getDirectoryContents p
     xs <- return [p </> x | x <- reverse $ sort xs, name == fst (rbreak (== '-') x)] -- make sure highest version comes first
@@ -98,15 +27,16 @@
 ---------------------------------------------------------------------
 -- READ PLATFORM
 
-listPlatform :: IO [(Name,String)]
-listPlatform = do
-    src <- readFile platform
+platformPackages :: String -> [String]
+platformPackages = map fst . parsePlatform
+
+parsePlatform src =
     let xs = takeWhile (not . isPrefixOf "build-tools:" . ltrim) $
              dropWhile (not . isPrefixOf "build-depends:" . ltrim) $
              lines src
-    return [(name, takeWhile (\x -> x == '.' || isDigit x) $ drop 1 b)
-           | x <- xs, (a,_:b) <- [break (== '=') x], let name = trim $ dropWhile (== '-') $ trim a
-           , not $ avoid name]
+    in [(name, takeWhile (\x -> x == '.' || isDigit x) $ drop 1 b)
+       | x <- xs, (a,_:b) <- [break (== '=') x], let name = trim $ dropWhile (== '-') $ trim a
+       , not $ avoid name]
     where
         avoid x = ("haskell" `isPrefixOf` x && all isDigit (drop 7 x)) ||
                   (x `elem` words "Cabal hpc Win32")
diff --git a/src/Recipe/Keyword.hs b/src/Recipe/Keyword.hs
--- a/src/Recipe/Keyword.hs
+++ b/src/Recipe/Keyword.hs
@@ -1,18 +1,12 @@
 
-module Recipe.Keyword(makeKeyword) where
+module Recipe.Keyword(translateKeywords) where
 
 import General.Base
 import Text.HTML.TagSoup
-import Recipe.Type
-import Recipe.General
 
 
-makeKeyword :: IO ()
-makeKeyword = convertSrc noDeps [] "keyword" . translate =<< readFileUtf8' keywords
-
-
-translate :: String -> String
-translate src = unlines $ keywordPrefix ++ items
+translateKeywords :: String -> String
+translateKeywords src = unlines $ keywordPrefix ++ items
     where items = concatMap keywordFormat $ partitions (~== "<span class='mw-headline' id>") $
                   takeWhile (~/= "<div class=printfooter>") $ parseTags src
 
diff --git a/src/Recipe/Type.hs b/src/Recipe/Type.hs
deleted file mode 100644
--- a/src/Recipe/Type.hs
+++ /dev/null
@@ -1,76 +0,0 @@
-
-module Recipe.Type(
-    CmdLine(..), Name, noDeps, safeEncoding,
-    keywords, platform, cabals, inputs, listing, version,
-    resetWarnings, putWarning, recapWarnings,
-    outStr, outStrLn
-    ) where
-
-import CmdLine.All
-import Control.Concurrent
-import System.IO.Unsafe
-import General.Base
-import General.System
-
-
-type Name = String
-
-
-noDeps :: [Name] -> IO ()
-noDeps [] = return ()
-noDeps xs = error "Internal error: package with no dependencies had dependencies"
-
-
--- | Lots of things go slightly wrong if you use characters > 127 in places, this just replaces them with ?
-safeEncoding :: String -> String
-safeEncoding = map (\x -> if x <= '\0' || x > '\127' then '?' else x)
-
-
----------------------------------------------------------------------
--- DOWNLOADED INFORMATION
-
-keywords = "download/keyword.txt"
-platform = "download/haskell-platform.cabal"
-cabals = "download/hackage-cabal"
-inputs = "download/hackage-hoogle"
-
-listing :: FilePath -> IO [Name]
-listing dir = do
-    xs <- getDirectoryContents dir
-    return $ sortBy (comparing $ map toLower) $ filter (`notElem` [".","..","preferred-versions"]) xs
-
-version :: FilePath -> Name -> IO String
-version dir x = do
-    ys <- getDirectoryContents $ dir </> x
-    when (null ys) $ error $ "Couldn't find version for " ++ x ++ " in " ++ dir
-    let f = map (read :: String -> Int) . words . map (\x -> if x == '.' then ' ' else x)
-    return $ maximumBy (comparing f) $ filter (all (not . isAlpha)) ys
-
-
----------------------------------------------------------------------
--- WARNING MESSAGES
-
-{-# NOINLINE warnings #-}
-warnings :: MVar [String]
-warnings = unsafePerformIO $ newMVar []
-
-putWarning :: String -> IO ()
-putWarning x = do
-    outStrLn x
-    modifyMVar_ warnings $ return . (x:)
-
-recapWarnings :: IO ()
-recapWarnings = do
-    xs <- readMVar warnings
-    mapM_ outStrLn $ reverse xs
-
-resetWarnings :: IO ()
-resetWarnings = modifyMVar_ warnings $ const $ return []
-
-
-outputLock :: MVar ()
-outputLock = unsafePerformIO $ newMVar ()
-
-outStr, outStrLn :: String -> IO ()
-outStr x = withMVar outputLock $ \_ -> do putStr x; hFlush stdout
-outStrLn x = outStr $ x ++ "\n"
diff --git a/src/Recipe/Warning.hs b/src/Recipe/Warning.hs
new file mode 100644
--- /dev/null
+++ b/src/Recipe/Warning.hs
@@ -0,0 +1,25 @@
+
+module Recipe.Warning(
+    resetWarnings, putWarning, recapWarnings
+    ) where
+
+import Control.Concurrent
+import System.IO.Unsafe
+
+
+{-# NOINLINE warnings #-}
+warnings :: MVar [String]
+warnings = unsafePerformIO $ newMVar []
+
+putWarning :: String -> IO ()
+putWarning x = do
+    putStrLn x
+    modifyMVar_ warnings $ return . (x:)
+
+recapWarnings :: IO ()
+recapWarnings = do
+    xs <- readMVar warnings
+    mapM_ putStrLn $ reverse xs
+
+resetWarnings :: IO ()
+resetWarnings = modifyMVar_ warnings $ const $ return []
diff --git a/src/Test.hs b/src/Test.hs
--- a/src/Test.hs
+++ b/src/Test.hs
@@ -6,6 +6,7 @@
 import System.Directory
 import System.Exit
 import System.FilePath
+import System.Environment
 
 
 main :: IO ()
@@ -19,5 +20,9 @@
             res <- system cmd
             when (res /= ExitSuccess) $ error $ "Command: " ++ cmd ++ "\nFailed with: " ++ show res
 
-    hoogle "data"
-    hoogle "test --example"
+    args <- getArgs
+    if "--no-net" `elem` args then
+        hoogle "test"
+     else do
+        hoogle "data"
+        hoogle "test --example"
diff --git a/src/Web/Server.hs b/src/Web/Server.hs
--- a/src/Web/Server.hs
+++ b/src/Web/Server.hs
@@ -2,8 +2,13 @@
 
 module Web.Server(server) where
 
+#ifndef MIN_VERSION_wai
+#define MIN_VERSION_wai(a,b,c) 0
+#endif
+
 import General.Base
 import General.Web
+import System.FilePath
 import CmdLine.All
 import Web.Response
 import Network.HTTP.Types