keter 1.3.6 → 1.3.7
raw patch · 7 files changed
+428/−35 lines, 7 filesdep ~basedep ~wai-extra
Dependency ranges changed: base, wai-extra
Files
- ChangeLog.md +63/−0
- Keter/Proxy.hs +2/−1
- Keter/Types/Middleware.hs +100/−0
- Keter/Types/V10.hs +5/−0
- README.md +252/−0
- changelog.md +0/−29
- keter.cabal +6/−5
+ ChangeLog.md view
@@ -0,0 +1,63 @@+## 1.3.7++* Add ability to use middleware [#63](https://github.com/snoyberg/keter/pulls/63)++## 1.3.6++Support the `forward-env` setting.++## 1.3.5.3++More correct/complete solution for issue #44. Allows looking up hosts either with or without port numbers.++## 1.3.5.2++Partial workaround for keter.yaml files that give a port with the hostname.++## 1.3.5.1++Fix bug where the cleanup process would remain running.++## 1.3.5++All stanzas may have the `requires-secure` property to force redirect to HTTPS. You can set additional environment variables in your global Keter config file.++## 1.3.4++Support for overriding external ports. Support for keter.yml in addition to keter.yaml. Case insensitive hostname lookups.++## 1.3.3++Set the X-Forwarded-Proto header++## 1.3.2++Enable GZIP middleware++## 1.3.1++Upgrade to WAI 3.0++## 1.3.0++Upgrade to conduit 1.1++## 1.0.1++Permit use of wildcard subdomains and exceptions to wildcards. Convert internal strings to use Data.Text in more places. (Although internationalized domain names are not supported unless entered in punycode in configuration files.)++## 1.0.0++Significant overhaul. We now support monitoring of much more arbitrary jobs (e.g., background tasks), have a proper plugin system (PostgreSQL is no longer a required component), and have a much better system for tracking hostname mapping changes.++## 0.4.0++Switch to fsnotify to get cross-platform support. No longer using raw proxies, but instead WAI proxies.++## 0.3.7++Sending a HUP signal reloads the list of deployed apps. This is useful for circumstances where inotify does not work correctly, such as on file systems which do not support it.++## 0.3.5++You can now create Keter bundles without any applications. These can contain static hosts and redirects.
Keter/Proxy.hs view
@@ -19,6 +19,7 @@ import qualified Data.Vector as V import qualified Filesystem.Path.CurrentOS as F import Keter.Types+import Keter.Types.Middleware import Network.HTTP.Conduit (Manager) import Network.HTTP.ReverseProxy (ProxyDest (ProxyDest), SetIpHeader (..),@@ -130,7 +131,7 @@ : Wai.requestHeaders req } performAction _ (PAStatic StaticFilesConfig {..}) = do- return $ WPRApplication $ staticApp (defaultFileServerSettings sfconfigRoot)+ return $ WPRApplication $ processMiddleware sfconfigMiddleware $ staticApp (defaultFileServerSettings sfconfigRoot) { ssListing = if sfconfigListings then Just defaultListing
+ Keter/Types/Middleware.hs view
@@ -0,0 +1,100 @@+{-# LANGUAGE OverloadedStrings, DeriveGeneric #-}++module Keter.Types.Middleware where++import Data.Aeson+import GHC.Generics+import Prelude+import Network.Wai++import Control.Monad+import Control.Arrow ((***))+import Control.Applicative++-- various Middlewares+import Network.Wai.Middleware.AcceptOverride (acceptOverride)+import Network.Wai.Middleware.Autohead (autohead)+import Network.Wai.Middleware.Jsonp (jsonp)+import Network.Wai.Middleware.Local (local)+import Network.Wai.Middleware.AddHeaders (addHeaders)+import Network.Wai.Middleware.MethodOverride (methodOverride)+import Network.Wai.Middleware.MethodOverridePost (methodOverridePost)+import Network.Wai.Middleware.HttpAuth (basicAuth)++import Data.ByteString.Lazy as L (ByteString)+import Data.ByteString as S (ByteString)++import Data.Text.Lazy.Encoding as TL (encodeUtf8, decodeUtf8)+import Data.Text.Encoding as T (encodeUtf8, decodeUtf8)+import Data.String (fromString)+import qualified Data.HashMap.Strict as H++data MiddlewareConfig = AcceptOverride+ | Autohead+ | Jsonp+ | MethodOverride+ | MethodOverridePost+ | AddHeaders ![(S.ByteString, S.ByteString)]+ | BasicAuth !String ![(S.ByteString, S.ByteString)]+ -- ^ Realm [(username,password)]+ | Local !Int !L.ByteString+ -- ^ Status Message+ deriving (Show,Generic)++instance FromJSON MiddlewareConfig where+ parseJSON (String "accept-override" ) = pure AcceptOverride+ parseJSON (String "autohead" ) = pure Autohead+ parseJSON (String "jsonp" ) = pure Jsonp+ parseJSON (String "method-override" ) = pure MethodOverride+ parseJSON (String "method-override-post") = pure MethodOverridePost+ parseJSON (Object o) =+ case H.toList o of+ [("basic-auth", Object ( o'))] -> BasicAuth <$> o' .:? "realm" .!= "keter"+ <*> (map (T.encodeUtf8 *** T.encodeUtf8) . H.toList <$> o' .:? "creds" .!= H.empty)+ [("headers" , Object _ )] -> AddHeaders . map (T.encodeUtf8 *** T.encodeUtf8) . H.toList <$> o .:? "headers" .!= H.empty+ [("local" , Object o')] -> Local <$> o' .:? "status" .!= 401+ <*> (TL.encodeUtf8 <$> o' .:? "message" .!= "Unauthorized Accessing from Localhost ONLY" )+ _ -> mzero -- fail "Rule: unexpected format"+ parseJSON _ = mzero++instance ToJSON MiddlewareConfig where+ toJSON AcceptOverride = "accept-override"+ toJSON Autohead = "autohead"+ toJSON Jsonp = "jsonp"+ toJSON MethodOverride = "method-override"+ toJSON MethodOverridePost = "method-override-post"+ toJSON (BasicAuth realm cred) = object [ "basic-auth" .= object [ "realm" .= realm+ , "creds" .= object ( map ( T.decodeUtf8 *** (String . T.decodeUtf8)) cred )+ ]+ ]+ toJSON (AddHeaders headers) = object [ "headers" .= object ( map (T.decodeUtf8 *** String . T.decodeUtf8) headers) ]+ toJSON (Local sc msg) = object [ "local" .= object [ "status" .= sc+ , "message" .= TL.decodeUtf8 msg + ]+ ]+++{-- Still missing+-- CleanPath+-- Gzip+-- RequestLogger+-- Rewrite+-- Vhost+--}++processMiddleware :: [MiddlewareConfig] -> Middleware+processMiddleware = composeMiddleware . map toMiddleware++toMiddleware :: MiddlewareConfig -> Middleware+toMiddleware AcceptOverride = acceptOverride+toMiddleware Autohead = autohead+toMiddleware Jsonp = jsonp+toMiddleware (Local s c ) = local ( responseLBS (toEnum s) [] c )+toMiddleware MethodOverride = methodOverride+toMiddleware MethodOverridePost = methodOverridePost+toMiddleware (BasicAuth realm cred) = basicAuth (\u p -> return $ maybe False (==p) $ lookup u cred ) (fromString realm)+toMiddleware (AddHeaders headers) = addHeaders headers++-- composeMiddleware :+composeMiddleware :: [Middleware] -> Middleware+composeMiddleware = foldl (flip (.)) id
Keter/Types/V10.hs view
@@ -11,6 +11,7 @@ import qualified Data.CaseInsensitive as CI import Keter.Types.Common import qualified Keter.Types.V04 as V04+import Keter.Types.Middleware import Data.Yaml.FilePath import Data.Aeson (FromJSON (..), (.:), (.:?), Value (Object, String), withObject, (.!=)) import Control.Applicative ((<$>), (<*>), (<|>))@@ -219,6 +220,7 @@ , sfconfigHosts :: !(Set Host) , sfconfigListings :: !Bool -- FIXME basic auth+ , sfconfigMiddleware :: ![ MiddlewareConfig ] } deriving Show @@ -228,6 +230,7 @@ { sfconfigRoot = root , sfconfigHosts = Set.singleton $ CI.mk host , sfconfigListings = True+ , sfconfigMiddleware = [] } instance ParseYamlFile StaticFilesConfig where@@ -235,12 +238,14 @@ <$> lookupBase basedir o "root" <*> (Set.map CI.mk <$> ((o .: "hosts" <|> (Set.singleton <$> (o .: "host"))))) <*> o .:? "directory-listing" .!= False+ <*> o .:? "middleware" .!= [] instance ToJSON StaticFilesConfig where toJSON StaticFilesConfig {..} = object [ "root" .= F.encodeString sfconfigRoot , "hosts" .= Set.map CI.original sfconfigHosts , "directory-listing" .= sfconfigListings+ , "middleware" .= sfconfigMiddleware ] data RedirectConfig = RedirectConfig
+ README.md view
@@ -0,0 +1,252 @@+Deployment system for web applications, originally intended for hosting Yesod+applications. Keter does the following actions for your application:++* Binds to the main port (usually port 80) and reverse proxies requests to your application based on virtual hostnames.+* Provides SSL support if requested.+* Automatically launches applications, monitors processes, and relaunches any processes which die.+* Provides graceful redeployment support, by launching a second copy of your application, performing a health check, and then switching reverse proxying to the new process.+* Management of log files.++Keter provides many more advanced features and extension points. It allows+configuration of static hosts, redirect rules, management of PostgreSQL+databases, and more. It supports a simple bundle format for applications which+allows for easy management of your web apps.++## Quick Start++Do get Keter up-and-running quickly on an Ubuntu system, run:++ wget -O - https://raw.githubusercontent.com/snoyberg/keter/master/setup-keter.sh | bash++(Note: you may need to run the above command twice, if the shell exits after+`apt-get` but before running the rest of its instructions.) This will download+and build Keter from source and get it running with a+default configuration. By default Keter will be set up to also support HTTPS, and+will require you to provide a key and certificate in `/opt/keter/etc`. You can+disable HTTPS in `/opt/keter/etc/keter-config.yaml` by commenting the certificate+and key lines.++_This approach is not recommended for a production system_. We do not recommend+installing a full GHC toolchain on a production server, nor running such ad-hoc+scripts. This is intended to provide a quick way to play with Keter, especially+for temporary virtual machines. For a production system, we recommend building+the `keter` binary on a separate system, and tracking it via a package manager+or similar strategy.++## Bundling your app for Keter++1. Modify your web app to check for the `PORT` environment variable, and have+ it listen for incoming HTTP requests on that port. Keter automatically+ assigns arbitrary ports to each web app it manages. The Yesod scaffold+ site is already equipped to read the `PORT` environment variable when+ it is set.++2. Create a file `config/keter.yaml`. The minimal file just has two settings:++ ```yaml+ exec: ../path/to/executable+ host: mydomainname.example.com+ ```++ See the bundles section below for more available settings.++3. Create a gzipped tarball with the `config/keter.yaml` file, your+ executable, and any other static resources you would like available to your+ application. This file should be given a `.keter` file extension, e.g.+ `myapp.keter`.++4. Copy the `.keter` file to `/opt/keter/incoming`. Keter will monitor this+ directory for file updates, and automatically redeploy new versions of your+ bundle.++## Setup++Instructions are for an Ubuntu system. Eventually, I hope to provide a PPA for+this (please contact me if you would like to assist with this). For now, the+following steps should be sufficient:++First, install PostgreSQL++ sudo apt-get install postgresql++Second, build the `keter` binary and place it at `/opt/keter/bin`. To do so,+you'll need to install the Haskell Platform, and can then build with `cabal`.+This would look something like:++ sudo apt-get install haskell-platform+ cabal update+ cabal install keter+ sudo mkdir -p /opt/keter/bin+ sudo cp ~/.cabal/bin/keter /opt/keter/bin++Third, create a Keter config file. You can view a sample at+https://github.com/snoyberg/keter/blob/master/etc/keter-config.yaml.++Fourth, set up an Upstart job to start `keter` when your system boots.++```+# /etc/init/keter.conf+start on (net-device-up and local-filesystems and runlevel [2345])+stop on runlevel [016]+respawn++console none++exec /opt/keter/bin/keter /opt/keter/etc/keter-config.yaml+```++Finally, start the job for the first time:++ sudo start keter++Optionally, you may wish to change the owner on the `/opt/keter/incoming`+folder to your user account, so that you can deploy without `sudo`ing.++ sudo mkdir -p /opt/keter/incoming+ sudo chown $USER /opt/keter/incoming++## Bundles++An application needs to be set up as a keter bundle. This is a GZIPed tarball+with a `.keter` filename extension and which has one special file:+`config/keter.yaml`. A sample file is available at+https://github.com/snoyberg/keter/blob/master/incoming/foo1_0/config/keter.yaml.++Keter as well supports wildcard subdomains and exceptions, as in this example+configuration:++```yaml+exec: ../com.example.app+args:+ - Hello+ - World+ - 1+host: www.example.com+extra-hosts:+ - "*.example.com"+ - foo.bar.example.com+static-hosts:+ - host: static.example.com+ root: ../static+redirects:+ - from: example.com+ to: www.example.com+```++Due to YAML parsing, wildcard hostnames will need to be quoted as above.+Wildcard hostnames are not recursive, so `foo.bar.example.com` must be+explicitly added as an extra hostname in the above example, or+alternatively, `*.*.example.com` would cover all host names two levels+deep. It would not cover host names only one level deep, such as+`qux.example.com`. In this manner, wildcard hostnames correspond to the+manner in which SSL certificates are handled per RFC2818. Wildcards may+be used in only one level of a hostname, as in `foo.*.example.com`.++Full RFC2818 compliance is not present - `f*.example.com` will not be+handled as a wildcard with a prefix.++A sample Bash script for producing a Keter bundle is:++```bash+#!/bin/bash -ex++cabal build+strip dist/build/yesodweb/yesodweb+rm -rf static/tmp+tar czfv yesodweb.keter dist/build/yesodweb/yesodweb config static+```++For users of Yesod, The `yesod` executable provides a `keter` command for+creating the bundle, and the scaffolded site provides a `keter.yaml` file.++## Deploying++In order to deploy, you simply copy the keter bundle to `/opt/keter/incoming`.+To update an app, copy in the new version. The old process will only be+terminated after the new process has started answering requests. To stop an+application, delete the file from incoming.++## PostgreSQL support++Keter ships by default with a PostgreSQL plugin, which will handle management of PostgreSQL databases for your application. To use this, make the following changes:++* Add the following lines to your `config/keter.yaml` file:++```yaml+plugins:+ postgres: true+```++(Note: The `plugins` configuration option was added in v1.0 of the+keter configuration syntax. If you are using v0.4 then use `postgres: true`.)++* Modify your application to get its database connection settings from the following environment variables:+ * `PGHOST`+ * `PGPORT`+ * `PGUSER`+ * `PGPASS`+ * `PGDATABASE`++* The Yesod scaffold site is already equipped to read these environment+ variables when they are set.++## Known issues++* There are reports of Keter not working behind an nginx reverse proxy. From+ the reports, this appears to be a limitation in nginx's implementation, not a+ problem with Keter. Keter works fine behind other reverse proxies, including+ Apache and Amazon ELB.++ One possible workaround is to add the following lines to your nginx configuration:++ proxy_set_header Connection "";+ proxy_http_version 1.1;++ This has not yet been confirmed to work in production. If you use this,+ please report either its success or failure back to me.++* Keter does not handle password-protected SSL key files well. When provided+ with such a key file, unlike Apache and Nginx, Keter will not pause to ask+ for the password. Instead, your https connections will merely stall.++ To get around this, you need to create a copy of the key without password+ and deploy this new key:++ openssl rsa -in original.key -out new.key++ (Back up the original key first, just in case.)++## Stanza-based config files++Starting with Keter 1.0, there is an alternate format for application Keter+config files, which allows much more flexibility in defining multiple+functionality for a single bundle (e.g., more than one web app, multiple+redirects, etc). This README will eventually be updated to reflect all various+options. In the meanwhile, please see the following examples of how to use this+file format:++* https://github.com/yesodweb/yesod-scaffold/blob/postgres/config/keter.yml+* https://github.com/snoyberg/keter/blob/master/incoming/foo1_0/config/keter.yaml++## Multiple SSL Certificates+Keter is able to serve different certificates for different hosts, allowing for the deployment of distinct domains using the same server. An example `keter-config.yaml` would look like::+```+root: ..+listeners:+ - host: "*4" # Listen on all IPv4 hosts+ port: 80+ - host: 127.0.0.1+ key: key.pem+ certificate: certificate1.pem+ - host: 127.0.0.2+ key: key.pem+ certificate: certificate2.pem+```+## FAQ+* Keter spawns multiple failing process when run with `sudo start keter`.+ * This may be due to Keter being unable to find the SSL certificate and key.+ Try to run `sudo /opt/keter/bin/keter /opt/keter/etc/keter-config.yaml`.+ If it fails with `keter: etc/certificate.pem: openBinaryFile: does not exist`+ or something like it, you may need to provide valid SSL certificates and keys+ or disable HTTPS, by uncommenting the key and certificate lines from + `/opt/keter/etc/keter-config.yaml`.
− changelog.md
@@ -1,29 +0,0 @@-__1.3.6__ Support the `forward-env` setting.--__1.3.5.3__ More correct/complete solution for issue #44. Allows looking up hosts either with or without port numbers.--__1.3.5.2__ Partial workaround for keter.yaml files that give a port with the hostname.--__1.3.5.1__ Fix bug where the cleanup process would remain running.--__1.3.5__ All stanzas may have the `requires-secure` property to force redirect to HTTPS. You can set additional environment variables in your global Keter config file.--__1.3.4__ Support for overriding external ports. Support for keter.yml in addition to keter.yaml. Case insensitive hostname lookups.--__1.3.3__ Set the X-Forwarded-Proto header--__1.3.2__ Enable GZIP middleware--__1.3.1__ Upgrade to WAI 3.0--__1.3.0__ Upgrade to conduit 1.1--__1.0.1__ Permit use of wildcard subdomains and exceptions to wildcards. Convert internal strings to use Data.Text in more places. (Although internationalized domain names are not supported unless entered in punycode in configuration files.)--__1.0.0__ Significant overhaul. We now support monitoring of much more arbitrary jobs (e.g., background tasks), have a proper plugin system (PostgreSQL is no longer a required component), and have a much better system for tracking hostname mapping changes.--__0.4.0__ Switch to fsnotify to get cross-platform support. No longer using raw proxies, but instead WAI proxies.--__0.3.7__ Sending a HUP signal reloads the list of deployed apps. This is useful for circumstances where inotify does not work correctly, such as on file systems which do not support it.--__0.3.5__ You can now create Keter bundles without any applications. These can contain static hosts and redirects.
keter.cabal view
@@ -1,8 +1,7 @@ Name: keter-Version: 1.3.6+Version: 1.3.7 Synopsis: Web application deployment manager, focusing on Haskell web frameworks-Description:- Handles deployment of web apps, providing a reverse proxy to achieve zero downtime deployments. For more information, please see the README on Github: <https://github.com/snoyberg/keter#readme>+Description: Hackage documentation generation is not reliable. For up to date documentation, please see: <http://www.stackage.org/package/keter>. Homepage: http://www.yesodweb.com/ License: MIT License-file: LICENSE@@ -11,7 +10,8 @@ Category: Web, Yesod Build-type: Simple Cabal-version: >=1.8-extra-source-files: changelog.md+extra-source-files: ChangeLog.md+ README.md --Data-Files: incoming/foo/bundle.sh, incoming/foo/config/keter.yaml @@ -44,7 +44,7 @@ , unix >= 2.5 , wai-app-static >= 3.0 && < 3.1 , wai >= 3.0 && < 3.1- , wai-extra >= 3.0 && < 3.1+ , wai-extra >= 3.0.3 && < 3.1 , http-types , regex-tdfa >= 1.1 , attoparsec >= 0.10@@ -66,6 +66,7 @@ Keter.Types.V04 Keter.Types.V10 Keter.Types.Common+ Keter.Types.Middleware Keter.App Keter.AppManager Keter.LabelMap