sproxy2 1.90.2 → 1.91.0
raw patch · 9 files changed
+264/−82 lines, 9 files
Files
- ChangeLog.md +9/−0
- README.md +22/−14
- datafile.yml.example +34/−0
- sproxy.yml.example +38/−24
- sproxy2.cabal +8/−2
- src/Sproxy/Config.hs +2/−0
- src/Sproxy/Server.hs +18/−11
- src/Sproxy/Server/DB.hs +64/−31
- src/Sproxy/Server/DB/DataFile.hs +69/−0
ChangeLog.md view
@@ -1,5 +1,14 @@ For differences with the original Sproxy scroll down. +1.91.0+======++ * In addition to good old PostgreSQL data source, made it possible+ to import permission data from a YAML file. This means that Sproxy2+ can work without any PostgreSQL database, just using file-only configuration.+ Useful for development or trivial deployments. Added new `datafile` option+ in configuration file.+ 1.90.2 ======
README.md view
@@ -55,8 +55,27 @@ Permissions system ------------------+Permissions are stored in internal SQLite3 database and imported+from data sources, which can be a PostgreSQL database or a file. See+[sproxy.sql](./sproxy.sql) and [datafile.yml.example](./datafile.yml.example)+for details. -Permissions are stored in a PostgreSQL database. See sproxy.sql for details.+Do note that Sproxy2 fetches only `group_member`, `group_privilege`+and `privilege_rule` tables, because only these tables are used for+authorization. The other tables in PostgreSQL schema serve for data+integrity. Data integrity of the data file is not verfied, though import+may fail due to primary key restrictions.++Only one data source can be used. The data in internal database, if any,+is fully overwritten by the data from a data source. If no data source is+specified, the data in internal database remains unchanged, even between+restarts. Broken data source is _not_ fatal. Sproxy will keep using existing+internal database, or create a new empty one if missed. Broken data source+means inability to connect to PostgreSQL database, missed datafile, etc.++The data from a PostgreSQL database are periodically fetched into the internal+database, while the data file is read once at startup.+ Here are the main concepts: - A `group` is identified by a name. Every group has@@ -72,14 +91,10 @@ surprising, please see the following example: -Do note that Sproxy2 fetches only `group_member`, `group_privilege` and `privilege_rule`-tables, because only these tables are used for authorization. The other tables-serve for data integrity.- Keep in mind that: -- Domains are converted into lower case (coming from PostgreSQL or HTTP requests).-- Emails are converted into lower case (coming from PostgreSQL or OAuth2 providers).+- Domains are converted into lower case (coming from a data source or HTTP requests).+- Emails are converted into lower case (coming from a data source or OAuth2 providers). - Groups are case-sensitive and treated as is. - HTTP methods are *case-sensitive*. - HTTP query parameters are ignored when matching a request against the rules.@@ -177,13 +192,6 @@ All required Haskell libraries are listed in [sproxy2.cabal](sproxy2.cabal). Use [cabal-install](http://www.haskell.org/haskellwiki/Cabal-Install) to fetch and build all pre-requisites automatically.---Installation-============- $ git clone https://github.com/zalora/sproxy2.git- $ cd sproxy2- $ cabal install Configuration
+ datafile.yml.example view
@@ -0,0 +1,34 @@+--- # Data file. Don't remove this line. This is YAML: https://en.wikipedia.org/wiki/YAML++group_member:+ - group: "devops"+ email: "%"++ - group: "foo"+ email: "%"+++group_privilege:+ - group: "foo"+ domain: "example.com"+ privilege: "full"++ - group: "devops"+ domain: "example.com"+ privilege: "full"+++privilege_rule:+ - domain: "example.com"+ privilege: "full"+ path: "%"+ method: "GET"++ - domain: "example.com"+ privilege: "full"+ path: "%"+ method: "POST"+++... # End of data file. Don't remove this line. This is YAML: https://en.wikipedia.org/wiki/YAML+
sproxy.yml.example view
@@ -1,8 +1,13 @@ --- # Sproxy configuration. Don't remove this line. This is YAML: https://en.wikipedia.org/wiki/YAML +# Logging level: debug, info, warn, error.+# Optional. Default is debug.+#+# log_level: debug+ # The port Sproxy listens on (HTTPS). # Optional. Default is 443.-# +# # listen: 443 # Listen on port 80 and redirect HTTP requests to HTTPS.@@ -24,10 +29,32 @@ # # home: "." ++# File with SSL certificate. Required.+# It can be a bundle with the server certificate coming first:+# cat me-cert.pem CA-cert.pem > cert.pem+# Once again: most wanted certs go first ;-)+# Or you can opt in using of `ssl_cert_chain`+ssl_cert: /path/cert.pem++# File with SSL key (secret!). Required.+ssl_key: /path/key.pem++# Chain SSL certificate files.+# Optional. Default is an empty list+# Example:+# ssl_cert_chain:+# - /path/foo.pem+# - /path/bar.pem+#+# ssl_cert_chain: []++ # PostgreSQL database connection string. # Optional. If specified, sproxy will periodically pull the data from this # database into internal SQLite3 database. Define password in a file # referenced by the PGPASSFILE environment variable. Or use the "pgpassfile" option.+# Cannot be used with the "datafile" option. # Example: # database: "user=sproxy-readonly dbname=sproxy port=6001" #@@ -40,11 +67,17 @@ # # pgpassfile: -# Logging level: debug, info, warn, error.-# Optional. Default is debug.-# -# log_level: debug +# YAML file used to fill internal SQLite3 database.+# Optional. If specified, Sproxy will import it on start overwriting+# and existing data in the internal database.+# Useful for development or some simple deployments.+# Cannot be used with the "database" option.+# For example see the datafile.yml.example+#+# datafile: /path/data.yml++ # A file with arbitrary content used to sign sproxy cookie and other things (secret!). # Optional. If not specified, a random key is generated on startup, and # as a consequence, restaring sproxy will invalidate existing user sessions.@@ -53,25 +86,6 @@ # This should not be very large, a few random bytes are fine. # # key: /run/keys/sproxy.secret--# File with SSL certificate. Required.-# It can be a bundle with the server certificate coming first:-# cat me-cert.pem CA-cert.pem > cert.pem-# Once again: most wanted certs go first ;-)-# Or you can opt in using of `ssl_cert_chain`-ssl_cert: /path/cert.pem--# File with SSL key (secret!). Required.-ssl_key: /path/key.pem--# Chain SSL certificate files.-# Optional. Default is an empty list-# Example:-# ssl_cert_chain:-# - /path/foo.pem-# - /path/bar.pem-# -# ssl_cert_chain: [] # Credentials for supported OAuth2 providers.
sproxy2.cabal view
@@ -1,5 +1,5 @@ name: sproxy2-version: 1.90.2+version: 1.91.0 synopsis: Secure HTTP proxy for authenticating users via OAuth2 description: Sproxy is secure by default. No requests makes it to the backend@@ -13,8 +13,13 @@ copyright: 2016, Zalora South East Asia Pte. Ltd category: Databases, Web build-type: Simple-extra-source-files: README.md ChangeLog.md sproxy.yml.example sproxy.sql cabal-version: >= 1.20+extra-source-files:+ ChangeLog.md+ README.md+ datafile.yml.example+ sproxy.sql+ sproxy.yml.example source-repository head type: git@@ -37,6 +42,7 @@ Sproxy.Logging Sproxy.Server Sproxy.Server.DB+ Sproxy.Server.DB.DataFile build-depends: base >= 4.8 && < 50 , aeson
src/Sproxy/Config.hs view
@@ -27,6 +27,7 @@ , cfListen80 :: Maybe Bool , cfBackends :: [BackendConf] , cfOAuth2 :: HashMap Text OAuth2Conf+, cfDataFile :: Maybe FilePath , cfDatabase :: Maybe String , cfPgPassFile :: Maybe FilePath , cfHTTP2 :: Bool@@ -45,6 +46,7 @@ <*> m .:? "listen80" <*> m .: "backends" <*> m .: "oauth2"+ <*> m .:? "datafile" <*> m .:? "database" <*> m .:? "pgpassfile" <*> m .:? "http2" .!= True
src/Sproxy/Server.hs view
@@ -67,13 +67,8 @@ setGroupID $ userGroupID u setUserID $ userID u - case cfPgPassFile cf of- Nothing -> return ()- Just f -> do- Log.info $ "pgpassfile: " ++ show f- setEnv "PGPASSFILE" f-- db <- DB.start (cfHome cf) (newDataSource cf)+ ds <- newDataSource cf+ db <- DB.start (cfHome cf) ds key <- maybe (Log.info "using new random key" >> getEntropy 32)@@ -112,11 +107,23 @@ (sproxy key db oauth2clients backends) -newDataSource :: ConfigFile -> Maybe DB.DataSource+newDataSource :: ConfigFile -> IO (Maybe DB.DataSource) newDataSource cf =- case cfDatabase cf of- Just str -> Just $ DB.PostgreSQL str- Nothing -> Nothing+ case (cfDataFile cf, cfDatabase cf) of+ (Nothing, Just str) -> do+ case cfPgPassFile cf of+ Nothing -> return ()+ Just f -> do+ Log.info $ "pgpassfile: " ++ show f+ setEnv "PGPASSFILE" f+ return . Just $ DB.PostgreSQL str++ (Just f, Nothing) -> return . Just $ DB.File f++ (Nothing, Nothing) -> return Nothing+ _ -> do+ Log.error "only one data source can be used"+ exitFailure newOAuth2Client :: (Text, OAuth2Conf) -> IO (Text, OAuth2Client)
src/Sproxy/Server/DB.hs view
@@ -14,17 +14,20 @@ import Data.ByteString.Char8 (pack) import Data.Pool (Pool, createPool, withResource) import Data.Text (Text, toLower, unpack)+import Data.Yaml (decodeFileEither) import Database.SQLite.Simple (NamedParam((:=))) import Text.InterpolatedString.Perl6 (q, qc) import qualified Database.PostgreSQL.Simple as PG import qualified Database.SQLite.Simple as SQLite +import Sproxy.Server.DB.DataFile ( DataFile(..), GroupMember(..),+ GroupPrivilege(..), PrivilegeRule(..) ) import qualified Sproxy.Logging as Log type Database = Pool SQLite.Connection -data DataSource = PostgreSQL String -- | File FilePath+data DataSource = PostgreSQL String | File FilePath {- TODO: - Hash remote tables and update the local only when the remote change@@ -87,9 +90,26 @@ createGroupPrivilege c createPrivilegeRule c --- XXX We keep only required minimum of the data, without any integrity check.--- XXX Integrity check should be done somewhere else, e. g. in the master PostgreSQL database,--- XXX or during importing the config file.+populate db (Just (File f)) = do+ Log.info $ "db: reading " ++ show f+ r <- decodeFileEither f+ case r of+ Left e -> Log.error $ f ++ ": " ++ show e+ Right df ->+ withResource db $ \c -> SQLite.withTransaction c $ do+ refreshGroupMembers c $ \st ->+ mapM_ (\gm -> submit st (gmGroup gm, toLower $ gmEmail gm)+ ) (groupMember df)++ refreshGroupPrivileges c $ \st ->+ mapM_ (\gp -> submit st (gpGroup gp, toLower $ gpDomain gp, gpPrivilege gp)+ ) (groupPrivilege df)++ refreshPrivilegeRule c $ \st ->+ mapM_ (\pr -> submit st (toLower $ prDomain pr, prPrivilege pr, prPath pr, prMethod pr)+ ) (privilegeRule df)++ populate db (Just (PostgreSQL connstr)) = void . forkIO . forever . flip finally (7 `minutes` threadDelay) . logException $ do@@ -99,39 +119,34 @@ \pg -> PG.withTransaction pg $ do Log.info "db: syncing group_member"- dropGroupMember c- createGroupMember c- PG.forEach_ pg- [q|SELECT "group", lower(email) FROM group_member|] $ \r ->- SQLite.execute c- [q|INSERT INTO group_member("group", email) VALUES (?, ?)|]- (r :: (Text, Text))+ refreshGroupMembers c $ \st ->+ PG.forEach_ pg+ [q|SELECT "group", lower(email) FROM group_member|] $ \r ->+ submit st (r :: (Text, Text)) count c "group_member" Log.info "db: syncing group_privilege"- dropGroupPrivilege c- createGroupPrivilege c- PG.forEach_ pg- [q|SELECT "group", lower(domain), privilege FROM group_privilege|] $ \r ->- SQLite.execute c- [q|INSERT INTO group_privilege("group", domain, privilege) VALUES (?, ?, ?)|]- (r :: (Text, Text, Text))+ refreshGroupPrivileges c $ \st ->+ PG.forEach_ pg+ [q|SELECT "group", lower(domain), privilege FROM group_privilege|] $ \r ->+ submit st (r :: (Text, Text, Text)) count c "group_privilege" Log.info "db: syncing privilege_rule"- dropPrivilegeRule c- createPrivilegeRule c- PG.forEach_ pg- [q|SELECT lower(domain), privilege, path, method FROM privilege_rule|] $ \r ->- SQLite.execute c- [q|INSERT INTO privilege_rule(domain, privilege, path, method) VALUES (?, ?, ?, ?)|]- (r :: (Text, Text, Text, Text))+ refreshPrivilegeRule c $ \st ->+ PG.forEach_ pg+ [q|SELECT lower(domain), privilege, path, method FROM privilege_rule|] $ \r ->+ submit st (r :: (Text, Text, Text, Text)) count c "privilege_rule" -dropGroupMember :: SQLite.Connection -> IO ()-dropGroupMember c = SQLite.execute_ c "DROP TABLE IF EXISTS group_member"+-- FIXME short-cut for https://github.com/nurpax/sqlite-simple/issues/50+-- FIXME nextRow is the only way to execute a prepared statement+-- FIXME with bound parameters, but we don't expect any results.+submit :: SQLite.ToRow values => SQLite.Statement -> values -> IO ()+submit st v = SQLite.withBind st v $ void (SQLite.nextRow st :: IO (Maybe [Int])) + createGroupMember :: SQLite.Connection -> IO () createGroupMember c = SQLite.execute_ c [q| CREATE TABLE IF NOT EXISTS group_member (@@ -141,9 +156,14 @@ ) |] +refreshGroupMembers :: SQLite.Connection -> (SQLite.Statement -> IO ()) -> IO ()+refreshGroupMembers c a = do+ SQLite.execute_ c "DROP TABLE IF EXISTS group_member"+ createGroupMember c+ SQLite.withStatement c+ [q|INSERT INTO group_member("group", email) VALUES (?, ?)|]+ a -dropGroupPrivilege :: SQLite.Connection -> IO ()-dropGroupPrivilege c = SQLite.execute_ c "DROP TABLE IF EXISTS group_privilege" createGroupPrivilege :: SQLite.Connection -> IO () createGroupPrivilege c = SQLite.execute_ c [q|@@ -155,9 +175,14 @@ ) |] +refreshGroupPrivileges :: SQLite.Connection -> (SQLite.Statement -> IO ()) -> IO ()+refreshGroupPrivileges c a = do+ SQLite.execute_ c "DROP TABLE IF EXISTS group_privilege"+ createGroupPrivilege c+ SQLite.withStatement c+ [q|INSERT INTO group_privilege("group", domain, privilege) VALUES (?, ?, ?)|]+ a -dropPrivilegeRule :: SQLite.Connection -> IO ()-dropPrivilegeRule c = SQLite.execute_ c "DROP TABLE IF EXISTS privilege_rule" createPrivilegeRule :: SQLite.Connection -> IO () createPrivilegeRule c = SQLite.execute_ c [q|@@ -169,6 +194,14 @@ PRIMARY KEY (domain, path, method) ) |]++refreshPrivilegeRule :: SQLite.Connection -> (SQLite.Statement -> IO ()) -> IO ()+refreshPrivilegeRule c a = do+ SQLite.execute_ c "DROP TABLE IF EXISTS privilege_rule"+ createPrivilegeRule c+ SQLite.withStatement c+ [q|INSERT INTO privilege_rule(domain, privilege, path, method) VALUES (?, ?, ?, ?)|]+ a count :: SQLite.Connection -> String -> IO ()
+ src/Sproxy/Server/DB/DataFile.hs view
@@ -0,0 +1,69 @@+{-# LANGUAGE OverloadedStrings #-}+module Sproxy.Server.DB.DataFile (+ DataFile(..)+, GroupMember(..)+, GroupPrivilege(..)+, PrivilegeRule(..)+) where++import Control.Applicative (empty)+import Data.Aeson (FromJSON, parseJSON)+import Data.Text (Text)+import Data.Yaml (Value(Object), (.:))+++data DataFile = DataFile {+ groupMember :: [GroupMember]+, groupPrivilege :: [GroupPrivilege]+, privilegeRule :: [PrivilegeRule]+} deriving (Show)++instance FromJSON DataFile where+ parseJSON (Object m) = DataFile <$>+ m .: "group_member"+ <*> m .: "group_privilege"+ <*> m .: "privilege_rule"+ parseJSON _ = empty+++data GroupMember = GroupMember {+ gmGroup :: Text+, gmEmail :: Text+} deriving (Show)++instance FromJSON GroupMember where+ parseJSON (Object m) = GroupMember <$>+ m .: "group"+ <*> m .: "email"+ parseJSON _ = empty+++data GroupPrivilege = GroupPrivilege {+ gpGroup :: Text+, gpDomain :: Text+, gpPrivilege :: Text+} deriving (Show)++instance FromJSON GroupPrivilege where+ parseJSON (Object m) = GroupPrivilege <$>+ m .: "group"+ <*> m .: "domain"+ <*> m .: "privilege"+ parseJSON _ = empty+++data PrivilegeRule = PrivilegeRule {+ prDomain :: Text+, prPrivilege :: Text+, prPath :: Text+, prMethod :: Text+} deriving (Show)++instance FromJSON PrivilegeRule where+ parseJSON (Object m) = PrivilegeRule <$>+ m .: "domain"+ <*> m .: "privilege"+ <*> m .: "path"+ <*> m .: "method"+ parseJSON _ = empty+