diff --git a/changelog b/changelog new file mode 100644 --- /dev/null +++ b/changelog @@ -0,0 +1,30 @@ +-*-change-log-*- + +0.1.0.0 Chris Dornan 2014-03-30 + * First public release + +0.1.1.0 Chris Dornan 2014-06-08 + * Fix read-only behaviour, adding --read-only flag + +0.2.0.0 Chris Dornan 2014-07-06 + * Reorganise module hierarchy and code base + * Add Data.KeyStore.Sections + * Replace 'psd' example with 'deploy' example + +0.2.0.1 Chris Dornan 2014-07-07 + * Fix 'deploy' example + +0.3.0.0 Chris Dornan 2014-07-23 + * Fix CLI organization + * Add hostRSection to Sections class + +0.3.0.1 Adam Gundry 2014-07-23 + * Add GHC 0.7.8.3 compatability + +0.4.0.0 Chris Dornan 2014-07-26 + * Remove hostRSection from Sections class! + * Reorganise Sections simplifying and fixing host/section relationship + * Fix 'deploy' example to work with new sections + * Write Haddock annotations for Sections module. + * Tidy cabal file + * Revise readme and cabal documentation diff --git a/examples/deploy/Deploy/HostSectionKey.hs b/examples/deploy/Deploy/HostSectionKey.hs --- a/examples/deploy/Deploy/HostSectionKey.hs +++ b/examples/deploy/Deploy/HostSectionKey.hs @@ -65,22 +65,21 @@ encode = drop 2 . show instance Sections HostID SectionID KeyID where - hostSection = host_section - hostRSection = host_resident_section - sectionType = section_type - superSections = super_sections - keyIsHostIndexed = key_is_host_indexed - keyIsInSection = key_is_in_section - getKeyData = get_key_data - sectionSettings = section_settings - describeKey = describe_key - describeSection = describe_section + hostDeploySection = host_deploy_section + sectionType = section_type + superSections = super_sections + keyIsHostIndexed = key_is_host_indexed + keyIsInSection = key_is_in_section + getKeyData = get_key_data + sectionSettings = section_settings + describeKey = describe_key + describeSection = describe_section sections :: SECTIONS HostID SectionID KeyID sections = SECTIONS -host_section :: HostID -> SectionID -host_section h = +host_deploy_section :: HostID -> SectionID +host_deploy_section h = case h of H_live_eu -> S_eu_admin H_staging_eu -> S_eu_staging @@ -88,15 +87,6 @@ H_staging_us -> S_us_staging H_dev -> S_dev -host_resident_section :: HostID -> SectionID -host_resident_section h = - case h of - H_live_eu -> S_eu_deploy - H_staging_eu -> S_eu_staging - H_live_us -> S_us_deploy - H_staging_us -> S_us_staging - H_dev -> S_dev - section_type :: SectionID -> SectionType section_type s = case s of @@ -145,7 +135,7 @@ return KeyData { kd_identity = Identity $ T.pack $ mk "id" - , kd_comment = Comment $ T.pack $ mk "id" + , kd_comment = Comment $ T.pack $ mk "comment" , kd_secret = B.pack $ mk "secret" } where diff --git a/examples/deploy/deploy.hs b/examples/deploy/deploy.hs --- a/examples/deploy/deploy.hs +++ b/examples/deploy/deploy.hs @@ -75,7 +75,7 @@ key_prededicate = keyPrededicate sample_ln :: SectionID -> IO () -sample_ln s = putStrLn $ "export " ++ "KEY_pw_" ++ s_ ++ "=pw_" ++ s_ +sample_ln s = putStrLn $ "export " ++ "KEY_pw_" ++ s_ ++ "=pw_" ++ s_ ++ ";" where s_ = encode s diff --git a/keystore.cabal b/keystore.cabal --- a/keystore.cabal +++ b/keystore.cabal @@ -1,5 +1,5 @@ Name: keystore -Version: 0.3.0.1 +Version: 0.4.0.0 Synopsis: Managing stores of secret things Homepage: http://github.com/cdornan/keystore Author: Chris Dornan @@ -10,11 +10,114 @@ Category: Cryptography Build-type: Simple Description: - Provides a program, an IO-based API and its underlying functional API for - managing a multi-level JSON-encoded store of encrypted and hashed symmetric - and public keypairs and associated utilities for encrypting and signing - files. + Writing deployment scripts is a critical yet error-prone activity which we + would rather do in Haskell. One of the most difficult aspect of deployment + scripts is the management of credentials: they cannot be stored in the + VCS like almost everything else, but need to be organised and accessed + while under lock and key. This is the problem that keystore is trying to solve: + flexible, secure and well-typed deployment scripts. + . + /All Haskell/ + . + This package is written purely in Hakell and all of the cryptographic packages + it relies upon are written in Haskell. + . + /JSON Format/ + . + It stores everything in a JSON format that has proven to be stable. We can can + use in future + should the store need to be reorganized. + . + /Simple and Flexible Underlying Model/ + . + * /Named Keys/: every key has an name within the store that is associated + with some secret data. If the secret data for that key is to be stored then + it must identify another key in the store that will be used to encrypt the + data. (Some keys -- the passwords -- will typically be auto-loaded from + environment variables.) + * *Functional model*: keys can be deleted and added again but the design + encourages the retention of the history. The old keys remain available + but deployment scripts will naturally select the latest version of a key. + When a key is rotated this merely loads a new generation for the rotated + key. + . + * /Simple Metadata/: oher information, such as the identity of the key + with its originating system (e.g., the identifier of an AWS IAM key) + and some arbitrary textual information (the 'comment') may be associated + with a key and accessible without recourse to the key or password needed + to access the secret information. + . + * /PKS/: the seret may be a RSA provate key with the public key stored + separately in the cler. + * *MFA*: a secret may be protected with multiple named keys, all of which + will be needed to recover the secret text. + . + * /Hashing/: all keys can be hashed with an appropriate PBKDF-2 function + and the hashes stored in the clear. These hashes may be sued to verify + passwords but also can be inserted directly into configuration files + for deployment. Precise control of the PBKDF-2 hash paramers is + avaiable. + . + * /Hierarchical Organization/: keys can be stored in different sections + with each key being protected by a master key for that section. Sections + can be configured to store the master keys of other sections thereby + gaining acces to all of the keys in those sections and the keys they + have access to. + . + * /Systems Integration/: keys can automatically loaded from Environment + variables. Typically a keystore session will start by settingb up an + environment variable for the deployment section corresponding for + the node that you need to deploy to. This will provide access to + precisely the keys whose secrets you need to carry out the deployment + and no more. It only needs access to the hashes of admin keys then they + can be placed in separate higher-level @admin@ sections. Provided care + is taken preparing the environment you will not deploy to the wrong host + (e.g., a live server rather than a staging server, or the wrong live + server) because those keys will not be accessible. + . + * /Configuration Control/: the parameters controling the encryption and + hashing functions can be set up independently in each section of the + store, allowing for heavier hashing to be used on live servers and + light hashing to be used on development and staging servers where + authentication needs to be quick. + . + * /Keystore Integrity/: the keystore can be signed and every operation + made to check that the keystore matches its signature (and the public + signing key matches an independent copy on the client). + . + * /External Crypto Operations/: keys in the keystore can be used to sign + or encrypt external obejcts (provided they can be loaded into memory). + . + /The Onion/ + . + Perhaps apropriately, the keystore package has several layers. Most users + will probably need only the top "batteries-included" layer: + . + * @Data.KeyStore.Sections@: this provides a high-level model that allows + a flexible hierarchical keystore to be set up relatively easily. + See the 'deploy' example for details. + . + * @Data.KeyStore.CLI@ : This provides a stanalone program for inspecting + and editing your keystores. It can also be embedded into your own + deployment app. See the @deploy@ example for details. + . + * @Data.KeyStore.IO@: this library provides general programatic access to + a keystore through @IO@ primitives. See the source code for the @Sections@ + for an example of this module in use. + . + * @Data.KeyStore.KS@: this library provides general programatic access to + a keystore through functional @KS@ primitives. See the source code for + the @IO@ for an exteded example this system in action. + . + * @Data.KeyStore.Types@: This provides access to keystores at the types + level. + . + /Launch Instructions/ + . + See the bottom + on GitHub home page for launch instructions for the deploy example. +Extra-source-files: changelog Cabal-version: >= 1.14 Source-repository head @@ -66,26 +169,18 @@ containers >= 0.4 , directory >= 1.2 , filepath >= 1.3 , + lens >= 3.9.2 , mtl >= 2 , + old-locale >= 1.0.0.5 , optparse-applicative >= 0.9.0 , pbkdf >= 1.1.1.0 , + regex-compat-tdfa >= 0.95.1 , safe >= 0.3.3 , text >= 0.11.3 , - unordered-containers >= 0.2.3.0 , - - Cabal >= 1.16 , - QuickCheck >= 2.6 , - array >= 0.4 , - case-insensitive >= 1.0.0.2 , - lens >= 3.9.2 , - old-locale >= 1.0.0.5 , - regex-compat-tdfa >= 0.95.1 , - safecopy >= 0.8.2 , - template-haskell , time >= 1.4 , + unordered-containers >= 0.2.3.0 , vector >= 0.10.0.1 - Default-Language: Haskell2010 GHC-Options: @@ -95,13 +190,13 @@ Executable ks Hs-Source-Dirs: main - Main-is: ks.hs + Main-is: ks.hs Default-Language: Haskell2010 Build-depends: base > 4 && < 5 , - keystore >= 0.0.0.1 + keystore GHC-Options: -fwarn-tabs diff --git a/src/Data/KeyStore/CLI.hs b/src/Data/KeyStore/CLI.hs --- a/src/Data/KeyStore/CLI.hs +++ b/src/Data/KeyStore/CLI.hs @@ -18,7 +18,7 @@ version :: String -version = "0.3.0.1" +version = "0.4.0.0" cli :: IO () cli = parseCLI >>= command Nothing diff --git a/src/Data/KeyStore/KS/Opt.hs b/src/Data/KeyStore/KS/Opt.hs --- a/src/Data/KeyStore/KS/Opt.hs +++ b/src/Data/KeyStore/KS/Opt.hs @@ -34,7 +34,6 @@ import qualified Data.ByteString.Lazy.Char8 as LBS import qualified Data.HashMap.Strict as HM import Data.Aeson -import Data.Attoparsec.Number import qualified Data.Text as T import Data.Monoid import Data.Maybe @@ -223,7 +222,7 @@ frm v = case fromJSON v of Success i -> inj i - Error _ -> x0 + _ -> x0 text_opt :: [T.Text] -> (T.Text->a,a->T.Text) -> a -> OptEnum -> Opt a text_opt hp (inj,prj) x0 ce = diff --git a/src/Data/KeyStore/Sections.hs b/src/Data/KeyStore/Sections.hs --- a/src/Data/KeyStore/Sections.hs +++ b/src/Data/KeyStore/Sections.hs @@ -57,28 +57,42 @@ decode s = listToMaybe [ k | k<-[minBound..maxBound], encode k==s ] +-- | This class describes the relationship between the host-id, section-id +-- and key-id types used to build a hierarchical deployment model for a +-- keystore. A minimal instance would have to define hostDeploySection. +-- The deploy example program contains a fairly thorough example of this +-- class being used to implement a quite realitic deploymrnt scenario. class (Code h, Code s, Code k) => Sections h s k | s -> h, k -> h , h -> s, k -> s , s -> k, h -> k where - hostSection :: h -> s -- ^ the deployment section - hostRSection :: h -> s -- ^ section where host-indexed - -- keys reside for given host - sectionType :: s -> SectionType - superSections :: s -> [s] - keyIsHostIndexed :: k -> Maybe (h->Bool) - keyIsInSection :: k -> s -> Bool - getKeyData :: Maybe h -> s -> k -> IO KeyData - sectionSettings :: Maybe s -> IO Settings - describeKey :: k -> String - describeSection :: s -> String - sectionPWEnvVar :: s -> EnvVar - - hostRSection = hostSection + hostDeploySection :: h -> s -- ^ the deployment section: for a given host, + -- the starting section for locating the keys + -- during a deployment ('higher'/closer sections + -- taking priority) + sectionType :: s -> SectionType -- ^ whether the section holds the top key for the + -- keystore (i.e., keystore master key), the signing key + -- for the keystore or is a normal section containing + -- deployment keys + superSections :: s -> [s] -- ^ the sections that get a copy of the master + -- for this section (making all of its keys + -- available to them); N.B., the graph formed by this + -- this relationship over the sections must be acyclic + keyIsHostIndexed :: k -> Maybe (h->Bool) -- ^ if the key is host-indexed then the predicate + -- specifies the hosts that use this key + keyIsInSection :: k -> s -> Bool -- ^ specifies which sections a key is resident in + getKeyData :: Maybe h -> s -> k -> IO KeyData -- ^ loads the data for a particular key + sectionSettings :: Maybe s -> IO Settings -- ^ loads the setting for a given settings + describeKey :: k -> String -- ^ describes the key (for the ks help command) + describeSection :: s -> String -- ^ describes the section (for the ks help command) + sectionPWEnvVar :: s -> EnvVar -- ^ secifies the environment variable containing the + -- ^ master password/provate key for for the given section sectionType = const ST_keys + superSections = const [] + keyIsHostIndexed = const Nothing keyIsInSection = const $ const True @@ -95,13 +109,19 @@ sectionPWEnvVar = EnvVar . T.pack . ("KEY_" ++) . _name . passwordName +-- | Sections are used to hold the top (master) key for the keystore, +-- its signing key, or deployment keys data SectionType = ST_top | ST_signing | ST_keys deriving (Show,Eq,Ord) - +-- | A key is triple containing some (plain-text) identity information for the +-- key, some comment text and the secret text to be encrypted. Note that +-- the keystore doesn't rely on this information but merely stores it. (They +-- can be empty.) The identity field will often be used to storte the key's +-- identity within the system that generates and uses it, ofor example. data KeyData = KeyData { kd_identity :: Identity @@ -109,16 +129,25 @@ , kd_secret :: B.ByteString } - +-- | One, many or all of the keys in a store may be rotated at a time. +-- we use one of these to specify which keys are to be rotated. type KeyPredicate h s k = Maybe h -> s -> k -> Bool +-- | Requests to retrieve a key from the staor can fail for various reasons. +type Retrieve a = Either RetrieveDg a + +-- | This type specifies the reasons that an attempt to access a key from the +-- store has failed. This kind of failure suggests an inconsistent model +-- and will be raised regardless of which keys have been stored in the store. data RetrieveDg = RDG_key_not_reachable | RDG_no_such_host_key deriving (Show,Eq,Ord) - +-- | Here we create the store and rotate in a buch of keys. N.B. All of the +-- section passwords must be bound in the process environment before calling +-- procedure. initialise :: Sections h s k => CtxParams -> KeyPredicate h s k -> IO () initialise cp kp = do stgs <- scs kp Nothing @@ -134,37 +163,41 @@ mks :: Sections h s k => KeyPredicate h s k -> IC -> s -> IO () mks = const mk_section +-- | Rotate in a set of keys spwecified by the predicate. rotate :: Sections h s k => IC -> KeyPredicate h s k -> IO () rotate ic kp = sequence_ [ rotate' ic mb_h s k | (mb_h,s,k)<-host_keys++non_host_keys, kp mb_h s k ] where - host_keys = [ (Just h ,s,k) | k<-[minBound..maxBound], Just hp<-[keyIsHostIndexed k], h<-[minBound..maxBound], hp h, let s=hostRSection h ] - non_host_keys = [ (Nothing,s,k) | k<-[minBound..maxBound], Nothing<-[keyIsHostIndexed k], s<-[minBound..maxBound], keyIsInSection k s ] - -retrieve :: Sections h s k => IC -> h -> k -> IO (Either RetrieveDg [Key]) -retrieve ic h k = either (return . Left) (\nm->Right <$> locateKeys ic nm) ei_nm - where - ei_nm = case keyIsHostIndexed k of - Nothing -> ei_nm' Nothing - Just hp | hp h -> ei_nm' $ Just h - | otherwise -> Left RDG_no_such_host_key - - ei_nm' mb_h = maybe (Left RDG_key_not_reachable) Right $ - listToMaybe [ key_nme mb_h s_ k | s_ <- lower_sections s0, keyIsInSection k s_ ] + host_keys = [ (Just h ,s,k) | k<-[minBound..maxBound], Just isp<-[keyIsHostIndexed k], h<-[minBound..maxBound], isp h, let s = key_section h k ] + non_host_keys = [ (Nothing,s,k) | k<-[minBound..maxBound], Nothing <-[keyIsHostIndexed k], s<-[minBound..maxBound], keyIsInSection k s ] - s0 = hostSection h +-- | Retrieve the keys for a given host from the store. Note that the whole history for the given key is returned. +-- Note also that the secret text may not be present if it si not accessible (depnding upon hwich section passwords +-- are correctly bound in the process environment). Note also that the 'Retrieve' diagnostic should not fail if a +-- coherent model has been ddefined for 'Sections'. +retrieve :: Sections h s k => IC -> h -> k -> IO (Retrieve [Key]) +retrieve ic h k = either (return . Left) (\nm->Right <$> locateKeys ic nm) $ keyName h k +-- | Sign the keystore. (Requites the password for the signing section to be correctly +-- bound in the environment) signKeystore :: Sections h s k => IC -> SECTIONS h s k -> IO B.ByteString signKeystore ic scn = B.readFile (the_keystore $ ic_ctx_params ic) >>= sign_ ic (sgn_nme $ signing_key scn) +-- Verify that the signature for a keystore matches the keystore. verifyKeystore :: IC -> B.ByteString -> IO Bool verifyKeystore ic sig = B.readFile (the_keystore $ ic_ctx_params ic) >>= flip (verify_ ic) sig +-- | A predicate specifying all of the keys in the store. noKeys :: KeyPredicate h s k noKeys _ _ _ = False +-- | A predicate specifying none of the keys in the keystore. allKeys :: KeyPredicate h s k allKeys _ _ _ = True +-- | A utility for specifing a slice of the keys in the store, optionally specifying +-- host section and key that should belong to the slice. (If the host is specified then +-- the resulting predicate will only include host-indexed keys belonging to the +-- given host.) keyPrededicate :: Sections h s k => Maybe h -> Maybe s -> Maybe k -> KeyPredicate h s k keyPrededicate mbh mbs mbk mbh_ s k = h_ok && s_ok && k_ok where @@ -172,6 +205,8 @@ s_ok = maybe True (s==) mbs k_ok = maybe True (k==) mbk +-- Generate some help text for the keys. If no key is specified then they are +-- merely listed, otherwise the help for the given key is listed. keyHelp :: Sections h s k => Maybe k -> T.Text keyHelp x@Nothing = T.unlines $ map (T.pack . encode) [minBound..maxBound `asTypeOf` fromJust x ] keyHelp (Just k) = T.unlines $ map T.pack $ (map f $ concat @@ -185,6 +220,8 @@ f = uncurry $ printf "%-10s %s" +-- Generate some help text for the sectionss. If no section is specified then they are +-- merely listed, otherwise the help for the given section is listed. sectionHelp :: Sections h s k => Maybe s -> IO T.Text sectionHelp x@Nothing = return $ T.unlines $ map (T.pack . encode) [minBound..maxBound `asTypeOf` fromJust x ] sectionHelp (Just s) = do @@ -204,7 +241,7 @@ ST_signing -> "(signing)" ST_keys -> "(keys)" env = "$" ++ T.unpack (_EnvVar $ sectionPWEnvVar s) - hln = unwords $ nub [ encode h | h<-[minBound..maxBound], hostSection h==s ] + hln = unwords $ nub [ encode h | h<-[minBound..maxBound], hostDeploySection h==s ] sln = unwords $ map encode $ superSections s uln = unwords $ map encode $ [ s_ | s_<-[minBound..maxBound], s `elem` superSections s_ ] kln = fmt $ flip keyIsInSection s @@ -213,6 +250,9 @@ fmt_s stgs = map (" "++) $ lines $ LBS.unpack $ A.encode $ A.Object $ _Settings stgs +-- | List a shell script for establishing all of the keys in the environment. NB For this +-- to work the password for the top section (or the passwords for all of the sections +-- must be bound if the store does not maintain a top key). secretKeySummary :: Sections h s k => IC -> SECTIONS h s k -> IO T.Text secretKeySummary ic scn = T.unlines <$> mapM f (sections scn) where @@ -220,11 +260,15 @@ sec <- T.pack . B.unpack <$> (showSecret ic False $ passwordName s) return $ T.concat ["export ",_EnvVar $ sectionPWEnvVar s,"=",sec] +-- | List a shell script for storing the public signing key for the store. publicKeySummary :: Sections h s k => IC -> SECTIONS h s k -> FilePath -> IO T.Text publicKeySummary ic scn fp = f <$> showPublic ic True (sgn_nme $ signing_key scn) where f b = T.pack $ "echo '" ++ B.unpack b ++ "' >" ++ fp ++ "\n" +-- | List all of the keys that have the given name as their prefix. If the +-- generic name of a key is given then it will list the complete history for +-- the key, the current (or most recent) entry first. locateKeys :: IC -> Name -> IO [Key] locateKeys ic nm = sortBy (flip $ comparing _key_name) . filter yup <$> keys ic where @@ -233,9 +277,32 @@ nm_s = _name nm -keyName :: Sections h s k => h -> k -> Name -keyName h k = key_nme (const h <$> keyIsHostIndexed k) (hostSection h) k +-- | Return the genertic name for a given key thst is used by the specified +-- host, returning a failure diagnostic if the host does not have such a key +-- on the given Section model. +keyName :: Sections h s k => h -> k -> Retrieve Name +keyName h k = do + mb_h <- case keyIsHostIndexed k of + Nothing -> return Nothing + Just hp | hp h -> return $ Just h + | otherwise -> Left RDG_no_such_host_key + s <- keySection h k + return $ key_nme mb_h s k +-- a wrapper on keySection used internally in functional contezxtx +key_section :: Sections h s k => h -> k -> s +key_section h k = either oops id $ keySection h k + where + oops = error "key_section" + +-- | Rerurn the section that a host sores a given key in, returning a +-- failure diagnostic if the host does not keep such a key in the given +-- 'Section' model. +keySection :: Sections h s k => h -> k -> Retrieve s +keySection h k = maybe (Left RDG_key_not_reachable) return $ listToMaybe $ + filter (keyIsInSection k) $ lower_sections $ hostDeploySection h + +-- | The name of the key that stores the password for a given sections. passwordName :: Sections h s k => s -> Name passwordName s = name' $ "pw_" ++ encode s