diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,12 +1,18 @@ # record-encode +## Encoding categorical variables + [![Build Status](https://travis-ci.org/ocramz/record-encode.png)](https://travis-ci.org/ocramz/record-encode) [![Hackage](https://img.shields.io/hackage/v/record-encode.svg)](https://hackage.haskell.org/package/record-encode) This library provides generic machinery to encode values of some algebraic type as points in a vector space. -Analyzing datasets that have one or more categorical variables (that is, values having a sum type) typically requires a series of boilerplate transformations, and the `encodeOneHot` function provided here does precisely that. +Values of a sum type (e.g. enumerations) are also called "categorical" variables in statistics, because they encode a choice between a number of discrete categories. +On the other hand, many data science / machine learning algorithms rely on a purely numerical representation of data; the conversion code from values of a static type is often "boilerplate", i.e. largely repeated and not informative. + +The `encodeOneHot` function provided here is a generic utility function (i.e. defined once and for all) to compute the one-hot representation of any sum type. + # Usage example ``` @@ -26,8 +32,9 @@ OH {oDim = 3, oIx = 1} ``` +Please refer to the documentation of Data.Record.Encode for more examples and details. # Acknowledgements -Gagandeep Bhatia (@gagandeepb) for his GSoC '18 work on `Frames-beam`, Mark Karpov (@mrkkrp) for his Template Haskell tutorial, Anthony Cowley (@acowley) for `vinyl` and `Frames`, @mniip on Freenode #haskell for helping me better understand what can be done with generic programming. +Gagandeep Bhatia (@gagandeepb) for his Google Summer of Code 2018 work on [`Frames-beam`](https://github.com/gagandeepb/Frames-beam), Mark Karpov (@mrkkrp) for his Template Haskell tutorial, Anthony Cowley (@acowley) for [`Frames`](https://hackage.haskell.org/package/Frames), @mniip on Freenode #haskell for helping me better understand what can be done with generic programming. diff --git a/record-encode.cabal b/record-encode.cabal --- a/record-encode.cabal +++ b/record-encode.cabal @@ -1,5 +1,5 @@ name: record-encode -version: 0.2.2 +version: 0.2.3 synopsis: Generic encoding of records description: Generic encoding of records. It currently provides a single, polymorphic function to encode sum types (i.e. categorical variables) as one-hot vectors. homepage: https://github.com/ocramz/record-encode @@ -19,7 +19,7 @@ ghc-options: -Wall hs-source-dirs: src exposed-modules: Data.Record.Encode - other-modules: Data.Record.Encode.Generics + Data.Record.Encode.Generics build-depends: base >= 4.7 && < 5 , generics-sop , vector @@ -38,6 +38,16 @@ , hspec , QuickCheck , vector + +test-suite doctest + default-language: Haskell2010 + ghc-options: -Wall + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: DocTest.hs + build-depends: base + , record-encode + , doctest source-repository head type: git diff --git a/src/Data/Record/Encode.hs b/src/Data/Record/Encode.hs --- a/src/Data/Record/Encode.hs +++ b/src/Data/Record/Encode.hs @@ -11,9 +11,7 @@ == Internals -This library makes use of generic programming to analyze both values and types (see the internal Data.Record.Encode.Generics module). - -Initially, it was relying on Template Haskell to analyze /types/, using the the instance generation machinery explained here: +This library makes use of generic programming to analyze both values and types (see the 'Data.Record.Encode.Generics' module). -} @@ -37,34 +35,53 @@ import Data.Record.Encode.Generics +-- $setup +-- >>> :set -XDeriveGeneric +-- >>> import qualified GHC.Generics as G +-- >>> import qualified Generics.SOP as SOP +-- >>> import Data.Record.Encode +-- >>> data X = A | B | C deriving (Enum, G.Generic) +-- >>> instance SOP.Generic X --- data X a = A | B a | C | D | E | F deriving G.Generic --- instance Generic (X a) - -data X = A | B | C deriving (G.Generic) -instance Generic X - -- | Constraints necessary to 'encodeOneHot' a value. -- -- NB: 'GVariants' is an internal typeclass, and this constraint is automatically satisfied if the type is an instance of 'G.Generic' type G a = (GVariants (G.Rep a), G.Generic a, Generic a) --- | Computes the one-hot encoding of a value of a sum type. +-- | Computes the one-hot encoding of a value of a sum type. A sum type is defined as a choice between N type constructors, each having zero or more fields. -- --- The type of the input value must be an instance of 'GHC.Generics.Generic' (from GHC.Generics) /and/ of 'Generics.SOP.Generic' (from the `generics-sop` library). +-- The number of constructors becomes the dimensionality of the embedding space, and the constructor position (as defined in its implementation) is interpreted as the index of the nonzero coordinate. -- --- >>> :set -XDeriveGeneric +-- NB : This function computes the generic representation /only/ up to the /outermost/ constructor (see examples below). -- --- >>> import qualified GHC.Generics as G --- >>> import qualified Generics.SOP as SOP --- >>> import Data.Record.Encode +-- The type of the input value must be an instance of 'GHC.Generics.Generic' (from GHC.Generics) /and/ of 'Generics.SOP.Generic' (from the `generics-sop` library). -- --- >>> data X = A | B | C deriving (G.Generic) --- >>> instance SOP.Generic X +-- @ +-- > :set -XDeriveGeneric +-- +-- > import qualified GHC.Generics as G +-- > import qualified Generics.SOP as SOP +-- > import Data.Record.Encode +-- +-- > data X = A | B | C deriving (Enum, G.Generic) +-- > instance SOP.Generic X +-- @ -- +-- The @B@ constructor is the second (i.e. position 1 counting from 0) of a choice of three : +-- -- >>> encodeOneHot B -- OH {oDim = 3, oIx = 1} +-- +-- The @Just@ constructor is the second of a choice of two: +-- +-- >>> encodeOneHot $ Just B +-- OH {oDim = 2, oIx = 1} +-- +-- The @Nothing@ constructor is the first: +-- +-- >>> encodeOneHot (Nothing :: Maybe Int) +-- OH {oDim = 2, oIx = 0} encodeOneHot :: forall a . G a => a -> OneHot encodeOneHot x = OH len i where len = fromIntegral $ gnconstructors (Proxy :: Proxy a) @@ -84,15 +101,23 @@ -- | A one-hot encoding is a d-dimensional vector having a single component equal to 1 and all others equal to 0. -- We represent it here compactly as two integers: an integer dimension and an index (which must both be nonnegative). data OneHot = OH { - oDim :: !Int -- ^ Dimension of ambient space (i.e. number of categories) - , oIx :: !Int -- ^ Index of nonzero entry + oDim :: !Int -- ^ Dimension of embedding space (i.e. number of categories) + , oIx :: !Int -- ^ Index of nonzero coordinate } deriving (Eq, Show) -- | Compares two one-hot encodings for equality. Returns Nothing if the operand dimensions are not equal. +-- +-- >>> compareOH (OH 3 2) (OH 3 1) +-- Just GT +-- +-- >>> compareOH (OH 3 2) (OH 5 1) +-- Nothing compareOH :: OneHot -> OneHot -> Maybe Ordering compareOH (OH d1 i1) (OH d2 i2) | d1 /= d2 = Nothing | otherwise = Just (compare i1 i2) + + -- class Encode i d where diff --git a/test/DocTest.hs b/test/DocTest.hs new file mode 100644 --- /dev/null +++ b/test/DocTest.hs @@ -0,0 +1,8 @@ +module Main where + +import Test.DocTest (doctest) + +main :: IO () +main = doctest [ + "src/Data/Record/Encode.hs" + ] diff --git a/test/LibSpec.hs b/test/LibSpec.hs --- a/test/LibSpec.hs +++ b/test/LibSpec.hs @@ -3,21 +3,16 @@ import Test.Hspec -- import Test.Hspec.QuickCheck - import qualified GHC.Generics as G import Generics.SOP --- import qualified Data.Vector as V - import Data.Record.Encode -data X = Xa | Xb | Xc deriving (Eq, Show, G.Generic) +data X = Xa | Xb | Xc deriving (Eq, Show, Enum, G.Generic) instance Generic X - - main :: IO () main = hspec spec @@ -32,48 +27,3 @@ -- ourAdd x y `shouldBe` ourAdd y x - - --- data P0 = P0 Bool Char deriving (Eq, Show, G.Generic) --- -- instance Generic P0 - --- deriveCountable ''Bool --- deriveCountable ''Char --- -- deriveCountable ''Integer --- deriveCountable ''P0 - - --- -- λ> hcmap (Proxy :: Proxy Show) (mapIK (const ())) $ from $ P0 42 'z' --- -- SOP (Z (K () :* K () :* Nil)) - - --- data Fx = Ax | Bx | Cx deriving (Eq, Show, Enum, G.Generic) --- -- instance Generic Fx - --- data Fy a = Ay a | By | Cy deriving (Eq, Show, G.Generic) --- -- instance Generic (Fy a) - --- data Gx = Ax' | Bx' | Cx' deriving (Eq, Show, Enum, G.Generic) --- -- instance Generic Gx - --- -- | a Product-Of-Sums --- data P1 a = P1 Fx (Fy a) deriving (Eq, Show, G.Generic) --- -- instance Generic (P1 a) - --- p10 :: P1 Integer --- p10 = P1 Ax (Ay 42) - --- -- gp10 :: SOP I '[ '[Fx, Fy Integer] ] --- -- gp10 = from p10 - --- -- λ> from $ P1 Ax (Ay 42) --- -- SOP (Z (I Ax :* I (Ay 42) :* Nil)) - - --- data P2 = P2 Fx Fx deriving (Eq, Show, G.Generic) --- -- instance Generic P2 - - - --- -- data Y = Y (X, X) deriving (Eq, Show, G.Generic) --- -- deriveCountable ''Y