packages feed

geolite-csv (empty) → 0.1.0

raw patch · 6 files changed

+275/−0 lines, 6 filesdep +HUnitdep +basedep +colonnadesetup-changed

Dependencies added: HUnit, base, colonnade, directory, geolite-csv, ip, pipes, pipes-bytestring, pipes-text, siphon, test-framework, test-framework-hunit, text

Files

+ LICENSE view
@@ -0,0 +1,30 @@+Copyright Andrew Martin (c) 2016++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Andrew Martin nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ geolite-csv.cabal view
@@ -0,0 +1,52 @@+name:                geolite-csv+version:             0.1.0+synopsis:            Initial project template from stack+description:         Please see README.md+homepage:            https://github.com/andrewthad/colonnade+license:             BSD3+license-file:        LICENSE+author:              Andrew Martin+maintainer:          andrew.thaddeus@gmail.com+copyright:           2016 Andrew Martin+category:            web+build-type:          Simple+-- extra-source-files:+cabal-version:       >=1.10++library+  hs-source-dirs:      src+  exposed-modules:+    Geolite.Types+    Geolite.Csv+  build-depends:+      base >= 4.7 && < 5+    , colonnade+    , siphon+    , ip   >= 0.8.4+    , text+    , pipes+  default-language:    Haskell2010++test-suite geolite-csv-test+  type:                exitcode-stdio-1.0+  hs-source-dirs:      test+  main-is:             Spec.hs+  build-depends:+      base+    , geolite-csv+    , siphon+    , colonnade+    , test-framework+    , text+    , pipes+    , HUnit+    , test-framework-hunit+    , pipes-bytestring+    , pipes-text+    , directory+  ghc-options:         -threaded -rtsopts -with-rtsopts=-N+  default-language:    Haskell2010++source-repository head+  type:     git+  location: https://github.com/andrewthad/colonnade
+ src/Geolite/Csv.hs view
@@ -0,0 +1,58 @@+{-# LANGUAGE OverloadedStrings #-}++module Geolite.Csv where++import Data.Text (Text)+import Pipes (Pipe)+import Colonnade.Types+import Geolite.Types++import qualified Data.Text as Text+import qualified Net.IPv4.Range.Text as IPv4RangeText+import qualified Data.Text.Read as TextRead+import qualified Siphon.Decoding as SD+import qualified Siphon.Content as SC+import qualified Colonnade.Decoding.Text as CDT+import qualified Colonnade.Decoding as CD++cities :: Monad m => Pipe Text City m (DecodingRowError Headed Text)+cities = SD.headedPipe SC.text decodingCity++blocks :: Monad m => Pipe Text Block m (DecodingRowError Headed Text)+blocks = SD.headedPipe SC.text decodingBlock++decodingCity :: Decoding Headed Text City+decodingCity = City+  <$> fmap GeonameId (CD.headed "geoname_id" CDT.int)+  <*> CD.headed "locale_code" CDT.text+  <*> CD.headed "continent_code" CDT.text+  <*> CD.headed "continent_name" CDT.text+  <*> CD.headed "country_iso_code" CDT.text+  <*> CD.headed "country_name" CDT.text+  <*> CD.headed "subdivision_1_iso_code" CDT.text+  <*> CD.headed "subdivision_1_name" CDT.text+  <*> CD.headed "subdivision_2_iso_code" CDT.text+  <*> CD.headed "subdivision_2_name" CDT.text+  <*> CD.headed "metro_code" (CDT.optional CDT.int)+  <*> CD.headed "time_zone" CDT.text++decodingBlock :: Decoding Headed Text Block+decodingBlock = Block+  <$> CD.headed "network" IPv4RangeText.decodeEither+  <*> CD.headed "geoname_id"+        (CDT.optional $ CDT.map GeonameId CDT.int)+  <*> CD.headed "registered_country_geoname_id"+        (CDT.optional $ CDT.map GeonameId CDT.int)+  <*> CD.headed "represented_country_geoname_id"+        (CDT.optional $ CDT.map GeonameId CDT.int)+  <*> CD.headed "is_anonymous_proxy" (CDT.trueFalse "1" "0")+  <*> CD.headed "is_satellite_provider" (CDT.trueFalse "1" "0")+  <*> CD.headed "postal_code" CDT.text+  <*> CD.headed "latitude"+        (CDT.optional $ CDT.fromReader TextRead.rational)+  <*> CD.headed "longitude"+        (CDT.optional $ CDT.fromReader TextRead.rational)+  <*> CD.headed "accuracy_radius"+        (CDT.optional CDT.int)++
+ src/Geolite/Types.hs view
@@ -0,0 +1,42 @@+module Geolite.Types where++import Net.Types (IPv4Range)+import Data.Text (Text)+import Data.Fixed++data E4++instance HasResolution E4 where+  resolution _ = 4++newtype GeonameId = GeonameId { getGeonameId :: Int }+  deriving (Show,Read,Eq,Ord)++data City = City+  { cityGeonameId :: GeonameId+  , cityLocaleCode :: Text+  , cityContinentCode :: Text+  , cityContinentName :: Text+  , cityCountryIsoCode :: Text+  , cityCountryName :: Text+  , citySubdivision1IsoCode :: Text+  , citySubdivision1Name :: Text+  , citySubdivision2IsoCode :: Text+  , citySubdivision2Name :: Text+  , cityMetroCode :: Maybe Int+  , cityTimeZone :: Text+  } deriving (Show,Read,Eq,Ord)++data Block = Block+  { blockNetwork :: IPv4Range+  , blockGeonameId :: Maybe GeonameId+  , blockRegisteredCountryGeonameId :: Maybe GeonameId+  , blockRepresentedCountryGeonameId :: Maybe GeonameId+  , blockIsAnonymousProxy :: Bool+  , blockIsSatelliteProvider :: Bool+  , blockPostalCode :: Text+  , blockLatitude :: Maybe (Fixed E4)+  , blockLongitude :: Maybe (Fixed E4)+  , blockAccuracyRadius :: Maybe Int+  } deriving (Show,Read,Eq,Ord)+
+ test/Spec.hs view
@@ -0,0 +1,91 @@+{-# LANGUAGE OverloadedStrings #-}++module Main (main) where++import Test.HUnit                           (Assertion,(@?=),assertBool,assertFailure)+import Test.Framework                       (defaultMainWithOpts, interpretArgsOrExit,+                                             testGroup, Test)+import Test.Framework.Providers.HUnit       (testCase)+import Test.Framework.Runners.TestPattern   (parseTestPattern)+import Test.Framework.Runners.Options       (RunnerOptions'(..))+import Geolite.Csv                          (cities,blocks)+import Data.Text                            (Text)+import Colonnade.Types+import Siphon.Types+import Data.Functor.Identity+import Control.Monad                        (unless)+import System.Environment                   (getArgs)+import System.Directory                     (doesDirectoryExist)+import System.IO                            (withFile,IOMode(ReadMode))+import qualified Data.Text                  as Text+import qualified Pipes.Prelude              as Pipes+import qualified Pipes.ByteString           as PB+import qualified Pipes.Text.Encoding        as PT+import qualified Siphon.Decoding            as SD+import qualified Colonnade.Decoding         as Decoding+import Pipes++------------------------------------------------+-- The default behavior of this test suite is to+-- test the CSV decoders against small samples of+-- the GeoLite2 databases. These small samples are+-- included as part of this repository. If you give+-- this test suite an argument named "large", it+-- will run against the full CSVs, which are around+-- 350MB. These are not included+-- as part of the repository, so they need to be+-- downloaded. The script found in+-- scripts/load-full-databases will download the full+-- archive, decompress it, and move the files to+-- the appropriate directory for this test suite+-- to run on them.+-----------------------------------------------++main :: IO ()+main = do+  xs <- getArgs+  ropts' <- interpretArgsOrExit xs+  let ropts = ropts'+        { ropt_test_patterns = case ropt_test_patterns ropts' of+            Nothing -> Just [parseTestPattern "small"]+            Just xs -> Just xs+        }+  defaultMainWithOpts tests ropts++tests :: [Test]+tests = flip concatMap ["small","large"] $ \size ->+  [ testGroup size+    [ testCase "Network Blocks" $ streamFileWith+        ("data/" ++ size ++ "/GeoLite2-City-Blocks-IPv4.csv")+        blocks+    , testCase "English City Locations" $ streamFileWith+        ("data/" ++ size ++ "/GeoLite2-City-Locations-en.csv")+        cities+    , testCase "Japanese City Locations" $ streamFileWith+        ("data/" ++ size ++ "/GeoLite2-City-Locations-ja.csv")+        cities+    ]+  ]++streamFileWith ::+     String+  -> Pipe Text a IO (DecodingRowError Headed Text)+  -> Assertion+streamFileWith filename decodingPipe = do+  r <- withFile filename ReadMode $ \h -> runEffect $+        fmap (SD.convertDecodeError "utf-8") (PT.decode (PT.utf8 . PT.eof) $ PB.fromHandle h)+    >-> fmap Just decodingPipe+    >-> Pipes.drain+  case r of+    Nothing -> assertBool "impossible" True+    Just err -> assertFailure (Decoding.prettyError Text.unpack err)++-- let dirPiece = case xs of+--       ["full"] -> "large/"+--       _        -> "small/"+--     fullDirName = "data/" ++ dirPiece+--     errMsg = concat+--       [ "The "+--       , fullDirName+--       , " directory does not exist in the geolite project"+--       ]