haskell-google-trends (empty) → 0.0.1
raw patch · 5 files changed
+171/−0 lines, 5 filesdep +basedep +bytestringdep +haskell-fake-user-agentsetup-changed
Dependencies added: base, bytestring, haskell-fake-user-agent, lens, regex-base, regex-posix, tagsoup, text, wreq
Files
- Google/Trends.hs +124/−0
- LICENSE +1/−0
- README.md +21/−0
- Setup.hs +2/−0
- haskell-google-trends.cabal +23/−0
+ Google/Trends.hs view
@@ -0,0 +1,124 @@+{-# LANGUAGE OverloadedStrings, FlexibleContexts #-}+module Google.Trends(queryTrendsWithLogin) where++import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.ByteString.Char8 as BS+import qualified Data.ByteString.Lazy.Char8 as LBS+import Control.Lens+import HTTP.ThirdParty.FakeUserAgent+import Network.Wreq+import Text.HTML.TagSoup+import qualified Network.Wreq.Session as Ss+import Text.Regex.Posix+import Text.Regex.Base++urlServiceLoginBoxAuth :: String+urlServiceLoginBoxAuth = "https://accounts.google.com/ServiceLoginBoxAuth"+urlServiceLoginBoxAuthBS :: BS.ByteString+urlServiceLoginBoxAuthBS = "https://accounts.google.com/ServiceLoginBoxAuth"+urlTrends :: String+urlTrends = "http://www.google.com/trends"+urlTrendsComponent :: String+urlTrendsComponent = "http://www.google.com/trends/fetchComponent"+urlTrendsReport :: String+urlTrendsReport = "http://www.google.com/trends/trendsReport"+urlCookieCheck :: String+urlCookieCheck = "https://www.google.com/accounts/CheckCookie"+urlGoogle :: String+urlGoogle = "http://www.google.com"++defaultUserAgent = do+ bs <- getLatestBrowserString "Firefox"+ return (BS.pack bs)++toStrictString = BS.concat . LBS.toChunks++{- Google Login -}+loginHeaders options ua = options & +{- (proxy ?~ httpProxy "localhost" 8080) &-}+ (header "Accept" .~ ["text/plain"]) &+ (header "User-Agent" .~ [ua]) &+ (header "Content-Type" .~ ["application/x-www-form-urlencoded"]) &+ (header "Referrer" .~ [urlServiceLoginBoxAuthBS])++findLoginInputs tags = + [toPair tag | tag <- tags, tag ~== TagOpen ("input" :: String) []]+ where toPair tag = (toStrictString (fromAttrib "name" tag), fromAttrib "value" tag)++getLoginInputs headers session = do+ resp <- Ss.getWith headers session urlServiceLoginBoxAuth+ let body = resp ^. responseBody+ return $ (findLoginInputs . parseTags) body++makeLoginForm form [] email pass = (("Email" := email):("Passwd" := pass):form)+makeLoginForm form ((key, value):inputs) email pass =+ makeLoginForm newForm inputs email pass+ where newForm = ((key := value):form)++doLogin :: String -> String -> (Options -> Ss.Session -> IO a) -> (Options -> Ss.Session -> IO a) -> IO a+doLogin email pass continue fail = do+ Ss.withSession $ \session -> do+ userAgent <- defaultUserAgent+ let headers = loginHeaders defaults userAgent+ inputs <- getLoginInputs headers session+ let form = makeLoginForm [] inputs email pass+ resp <- Ss.postWith headers session urlServiceLoginBoxAuth form+ let cookies = (resp ^? responseCookie "SID", resp ^? responseCookie "HSID")+ case cookies of+ (Nothing, _) -> fail headers session+ (_, Nothing) -> fail headers session+ _ -> continue headers session++{- Google Trends -}+queryParams options keywords = options &+ (param "q" .~ [keywords]) &+ (param "hl" .~ ["en-US"]) &+ (param "cid" .~ ["TIMESERIES_GRAPH_0"]) &+ (param "export" .~ ["5"])++queryTrends keywords headers session = do+ let options = queryParams headers (T.pack keywords)+ resp <- Ss.getWith options session urlTrendsComponent+ let body = resp ^. responseBody+ let parsed = T.unpack (TE.decodeUtf8 (toStrictString body))+ return $ [processPoint point | point <- parseTrends parsed]++parseTrends inp =+ let points = getAllTextMatches (inp =~ pat :: AllTextMatches [] String) in+ processTrends points+ where pat = "([0-9]{4}, [0-9]+, [0-9]+, [0-9]+, [0-9]+[^}]+\\},[^,]+,[^,]+,[0-9]+,)" :: String++processTrends [] = []+processTrends (point:points) =+ ((drop 1 dates) ++ (drop 1 counts)):(processTrends points)+ where dates = getAllTextSubmatches (point =~ datePat :: AllTextSubmatches [] String)+ datePat = "([0-9]{4}), ([0-9]+), [0-9]+," :: String+ counts = getAllTextSubmatches (point =~ countPat :: AllTextSubmatches [] String)+ countPat = "\\},[^,]+,[^,]+,([0-9]+)," :: String++monthName :: String -> String+monthName "0" = "January"+monthName "1" = "February"+monthName "2" = "March"+monthName "3" = "April"+monthName "4" = "May"+monthName "5" = "June"+monthName "6" = "July"+monthName "7" = "August"+monthName "8" = "September"+monthName "9" = "October"+monthName "10" = "November"+monthName "11" = "December"++processPoint :: [String] -> (Integer, String, Integer)+processPoint (year:month:count:[]) = (read year :: Integer, monthName month, read count :: Integer)++queryTrendsWithLogin :: String -> String -> String -> IO (Maybe [(Integer, String, Integer)])+queryTrendsWithLogin email pass keywords = do+ doLogin email pass continue fail+ where continue h s = do + results <- queryTrends keywords h s+ return $ Just results+ fail _ _ = return Nothing+
+ LICENSE view
@@ -0,0 +1,1 @@+This code is public domain. You can do whatever you want with it.
+ README.md view
@@ -0,0 +1,21 @@+GOOGLE TRENDS+-------------++Very simple library for accessing Google Trends++HOW TO USE+----------++Currently library exports only one function.++### queryTrendsWithLogin :: String -> String -> String -> IO (Maybe [(Integer, String, Integer)])+`Returns maybe a list of tuples of form (Year, Month, Value)`++**Example:**+```haskell+import Google.Trends++main = do+ Just results <- queryTrendsWithLogin "your-login@gmail.com" "password" "pizza"+ print [(month, value) | (year, month, value) <- results, year == 2010]+```
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ haskell-google-trends.cabal view
@@ -0,0 +1,23 @@+-- Initial haskell-google-trends.cabal generated by cabal init. For +-- further documentation, see http://haskell.org/cabal/users-guide/++name: haskell-google-trends+version: 0.0.1+synopsis: Simple library for accessing Google Trends+license: PublicDomain+license-file: LICENSE+author: grzegorzgoldapl+maintainer: contact@grzegorzgolda.com+-- copyright: +category: Web+build-type: Simple+extra-source-files: README.md +cabal-version: >=1.10++library+ exposed-modules: Google.Trends+ -- other-modules: + -- other-extensions: + build-depends: base >=4.8 && <4.10, haskell-fake-user-agent >= 0.0.2, wreq, lens, bytestring, tagsoup, text, regex-base, regex-posix+ -- hs-source-dirs: + default-language: Haskell2010