diff --git a/bench/Bench.hs b/bench/Bench.hs --- a/bench/Bench.hs +++ b/bench/Bench.hs @@ -2,9 +2,9 @@ import Criterion.Main import qualified Data.ByteString as BS +import Text.Megaparsec +import Web.Tweet.Parser import Web.Tweet.Parser.FastParser -import Web.Tweet.Parser -import Text.Megaparsec setupEnv = BS.readFile "test/data" setupEnv' = readFile "test/data" diff --git a/src/Web/Tweet/API.hs b/src/Web/Tweet/API.hs --- a/src/Web/Tweet/API.hs +++ b/src/Web/Tweet/API.hs @@ -3,10 +3,10 @@ -- | Module containing the functions directly dealing with twitter's API. Most functions in this module have two versions - one which takes a path to a TOML file containing api keys/secrets and tokens/secrets, the other takes api keys/secrets and tokens/secrets as an argument. module Web.Tweet.API where +import Control.Composition import Control.Lens import Control.Monad import qualified Data.ByteString.Lazy.Char8 as BSL -import Data.Composition import Data.Void import Text.Megaparsec.Error import Web.Tweet.Types diff --git a/src/Web/Tweet/Parser.hs b/src/Web/Tweet/Parser.hs --- a/src/Web/Tweet/Parser.hs +++ b/src/Web/Tweet/Parser.hs @@ -4,26 +4,23 @@ , getData ) where import qualified Data.ByteString as BS -import qualified Data.ByteString.Char8 as BSW -import Text.Megaparsec import Text.Megaparsec.Byte import Text.Megaparsec.Byte.Lexer as L import Text.Megaparsec import Web.Tweet.Types import Data.Monoid import qualified Data.Map as M -import Data.Maybe import qualified Data.Text as T import qualified Data.Text.Encoding as TE import Control.Monad +import Data.Maybe import Data.Void -import Data.Word type Parser = Parsec Void String -- | Parse some number of tweets parseTweet :: Parser Timeline -parseTweet = many (try getData <|> (const (TweetEntity "" "" "" 0 Nothing 0 0) <$> eof)) +parseTweet = many (try getData <|> (const (TweetEntity "" "" "" 0 mempty Nothing 0 0) <$> eof)) -- | Parse a single tweet's: name, text, fave count, retweet count getData :: Parser TweetEntity @@ -33,17 +30,20 @@ skipMentions name <- filterStr "name" screenName' <- filterStr "screen_name" + --withheldCountries <- (catMaybes . sequence) <$> optional filterList + let withheldCountries = mempty + --let toBlock = "DE" `elem` (catMaybes (sequence bannedList)) isQuote <- filterStr "is_quote_status" case isQuote of "false" -> do rts <- read <$> filterStr "retweet_count" faves <- read <$> filterStr "favorite_count" - pure (TweetEntity text name screenName' id Nothing rts faves) + pure (TweetEntity text name screenName' id withheldCountries Nothing rts faves) "true" -> do quoted <- parseQuoted rts <- read <$> filterStr "retweet_count" faves <- read <$> filterStr "favorite_count" - pure $ TweetEntity text name screenName' id quoted rts faves + pure $ TweetEntity text name screenName' id withheldCountries quoted rts faves -- | Parse a the quoted tweet parseQuoted :: Parser (Maybe TweetEntity) @@ -75,6 +75,19 @@ char ',' filterTag str +filterList :: Parser [String] +filterList = try $ do + many $ try $ anyChar >> notFollowedBy (string ("\"withheld_in_countries\":")) + char ',' + filterWithheld + +filterWithheld :: Parser [String] +filterWithheld = do + string "\"withheld_in_countries\":[\"" + codes <- some $ do { char '\"' ; interior <- many (noneOf ("\"]" :: String)) ; char '\"' ; optional (char ',') ; pure interior } + char ']' + pure codes + -- | Parse a field given its tag filterTag :: String -> Parser String filterTag str = do @@ -123,7 +136,6 @@ pure . (\case (Just a) -> a Nothing -> '?') $ M.lookup innards (M.fromList [("amp",'&'),("gt",'>'),("lt",'<'),("quot",'"'),("euro",'€'),("ndash",'–'),("mdash",'—')]) - -- | Parse escaped characters specialChar :: Char -> Parser Char diff --git a/src/Web/Tweet/Parser/FastParser.hs b/src/Web/Tweet/Parser/FastParser.hs --- a/src/Web/Tweet/Parser/FastParser.hs +++ b/src/Web/Tweet/Parser/FastParser.hs @@ -32,7 +32,7 @@ instance FromJSON User fromFast :: FastTweet -> TweetEntity -fromFast FastTweet{..} = TweetEntity (unpack text) (unpack . name $ user) (unpack . screen_name $ user) id (fmap fromFast quoted_status) retweet_count favorite_count +fromFast FastTweet{..} = TweetEntity (unpack text) (unpack . name $ user) (unpack . screen_name $ user) id mempty (fmap fromFast quoted_status) retweet_count favorite_count fastParse :: BS.ByteString -> Either String [FastTweet] fastParse = eitherDecode . BSL.fromStrict diff --git a/src/Web/Tweet/Types.hs b/src/Web/Tweet/Types.hs --- a/src/Web/Tweet/Types.hs +++ b/src/Web/Tweet/Types.hs @@ -1,5 +1,5 @@ -{-# LANGUAGE DeriveGeneric #-} -{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric #-} -- | Exports the `Tweet` type, a datatype for building tweets easily module Web.Tweet.Types where @@ -18,13 +18,14 @@ -- | Data type for tweets as they are returned data TweetEntity = TweetEntity - { _text :: String - , _name :: String + { _text :: String + , _name :: String , _screenName :: String - , _tweetId :: Int - , _quoted :: Maybe TweetEntity - , _retweets :: Int - , _favorites :: Int + , _tweetId :: Int + , _withheld :: [String] + , _quoted :: Maybe TweetEntity + , _retweets :: Int + , _favorites :: Int } deriving (Generic, Default, Eq, Show) -- | Stores data like (name, text, favoriteCount, retweetCount) @@ -58,17 +59,17 @@ screenName f tweet@TweetEntity { _screenName = scr } = fmap (\scr' -> tweet { _screenName = scr'}) (f scr) -- | Lens for `TweetEntity` accessing the `_tweetId` field. -tweetId :: Lens' TweetEntity Int +tweetId :: Lens' TweetEntity Int tweetId f tweet@TweetEntity { _tweetId = tw } = fmap (\tw' -> tweet { _tweetId = tw'}) (f tw) -- | Lens for `TweetEntity` accessing the `_quoted` field. -quoted :: Lens' TweetEntity (Maybe TweetEntity) +quoted :: Lens' TweetEntity (Maybe TweetEntity) quoted f tweet@TweetEntity { _quoted = quot } = fmap (\quot' -> tweet { _quoted = quot'}) (f quot) -- | Lens for `TweetEntity` accessing the `_retweets` field. -retweets :: Lens' TweetEntity Int +retweets :: Lens' TweetEntity Int retweets f tweet@TweetEntity { _retweets = rts } = fmap (\rts' -> tweet { _retweets = rts'}) (f rts) -- | Lens for `TweetEntity` accessing the `_favorites` field. -favorites :: Lens' TweetEntity Int +favorites :: Lens' TweetEntity Int favorites f tweet@TweetEntity { _favorites = fav } = fmap (\fav' -> tweet { _favorites = fav'}) (f fav) diff --git a/src/Web/Tweet/Utils.hs b/src/Web/Tweet/Utils.hs --- a/src/Web/Tweet/Utils.hs +++ b/src/Web/Tweet/Utils.hs @@ -10,6 +10,7 @@ , bird , getConfigData ) where +import Control.Composition import Control.Lens hiding (noneOf) import qualified Data.ByteString as BS2 import qualified Data.ByteString.Char8 as BS @@ -53,7 +54,7 @@ -- | Display Timeline without color displayTimeline :: Timeline -> String -displayTimeline ((TweetEntity content user screenName idTweet Nothing rts fave):rest) = concat [user +displayTimeline ((TweetEntity content user screenName idTweet _ Nothing rts fave):rest) = concat [user , " (" , screenName , ")" @@ -70,7 +71,7 @@ , show idTweet ,"\n\n" ,displayTimeline rest] -displayTimeline ((TweetEntity content user screenName idTweet (Just quoted) rts fave):rest) = concat [user +displayTimeline ((TweetEntity content user screenName idTweet _ (Just quoted) rts fave):rest) = concat [user , " (" , screenName , ")" @@ -99,7 +100,7 @@ -- | Display Timeline in color displayTimelineColor :: Timeline -> String -displayTimelineColor ((TweetEntity content user screenName idTweet Nothing rts fave):rest) = concat [toYellow user +displayTimelineColor ((TweetEntity content user screenName idTweet countries Nothing rts fave):rest) = concat [toYellow user , " (" , screenName , ")" @@ -115,7 +116,7 @@ , toBlue (show idTweet) , "\n\n" , displayTimelineColor rest] -displayTimelineColor ((TweetEntity content user screenName idTweet (Just quoted) rts fave):rest) = concat [toYellow user +displayTimelineColor ((TweetEntity content user screenName idTweet countries (Just quoted) rts fave):rest) = concat [toYellow user , " (" , screenName , ")" @@ -147,7 +148,7 @@ -- | sort tweets by most successful sortTweets :: Timeline -> Timeline sortTweets = sortBy compareTweet - where compareTweet (TweetEntity _ _ _ _ _ r1 f1) (TweetEntity _ _ _ _ _ r2 f2) = compare (2*r2 + f2) (2*r1 + f1) + where compareTweet (TweetEntity _ _ _ _ _ _ r1 f1) (TweetEntity _ _ _ _ _ _ r2 f2) = compare (2*r2 + f2) (2*r1 + f1) -- | helper function to get the key as read from a file keyLinePie :: String -> String @@ -155,7 +156,7 @@ -- | Pick out a key value from a key lineByKey :: BS.ByteString -> [(BS.ByteString, BS.ByteString)] -> BS.ByteString -lineByKey key = snd . head . (filter (\i -> fst i == key)) +lineByKey = snd .* head .* (filter . (fst -.* (==))) -- | Filter a line of a file for only the actual data and no descriptors filterLine :: String -> String diff --git a/src/Web/Tweet/Utils/Colors.hs b/src/Web/Tweet/Utils/Colors.hs --- a/src/Web/Tweet/Utils/Colors.hs +++ b/src/Web/Tweet/Utils/Colors.hs @@ -1,9 +1,10 @@ -- | Helper functions to color strings module Web.Tweet.Utils.Colors where -import Text.PrettyPrint.ANSI.Leijen +import Text.PrettyPrint.ANSI.Leijen -- 😎 +-- 😐 -- | Make a string red toRed :: String -> String diff --git a/tweet-hs.cabal b/tweet-hs.cabal --- a/tweet-hs.cabal +++ b/tweet-hs.cabal @@ -1,13 +1,13 @@ name: tweet-hs -version: 1.0.1.2 +version: 1.0.1.3 synopsis: Command-line tool for twitter description: a Command Line Interface Tweeter homepage: https://github.com/vmchale/command-line-tweeter#readme license: BSD3 license-file: LICENSE author: Vanessa McHale -maintainer: tmchale@wisc.edu -copyright: 2016 Vanessa McHale +maintainer: vanessa.mchale@reconfigure.io +copyright: 2016, 2017 Vanessa McHale category: Web build-type: Simple stability: stable @@ -65,8 +65,8 @@ , containers , ansi-wl-pprint , directory + , composition-prelude , extra - , composition , aeson default-language: Haskell2010 default-extensions: LambdaCase