sequor-0.1: src/CorpusReader.hs

{-# LANGUAGE OverloadedStrings #-}
module CorpusReader 
    ( Token
    , corpus
    , corpusLabeled 
    , fromWords
    )
where
import ListZipper 
import Utils (splitWith)
import qualified Text
import Text (Txt)
import Data.Maybe (isJust)


type Token = [Txt]

corpus :: Txt -> [[ListZipper Token]]
corpus =
      map toZippers
    . map (map $ parseFields)
    . splitWith null
    . map Text.words
    . Text.lines 

corpusLabeled ::Txt -> [([ListZipper Token], [Txt])]
corpusLabeled = 
      map (\xys -> let (xs,ys) = unzip xys in (toZippers xs,ys))
    . map (map $ parseFieldsLabeled)
    . splitWith null
    . map Text.words
    . Text.lines
 
fromWords :: [Txt] -> [ListZipper Token] 
fromWords = toZippers . map (\ w -> [w])

parseFieldsLabeled :: [Txt] -> (Token, Txt)
parseFieldsLabeled ws = (init ws,last ws)

parseFields :: [Txt] -> Token
parseFields ws = ws

toZippers :: [a] -> [ListZipper a]
toZippers = takeWhile (isJust . focus) . iterate next . fromList