toktok-0.5: Toktok/Lexer.hs

module Toktok.Lexer where

import Toktok.Trie
import Toktok.Transducer
import Toktok.Sandhi

type Lexer = String -> [[String]]

mkLexer :: [String] -> Lexer
mkLexer = apply . fromList

mkLexerWithSandhis :: [Sandhi] -> [String] -> Lexer
mkLexerWithSandhis sandhis strings 
   = applyTransducer $ flip mkTransducer sandhis $ mkTrie strings