bloodhound-1.0.0.0: tests/Test/AnalyzeSpec.hs

{-# LANGUAGE OverloadedStrings #-}
{-# OPTIONS_GHC -Wno-incomplete-uni-patterns -Wno-x-partial #-}

module Test.AnalyzeSpec (spec) where

import Data.Aeson.Key (fromText)
import Data.Aeson.Types (parseMaybe)
import Data.ByteString.Lazy.Char8 qualified as BL8
import TestsUtils.Common
import TestsUtils.Import

spec :: Spec
spec = do
  describe "Analyze JSON round-trip" $ do
    let lookupField :: Value -> Text -> Maybe Value
        lookupField json field = parseMaybe (withObject "req" $ \o -> o .: fromText field) json

    it "encodes a minimal AnalyzeRequest (text only)" $
      let req =
            AnalyzeRequest
              { analyzeRequestText = ["foo"],
                analyzeRequestAnalyzer = Just "standard",
                analyzeRequestTokenizer = Nothing,
                analyzeRequestFilter = Nothing,
                analyzeRequestCharFilter = Nothing,
                analyzeRequestField = Nothing,
                analyzeRequestNormalizer = Nothing
              }
          json = toJSON req
       in do
            lookupField json "text" `shouldBe` Just (toJSON ["foo" :: Text])
            lookupField json "analyzer" `shouldBe` Just (String "standard")

    it "omits optional fields that are Nothing" $
      let req =
            AnalyzeRequest
              { analyzeRequestText = ["foo"],
                analyzeRequestAnalyzer = Nothing,
                analyzeRequestTokenizer = Nothing,
                analyzeRequestFilter = Nothing,
                analyzeRequestCharFilter = Nothing,
                analyzeRequestField = Nothing,
                analyzeRequestNormalizer = Nothing
              }
          json = toJSON req
       in do
            lookupField json "tokenizer" `shouldBe` (Nothing :: Maybe Value)
            lookupField json "analyzer" `shouldBe` (Nothing :: Maybe Value)
            lookupField json "filter" `shouldBe` (Nothing :: Maybe Value)

    it "includes filter, char_filter and field when provided" $
      let req =
            AnalyzeRequest
              { analyzeRequestText = ["foo"],
                analyzeRequestAnalyzer = Nothing,
                analyzeRequestTokenizer = Just "standard",
                analyzeRequestFilter = Just ["lowercase", "stop"],
                analyzeRequestCharFilter = Just ["html_strip"],
                analyzeRequestField = Just (FieldName "message"),
                analyzeRequestNormalizer = Nothing
              }
          json = toJSON req
       in do
            lookupField json "filter" `shouldBe` Just (toJSON ["lowercase", "stop" :: Text])
            lookupField json "char_filter" `shouldBe` Just (toJSON ["html_strip" :: Text])
            lookupField json "field" `shouldBe` Just (String "message")

    it "includes normalizer when provided" $
      let req =
            AnalyzeRequest
              { analyzeRequestText = ["foo"],
                analyzeRequestAnalyzer = Nothing,
                analyzeRequestTokenizer = Nothing,
                analyzeRequestFilter = Nothing,
                analyzeRequestCharFilter = Nothing,
                analyzeRequestField = Nothing,
                analyzeRequestNormalizer = Just "lowercase"
              }
          json = toJSON req
       in lookupField json "normalizer" `shouldBe` Just (String "lowercase")

    it "decodes an Elasticsearch AnalyzeResponse with tokens" $
      let bytes =
            BL8.pack
              "{\"tokens\":[{\"token\":\"foo\",\"start_offset\":0,\"end_offset\":3,\"type\":\"<ALPHANUM>\",\"position\":0},{\"token\":\"bar\",\"start_offset\":4,\"end_offset\":7,\"type\":\"<ALPHANUM>\",\"position\":1}]}"
          Just resp = decode bytes
       in do
            length (analyzeResponseTokens resp) `shouldBe` 2
            analyzeTokenToken (head (analyzeResponseTokens resp)) `shouldBe` "foo"
            analyzeTokenPosition (analyzeResponseTokens resp !! 1) `shouldBe` 1

    it "decodes an OpenSearch-style token with a kind field" $
      let bytes =
            BL8.pack
              "{\"tokens\":[{\"token\":\"foo\",\"start_offset\":0,\"end_offset\":3,\"type\":\"ALPHANUM\",\"position\":0,\"kind\":\"WORD\"}]}"
          Just resp = decode bytes
       in analyzeTokenKind (head (analyzeResponseTokens resp)) `shouldBe` TokenKindWord

    it "decodes an unknown kind value as TokenKindOther preserving the string" $
      let bytes =
            BL8.pack
              "{\"tokens\":[{\"token\":\"foo\",\"start_offset\":0,\"end_offset\":3,\"type\":\"ALPHANUM\",\"position\":0,\"kind\":\"CUSTOM\"}]}"
          Just resp = decode bytes
       in analyzeTokenKind (head (analyzeResponseTokens resp)) `shouldBe` TokenKindOther "CUSTOM"

  describe "Analyze endpoint" $ do
    it "analyzes text with the built-in standard analyzer (lowercases) via POST /_analyze" $
      withTestEnv $ do
        let req =
              AnalyzeRequest
                { analyzeRequestText = ["Foo Bar"],
                  analyzeRequestAnalyzer = Just "standard",
                  analyzeRequestTokenizer = Nothing,
                  analyzeRequestFilter = Nothing,
                  analyzeRequestCharFilter = Nothing,
                  analyzeRequestField = Nothing,
                  analyzeRequestNormalizer = Nothing
                }
        resp <- performBHRequest $ analyzeText Nothing req
        let toks = analyzeResponseTokens resp
        liftIO $ do
          length toks `shouldBe` 2
          map analyzeTokenToken toks `shouldBe` ["foo", "bar"]
          map analyzeTokenPosition toks `shouldBe` [0, 1]
          map analyzeTokenStartOffset toks `shouldBe` [0, 4]
          map analyzeTokenEndOffset toks `shouldBe` [3, 7]

    it "preserves case with the whitespace analyzer via POST /_analyze" $
      withTestEnv $ do
        let req =
              AnalyzeRequest
                { analyzeRequestText = ["Foo Bar"],
                  analyzeRequestAnalyzer = Just "whitespace",
                  analyzeRequestTokenizer = Nothing,
                  analyzeRequestFilter = Nothing,
                  analyzeRequestCharFilter = Nothing,
                  analyzeRequestField = Nothing,
                  analyzeRequestNormalizer = Nothing
                }
        resp <- performBHRequest $ analyzeText Nothing req
        let toks = analyzeResponseTokens resp
        liftIO $ map analyzeTokenToken toks `shouldBe` ["Foo", "Bar"]

    it "analyzes text with a lowercase filter chain via POST /_analyze" $
      withTestEnv $ do
        let req =
              AnalyzeRequest
                { analyzeRequestText = ["Foo Bar"],
                  analyzeRequestAnalyzer = Nothing,
                  analyzeRequestTokenizer = Just "standard",
                  analyzeRequestFilter = Just ["lowercase"],
                  analyzeRequestCharFilter = Nothing,
                  analyzeRequestField = Nothing,
                  analyzeRequestNormalizer = Nothing
                }
        resp <- performBHRequest $ analyzeText Nothing req
        let toks = analyzeResponseTokens resp
        liftIO $ do
          length toks `shouldBe` 2
          map analyzeTokenToken toks `shouldBe` ["foo", "bar"]

    it "returns no tokens for an empty text value" $
      withTestEnv $ do
        let req =
              AnalyzeRequest
                { analyzeRequestText = [""],
                  analyzeRequestAnalyzer = Just "standard",
                  analyzeRequestTokenizer = Nothing,
                  analyzeRequestFilter = Nothing,
                  analyzeRequestCharFilter = Nothing,
                  analyzeRequestField = Nothing,
                  analyzeRequestNormalizer = Nothing
                }
        resp <- performBHRequest $ analyzeText Nothing req
        liftIO $ analyzeResponseTokens resp `shouldBe` []

    it "analyzes text via POST /{index}/_analyze against an existing index" $
      withTestEnv $ do
        _ <- createExampleIndex
        let req =
              AnalyzeRequest
                { analyzeRequestText = ["Foo Bar"],
                  analyzeRequestAnalyzer = Just "standard",
                  analyzeRequestTokenizer = Nothing,
                  analyzeRequestFilter = Nothing,
                  analyzeRequestCharFilter = Nothing,
                  analyzeRequestField = Nothing,
                  analyzeRequestNormalizer = Nothing
                }
        resp <- performBHRequest $ analyzeText (Just testIndex) req
        _ <- deleteExampleIndex
        let toks = analyzeResponseTokens resp
        liftIO $ do
          length toks `shouldBe` 2
          map analyzeTokenToken toks `shouldBe` ["foo", "bar"]