bloodhound-1.0.0.0: tests/Test/EsQueryLanguageSpec.hs
{-# LANGUAGE OverloadedStrings #-}
module Test.EsQueryLanguageSpec (spec) where
import Data.Aeson
import Data.Aeson.Key (Key)
import Data.Aeson.KeyMap qualified as KM
import Data.ByteString.Lazy.Char8 qualified as LBS
import Data.Maybe (isJust, isNothing)
import Data.Text qualified as T
import Database.Bloodhound.Client.Cluster (BackendType (..))
import Database.Bloodhound.ElasticSearch8.Client qualified as ClientES8
import Database.Bloodhound.ElasticSearch8.Requests qualified as RequestsES8
import Database.Bloodhound.ElasticSearch8.Types qualified as ES8Types
import Database.Bloodhound.ElasticSearch9.Client qualified as ClientES9
import Database.Bloodhound.ElasticSearch9.Types qualified as ES9Types
import TestsUtils.Common
import TestsUtils.Import
hasKey :: Key -> LBS.ByteString -> Bool
hasKey k bs = case decode bs of
Just (Object obj) -> k `KM.member` obj
_ -> False
-- | A minimal 'ESQLRequest' with every optional field set to 'Nothing'.
-- Produces a body containing only the @query@ key.
minimalReq :: Text -> ES8Types.ESQLRequest
minimalReq q =
ES8Types.ESQLRequest
{ esqlQuery = q,
esqlLocale = Nothing,
esqlFilter = Nothing,
esqlParams = Nothing,
esqlColumnar = Nothing,
esqlProfile = Nothing,
esqlTables = Nothing,
esqlIncludeCcsMetadata = Nothing
}
minimalES9Req :: Text -> ES9Types.ESQLRequest
minimalES9Req q =
ES9Types.ESQLRequest
{ esqlQuery = q,
esqlLocale = Nothing,
esqlTimeZone = Nothing,
esqlFilter = Nothing,
esqlParams = Nothing,
esqlColumnar = Nothing,
esqlProfile = Nothing,
esqlTables = Nothing,
esqlIncludeCcsMetadata = Nothing,
esqlIncludeExecutionMetadata = Nothing
}
spec :: Spec
spec = describe "ES|QL Query API (POST /_query)" $ do
describe "ESQLRequest JSON" $ do
it "serializes the required query field" $ do
let json = encode (minimalReq "FROM logs | LIMIT 10")
json `shouldSatisfy` hasKey "query"
it "omits all optional Nothing fields" $ do
let json = encode (minimalReq "FROM logs | LIMIT 10")
json `shouldNotSatisfy` hasKey "locale"
json `shouldNotSatisfy` hasKey "time_zone"
json `shouldNotSatisfy` hasKey "filter"
json `shouldNotSatisfy` hasKey "params"
json `shouldNotSatisfy` hasKey "columnar"
json `shouldNotSatisfy` hasKey "profile"
json `shouldNotSatisfy` hasKey "tables"
json `shouldNotSatisfy` hasKey "include_ccs_metadata"
json `shouldNotSatisfy` hasKey "include_execution_metadata"
it "never emits the removed field_multi_value_leniency key" $ do
let json = encode (minimalReq "FROM logs | LIMIT 10")
json `shouldNotSatisfy` hasKey "field_multi_value_leniency"
it "includes the original optional fields when set" $ do
let req =
ES8Types.ESQLRequest
{ esqlQuery = "FROM logs | LIMIT 10",
esqlLocale = Just "en-US",
esqlFilter = Just (Filter (TermQuery (Term "user" "alice") Nothing)),
esqlParams = Just [toJSON ("foo" :: Text), toJSON (42 :: Int)],
esqlColumnar = Nothing,
esqlProfile = Nothing,
esqlTables = Nothing,
esqlIncludeCcsMetadata = Nothing
}
let json = encode req
json `shouldSatisfy` hasKey "locale"
json `shouldSatisfy` hasKey "filter"
json `shouldSatisfy` hasKey "params"
it "includes the new body fields when set" $ do
let req =
ES8Types.ESQLRequest
{ esqlQuery = "FROM logs | LIMIT 10",
esqlLocale = Nothing,
esqlFilter = Nothing,
esqlParams = Nothing,
esqlColumnar = Just True,
esqlProfile = Just True,
esqlTables = Just (object ["colors" .= [object ["id" .= (1 :: Int), "name" .= ("red" :: Text)]]]),
esqlIncludeCcsMetadata = Just True
}
let json = encode req
json `shouldSatisfy` hasKey "columnar"
json `shouldSatisfy` hasKey "profile"
json `shouldSatisfy` hasKey "tables"
json `shouldSatisfy` hasKey "include_ccs_metadata"
it "round-trips a minimal request through ToJSON/FromJSON" $ do
let req = minimalReq "FROM logs | LIMIT 10"
decode (encode req) `shouldBe` Just req
it "round-trips a fully-populated request through ToJSON/FromJSON" $ do
let req =
ES8Types.ESQLRequest
{ esqlQuery = "FROM logs | LIMIT 10",
esqlLocale = Just "en-US",
esqlFilter = Just (Filter (TermQuery (Term "user" "alice") Nothing)),
esqlParams = Just [toJSON ("foo" :: Text), toJSON (42 :: Int)],
esqlColumnar = Just True,
esqlProfile = Just False,
esqlTables = Just (object ["colors" .= [object ["id" .= (1 :: Int)]]]),
esqlIncludeCcsMetadata = Just True
}
decode (encode req) `shouldBe` Just req
describe "ES9 ESQLRequest JSON (time_zone + include_execution_metadata)" $ do
it "ES9: omits time_zone and include_execution_metadata when Nothing" $ do
let json = encode (minimalES9Req "FROM logs | LIMIT 10")
json `shouldNotSatisfy` hasKey "time_zone"
json `shouldNotSatisfy` hasKey "include_execution_metadata"
json `shouldSatisfy` hasKey "query"
it "ES9: emits time_zone when set" $ do
let req = (minimalES9Req "FROM x") {ES9Types.esqlTimeZone = Just "UTC"}
encode req `shouldSatisfy` hasKey "time_zone"
it "ES9: emits include_execution_metadata when set" $ do
let req = (minimalES9Req "FROM x") {ES9Types.esqlIncludeExecutionMetadata = Just True}
encode req `shouldSatisfy` hasKey "include_execution_metadata"
it "ES9: round-trips a fully-populated request through ToJSON/FromJSON" $ do
let req =
ES9Types.ESQLRequest
{ esqlQuery = "FROM logs | LIMIT 10",
esqlLocale = Just "en-US",
esqlTimeZone = Just "UTC",
esqlFilter = Just (Filter (TermQuery (Term "user" "alice") Nothing)),
esqlParams = Just [toJSON ("foo" :: Text), toJSON (42 :: Int)],
esqlColumnar = Just True,
esqlProfile = Just False,
esqlTables = Just (object ["colors" .= [object ["id" .= (1 :: Int)]]]),
esqlIncludeCcsMetadata = Just True,
esqlIncludeExecutionMetadata = Just True
}
decode (encode req) `shouldBe` Just req
it "ES9: AsyncESQLRequest forwards time_zone and include_execution_metadata" $ do
let base =
(minimalES9Req "FROM logs | LIMIT 10")
{ ES9Types.esqlTimeZone = Just "UTC",
ES9Types.esqlIncludeExecutionMetadata = Just True
}
areq =
(ES9Types.mkAsyncESQLRequest base)
{ ES9Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s"
}
json = encode areq
json `shouldSatisfy` hasKey "query"
json `shouldSatisfy` hasKey "time_zone"
json `shouldSatisfy` hasKey "include_execution_metadata"
json `shouldSatisfy` hasKey "wait_for_completion_timeout"
describe "ESQLResult JSON" $ do
it "decodes a complete synchronous response with all spec fields" $ do
let raw =
LBS.pack
"{\"is_partial\":false,\"all_columns\":true,\"columns\":[{\"name\":\"user\",\"type\":\"keyword\"},{\"name\":\"age\",\"type\":\"long\"}],\"values\":[[\"alice\",30],[\"bob\",40]],\"took\":5}"
let expected =
ES8Types.ESQLResult
{ esqlIsPartial = False,
esqlAllColumns = Just True,
esqlColumns =
[ ES8Types.ESQLColumn "user" "keyword",
ES8Types.ESQLColumn "age" "long"
],
esqlValues = [[String "alice", Number 30], [String "bob", Number 40]],
esqlClusters = Nothing,
esqlProfileValue = Nothing,
esqlTook = Just 5
}
decode raw `shouldBe` Just expected
it "decodes a response carrying _clusters and profile" $ do
let raw =
LBS.pack
"{\"is_partial\":false,\"columns\":[{\"name\":\"user\",\"type\":\"keyword\"}],\"values\":[[\"alice\"]],\"_clusters\":{\"total\":1,\"successful\":1,\"skipped\":0},\"profile\":{\"parsed\":{}},\"took\":7}"
let Just result = decode raw
ES8Types.esqlIsPartial result `shouldBe` False
ES8Types.esqlClusters result `shouldSatisfy` isJust
ES8Types.esqlProfileValue result `shouldSatisfy` isJust
let Just clusters = ES8Types.esqlClusters result
ES8Types.esqlClustersTotal clusters `shouldBe` 1
ES8Types.esqlClustersSuccessful clusters `shouldBe` 1
ES8Types.esqlClustersSkipped clusters `shouldBe` 0
-- The bare {"parsed":{}}} shape is captured verbatim in
-- esqlProfileOther since ES marks the profile payload as
-- unstable (no typed projections apply).
let Just profile = ES8Types.esqlProfileValue result
ES8Types.esqlProfileOther profile `shouldBe` object ["parsed" .= object []]
ES8Types.esqlTook result `shouldBe` Just 7
it "rejects a response missing the required is_partial key" $ do
let raw =
LBS.pack
"{\"columns\":[{\"name\":\"user\",\"type\":\"text\"}],\"values\":[[\"bitemyapp\"]]}"
decode raw `shouldSatisfy` (isNothing :: Maybe ES8Types.ESQLResult -> Bool)
it "rejects a response missing the required columns key" $ do
let raw =
LBS.pack "{\"is_partial\":false,\"values\":[]}"
decode raw `shouldSatisfy` (isNothing :: Maybe ES8Types.ESQLResult -> Bool)
it "rejects a response missing the required values key" $ do
let raw =
LBS.pack
"{\"is_partial\":false,\"columns\":[{\"name\":\"user\",\"type\":\"text\"}]}"
decode raw `shouldSatisfy` (isNothing :: Maybe ES8Types.ESQLResult -> Bool)
it "round-trips ESQLColumn through ToJSON/FromJSON" $ do
let col = ES8Types.ESQLColumn "user" "keyword"
decode (encode col) `shouldBe` Just col
it "never emits the removed total_rows or is_running keys" $ do
let r =
ES8Types.ESQLResult
{ esqlIsPartial = False,
esqlAllColumns = Nothing,
esqlColumns = [ES8Types.ESQLColumn "user" "keyword"],
esqlValues = [[String "alice"]],
esqlClusters = Nothing,
esqlProfileValue = Nothing,
esqlTook = Just 5
}
json = encode r
json `shouldNotSatisfy` hasKey "total_rows"
json `shouldNotSatisfy` hasKey "is_running"
it "round-trips a fully-populated ESQLResult through ToJSON/FromJSON" $ do
-- The @_clusters@/@profile@ records use the forward-compat @Other
-- Value@ pattern (see 'Database.Bloodhound.Internal.Versions.Common.Types.ClusterStats'):
-- decode captures the whole input object in @*Other@, so the pair
-- is idempotent (decode . encode . decode == decode) rather than
-- equality-preserving from a hand-built value. Assert idempotency
-- plus typed-field survival, mirroring 'Test.ClusterStatsSpec'.
let raw =
LBS.pack
"{\"is_partial\":false,\"all_columns\":true,\"columns\":[{\"name\":\"user\",\"type\":\"keyword\"}],\"values\":[[\"alice\",30]],\"_clusters\":{\"total\":2,\"successful\":1,\"running\":0,\"skipped\":0,\"partial\":0,\"failed\":1,\"details\":{\"remote_one\":{\"status\":\"successful\",\"indices\":\"logs-*\"}},\"future_key\":99},\"profile\":{\"parsed\":{}},\"took\":5}"
case decode raw of
Nothing -> expectationFailure "initial decode failed"
Just (firstDecode :: ES8Types.ESQLResult) ->
case decode (encode firstDecode) of
Nothing -> expectationFailure "re-decode failed"
Just again -> do
ES8Types.esqlIsPartial again `shouldBe` False
ES8Types.esqlTook again `shouldBe` Just 5
-- Typed _clusters fields survive.
let Just clusters = ES8Types.esqlClusters again
ES8Types.esqlClustersTotal clusters `shouldBe` 2
ES8Types.esqlClustersFailed clusters `shouldBe` 1
-- Forward-compat key preserved via esqlClustersOther.
case ES8Types.esqlClustersOther clusters of
Object o
| KM.member "future_key" o -> pure ()
_ -> expectationFailure "future_key dropped from _clusters round-trip"
-- Profile payload preserved verbatim.
let Just profile = ES8Types.esqlProfileValue again
case ES8Types.esqlProfileOther profile of
Object o
| KM.member "parsed" o -> pure ()
_ -> expectationFailure "parsed dropped from profile round-trip"
describe "EsqlClusters JSON" $ do
it "decodes the full documented shape (6 counters + details map)" $ do
let raw =
LBS.pack
"{\"total\":3,\"successful\":2,\"running\":0,\"skipped\":1,\"partial\":0,\"failed\":0,\"details\":{\"remote_one\":{\"status\":\"successful\",\"indices\":\"logs-*\",\"_shards\":{\"total\":4,\"successful\":4,\"skipped\":0,\"failed\":0}},\"remote_two\":{\"status\":\"skipped\",\"indices\":\"audit-*\"}}}"
Just clusters = decode raw
ES8Types.esqlClustersTotal clusters `shouldBe` 3
ES8Types.esqlClustersSuccessful clusters `shouldBe` 2
ES8Types.esqlClustersRunning clusters `shouldBe` 0
ES8Types.esqlClustersSkipped clusters `shouldBe` 1
ES8Types.esqlClustersPartial clusters `shouldBe` 0
ES8Types.esqlClustersFailed clusters `shouldBe` 0
-- Per-cluster entries decode in document order.
let details = ES8Types.esqlClustersDetails clusters
length details `shouldBe` 2
fst (head details) `shouldBe` "remote_one"
snd (head details)
`shouldBe` ES8Types.EsqlClusterDetails
{ esqlClusterDetailsStatus = Just ES8Types.EsqlClusterStatusSuccessful,
esqlClusterDetailsIndices = Just "logs-*",
esqlClusterDetailsShards =
Just
( ShardResult
{ shardTotal = 4,
shardsSuccessful = 4,
shardsSkipped = 0,
shardsFailed = 0
}
),
esqlClusterDetailsFailures = Nothing,
esqlClusterDetailsOther =
object
[ "status" .= ("successful" :: Text),
"indices" .= ("logs-*" :: Text),
"_shards"
.= object
[ "total" .= (4 :: Int),
"successful" .= (4 :: Int),
"skipped" .= (0 :: Int),
"failed" .= (0 :: Int)
]
]
}
it "decodes a minimal clusters object (missing counters and details default to 0/[])" $ do
let raw = LBS.pack "{\"total\":1,\"successful\":1,\"skipped\":0}"
Just clusters = decode raw
ES8Types.esqlClustersRunning clusters `shouldBe` 0
ES8Types.esqlClustersPartial clusters `shouldBe` 0
ES8Types.esqlClustersFailed clusters `shouldBe` 0
ES8Types.esqlClustersDetails clusters `shouldBe` []
it "preserves forward-compat keys through esqlClustersOther on a round trip" $ do
let raw =
LBS.pack
"{\"total\":1,\"successful\":1,\"running\":0,\"skipped\":0,\"partial\":0,\"failed\":0,\"details\":{},\"future_key\":42}"
Just clusters = decode raw :: Maybe ES8Types.EsqlClusters
-- Re-encoding must keep the unknown @future_key@ (captured in Other).
let roundTripped = encode clusters
roundTripped `shouldSatisfy` hasKey "future_key"
-- And the typed counters stay authoritative.
decode roundTripped `shouldBe` Just clusters
it "round-trips a fully-populated EsqlClusters (idempotent decode . encode . decode)" $ do
-- See the ESQLResult round-trip note: the forward-compat @Other
-- Value@ pattern is idempotent after the first decode, so we
-- verify (decode . encode . decode == decode) rather than
-- equality with a hand-built value.
let raw =
LBS.pack
"{\"total\":2,\"successful\":1,\"running\":0,\"skipped\":0,\"partial\":0,\"failed\":1,\"details\":{\"remote_one\":{\"status\":\"successful\",\"indices\":\"logs-*\"}},\"future_key\":99}"
case decode raw of
Nothing -> expectationFailure "initial decode failed"
Just (firstDecode :: ES8Types.EsqlClusters) -> do
case decode (encode firstDecode) of
Nothing -> expectationFailure "re-decode failed"
Just again -> do
again `shouldBe` firstDecode
ES8Types.esqlClustersTotal again `shouldBe` 2
ES8Types.esqlClustersFailed again `shouldBe` 1
case ES8Types.esqlClustersOther again of
Object o
| KM.member "future_key" o -> pure ()
_ -> expectationFailure "future_key dropped from round-trip"
describe "EsqlClusterStatus JSON" $ do
it "decodes every documented enum value" $ do
decode (encode ES8Types.EsqlClusterStatusRunning)
`shouldBe` Just ES8Types.EsqlClusterStatusRunning
decode (encode ES8Types.EsqlClusterStatusSuccessful)
`shouldBe` Just ES8Types.EsqlClusterStatusSuccessful
decode (encode ES8Types.EsqlClusterStatusPartial)
`shouldBe` Just ES8Types.EsqlClusterStatusPartial
decode (encode ES8Types.EsqlClusterStatusSkipped)
`shouldBe` Just ES8Types.EsqlClusterStatusSkipped
decode (encode ES8Types.EsqlClusterStatusFailed)
`shouldBe` Just ES8Types.EsqlClusterStatusFailed
it "encodes enum values as the documented wire strings" $ do
encode ES8Types.EsqlClusterStatusRunning `shouldBe` "\"running\""
encode ES8Types.EsqlClusterStatusSuccessful `shouldBe` "\"successful\""
encode ES8Types.EsqlClusterStatusPartial `shouldBe` "\"partial\""
encode ES8Types.EsqlClusterStatusSkipped `shouldBe` "\"skipped\""
encode ES8Types.EsqlClusterStatusFailed `shouldBe` "\"failed\""
it "round-trips an unknown status through OtherEsqlClusterStatus" $ do
let s = ES8Types.OtherEsqlClusterStatus "queued"
decode (encode s) `shouldBe` Just s
encode s `shouldBe` "\"queued\""
describe "EsqlProfile JSON" $ do
it "captures an unstable-shape payload verbatim in esqlProfileOther" $ do
let raw = LBS.pack "{\"parsed\":{\"query_type\":\"command\"}}"
Just profile = decode raw
ES8Types.esqlProfileShards profile `shouldBe` Nothing
ES8Types.esqlProfileProfiles profile `shouldBe` Nothing
ES8Types.esqlProfileOther profile
`shouldBe` object ["parsed" .= object ["query_type" .= ("command" :: Text)]]
it "decodes the documented Lucene-style projections when present" $ do
let raw =
LBS.pack
"{\"shards\":[{\"id\":0}],\"profiles\":[{\"phase\":\"query\",\"description\":\"compute\",\"time\":\"1ms\",\"children\":[]}]}"
Just profile = decode raw
ES8Types.esqlProfileShards profile `shouldBe` Just [object ["id" .= (0 :: Int)]]
let Just sections = ES8Types.esqlProfileProfiles profile
length sections `shouldBe` 1
let section = head sections
ES8Types.esqlProfileSectionPhase section `shouldBe` Just "query"
ES8Types.esqlProfileSectionDescription section `shouldBe` Just "compute"
ES8Types.esqlProfileSectionChildren section `shouldBe` Just []
it "round-trips an EsqlProfile preserving both typed and Other keys" $ do
let profile =
ES8Types.EsqlProfile
{ esqlProfileShards = Nothing,
esqlProfileProfiles = Nothing,
esqlProfileOther = object ["parsed" .= object []]
}
decode (encode profile) `shouldBe` Just profile
it "preserves populated typed projections through a decode/encode cycle" $ do
-- Exercises the encode path when esqlProfileProfiles is populated
-- (the decode-only test above does not re-encode). Uses the
-- idempotency property of the Other-Value pattern.
let raw =
LBS.pack
"{\"shards\":[{\"id\":0}],\"profiles\":[{\"phase\":\"query\",\"description\":\"compute\",\"time\":\"1ms\",\"children\":[{\"phase\":\"aggregation\",\"description\":\"stats\",\"time\":\"0ms\",\"children\":[]}]}]}"
case decode raw of
Nothing -> expectationFailure "initial decode failed"
Just (firstDecode :: ES8Types.EsqlProfile) -> do
case decode (encode firstDecode) of
Nothing -> expectationFailure "re-decode failed"
Just again -> do
again `shouldBe` firstDecode
-- Recursive children survive the round trip with their
-- typed projections intact.
case ES8Types.esqlProfileProfiles again of
Just (topSection : _)
| Just (child : _) <- ES8Types.esqlProfileSectionChildren topSection ->
ES8Types.esqlProfileSectionPhase child `shouldBe` Just "aggregation"
_ -> expectationFailure "expected nested profile children"
describe "EsqlClusterDetails JSON" $ do
it "preserves a populated _shards block through a decode/encode cycle" $ do
-- Exercises the encode path for the ShardResult projection (the
-- parent EsqlClusters decode test covers decode only).
let raw =
LBS.pack
"{\"status\":\"successful\",\"indices\":\"logs-*\",\"_shards\":{\"total\":4,\"successful\":3,\"skipped\":0,\"failed\":1},\"failures\":[{\"type\":\"shard_failed\",\"reason\":\"corrupt\"}]}"
case decode raw of
Nothing -> expectationFailure "initial decode failed"
Just (firstDecode :: ES8Types.EsqlClusterDetails) -> do
case decode (encode firstDecode) of
Nothing -> expectationFailure "re-decode failed"
Just again -> do
again `shouldBe` firstDecode
ES8Types.esqlClusterDetailsStatus again `shouldBe` Just ES8Types.EsqlClusterStatusSuccessful
let Just shards = ES8Types.esqlClusterDetailsShards again
shardsFailed shards `shouldBe` 1
ES8Types.esqlClusterDetailsFailures again
`shouldBe` Just [object ["type" .= ("shard_failed" :: Text), "reason" .= ("corrupt" :: Text)]]
it "re-encodes an empty details map as an empty object (not dropped)" $ do
let clusters =
ES8Types.EsqlClusters
{ esqlClustersTotal = 0,
esqlClustersSuccessful = 0,
esqlClustersRunning = 0,
esqlClustersSkipped = 0,
esqlClustersPartial = 0,
esqlClustersFailed = 0,
esqlClustersDetails = [],
esqlClustersOther = object ["details" .= object []]
}
encoded = encode clusters
encoded `shouldSatisfy` hasKey "details"
case decode encoded of
Just (again :: ES8Types.EsqlClusters) ->
ES8Types.esqlClustersDetails again `shouldBe` []
Nothing -> expectationFailure "re-decode failed"
describe "endpoint shape" $ do
it "esql POSTs to /_query with no query string" $ do
let req = RequestsES8.esql (minimalReq "FROM x")
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
it "esql embeds the query field in the body" $ do
let req = RequestsES8.esql (minimalReq "FROM x | LIMIT 1")
let body = bhRequestBody req
body `shouldSatisfy` isJust
let Just bodyBytes = body
hasKey "query" bodyBytes `shouldBe` True
it "esqlWith with default options is byte-identical to esql (no query string)" $ do
let r1 = RequestsES8.esql (minimalReq "FROM x")
let r2 = RequestsES8.esqlWith (minimalReq "FROM x") ES8Types.defaultESQLQueryOptions
getRawEndpoint (bhRequestEndpoint r1) `shouldBe` getRawEndpoint (bhRequestEndpoint r2)
getRawEndpointQueries (bhRequestEndpoint r1)
`shouldBe` getRawEndpointQueries (bhRequestEndpoint r2)
bhRequestBody r1 `shouldBe` bhRequestBody r2
it "esqlWith emits delimiter as a query parameter" $ do
let opts =
ES8Types.defaultESQLQueryOptions
{ ES8Types.esqloDelimiter = Just "|"
}
let req = RequestsES8.esqlWith (minimalReq "FROM x") opts
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query"]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldBe` [("delimiter", Just "|")]
it "esqlWith renders booleans as true/false strings" $ do
let opts =
ES8Types.defaultESQLQueryOptions
{ ES8Types.esqloDropNullColumns = Just True,
ES8Types.esqloAllowPartialResults = Just False
}
let req = RequestsES8.esqlWith (minimalReq "FROM x") opts
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("drop_null_columns", Just "true")]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("allow_partial_results", Just "false")]
describe "live integration (requires ES8+ backend)" $
backendSpecific [ElasticSearch8, ElasticSearch9] $ do
it "returns rows from an indexed document" $
withTestEnv $ do
_ <- insertData
result <- do
backend <- liftIO detectBackendType
let q = "FROM \"" <> unIndexName testIndex <> "\" | LIMIT 5"
if backend == Just ElasticSearch9
then ClientES9.esql (minimalES9Req q)
else ClientES8.esql (minimalReq q)
case result of
Left e -> liftIO $ expectationFailure $ "esql failed: " <> show e
Right esqlResult -> liftIO $ do
ES8Types.esqlIsPartial esqlResult `shouldBe` False
ES8Types.esqlColumns esqlResult `shouldNotBe` []
let values = ES8Types.esqlValues esqlResult
values `shouldNotBe` []
-- Canned tweet is `user = "bitemyapp"`; assert it appears in the rows
let flattened = concat values
flattened `shouldContain` [String "bitemyapp"]
describe "Async ES|QL API (POST/GET/DELETE /_query/async)" $ do
describe "AsyncESQLId JSON" $ do
it "round-trips as a bare JSON string" $ do
let i = ES8Types.AsyncESQLId "FmRleEct"
decode (encode i) `shouldBe` Just i
encode i `shouldBe` "\"FmRleEct\""
describe "AsyncESQLRequest JSON" $ do
it "merges async-only fields onto the base ESQLRequest body" $ do
let areq =
(ES8Types.mkAsyncESQLRequest (minimalReq "FROM logs | LIMIT 10"))
{ ES8Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s",
ES8Types.asyncESQLRequestKeepAlive = Just "10d",
ES8Types.asyncESQLRequestKeepOnCompletion = Just True
}
json = encode areq
json `shouldSatisfy` hasKey "query"
json `shouldSatisfy` hasKey "wait_for_completion_timeout"
json `shouldSatisfy` hasKey "keep_alive"
json `shouldSatisfy` hasKey "keep_on_completion"
it "omits async fields when they are Nothing" $ do
let areq = ES8Types.mkAsyncESQLRequest (minimalReq "FROM logs | LIMIT 10")
json = encode areq
json `shouldNotSatisfy` hasKey "wait_for_completion_timeout"
json `shouldNotSatisfy` hasKey "keep_alive"
json `shouldNotSatisfy` hasKey "keep_on_completion"
it "forwards base body fields (columnar/profile/ccs) onto the async body" $ do
let base =
ES8Types.ESQLRequest
{ esqlQuery = "FROM logs | LIMIT 10",
esqlLocale = Nothing,
esqlFilter = Nothing,
esqlParams = Nothing,
esqlColumnar = Just True,
esqlProfile = Just True,
esqlTables = Nothing,
esqlIncludeCcsMetadata = Just True
}
areq = ES8Types.mkAsyncESQLRequest base
json = encode areq
json `shouldSatisfy` hasKey "columnar"
json `shouldSatisfy` hasKey "profile"
json `shouldSatisfy` hasKey "include_ccs_metadata"
it "round-trips a fully-populated AsyncESQLRequest" $ do
let base =
ES8Types.ESQLRequest
{ esqlQuery = "FROM logs | LIMIT 10",
esqlLocale = Just "en-US",
esqlFilter = Just (Filter (TermQuery (Term "user" "alice") Nothing)),
esqlParams = Just [toJSON ("foo" :: Text), toJSON (42 :: Int)],
esqlColumnar = Just True,
esqlProfile = Just True,
esqlTables = Nothing,
esqlIncludeCcsMetadata = Just True
}
areq =
(ES8Types.mkAsyncESQLRequest base)
{ ES8Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s",
ES8Types.asyncESQLRequestKeepAlive = Just "10d",
ES8Types.asyncESQLRequestKeepOnCompletion = Just True
}
decode (encode areq) `shouldBe` Just areq
describe "AsyncESQLResult JSON" $ do
it "decodes a deferred submission (id present, query still running)" $ do
let raw =
LBS.pack
"{\"id\":\"FmRleEct\",\"is_running\":true,\"documents_found\":0,\"values_loaded\":0,\"start_time_in_millis\":1700000000000,\"expiration_time_in_millis\":1700000060000,\"columns\":[],\"values\":[]}"
Just result = decode raw
ES8Types.asyncESQLResultId result
`shouldBe` Just (ES8Types.AsyncESQLId "FmRleEct")
ES8Types.asyncESQLResultIsRunning result `shouldBe` True
ES8Types.asyncESQLResultStartTimeInMillis result `shouldBe` Just 1700000000000
ES8Types.asyncESQLResultExpirationTimeInMillis result `shouldBe` Just 1700000060000
ES8Types.asyncESQLResultDocumentsFound result `shouldBe` Just 0
ES8Types.asyncESQLResultValuesLoaded result `shouldBe` Just 0
ES8Types.asyncESQLResultCompletionTimeInMillis result `shouldSatisfy` isNothing
it "decodes a completed result with columns, values, completion_time and counters" $ do
let raw =
LBS.pack
"{\"id\":\"FmRleEct\",\"is_running\":false,\"is_partial\":false,\"completion_time_in_millis\":1700000010000,\"took\":42,\"documents_found\":2,\"values_loaded\":4,\"start_time_in_millis\":1700000000000,\"expiration_time_in_millis\":1700000060000,\"columns\":[{\"name\":\"user\",\"type\":\"keyword\"}],\"values\":[[\"alice\"]]}"
Just result = decode raw
ES8Types.asyncESQLResultIsRunning result `shouldBe` False
ES8Types.asyncESQLResultIsPartial result `shouldBe` False
ES8Types.asyncESQLResultCompletionTimeInMillis result `shouldBe` Just 1700000010000
ES8Types.asyncESQLResultTook result `shouldBe` Just 42
ES8Types.asyncESQLResultDocumentsFound result `shouldBe` Just 2
ES8Types.asyncESQLResultValuesLoaded result `shouldBe` Just 4
ES8Types.asyncESQLResultColumns result
`shouldBe` Just [ES8Types.ESQLColumn "user" "keyword"]
ES8Types.asyncESQLResultValues result `shouldBe` Just [[String "alice"]]
it "decodes a wait_for_completion_timeout synchronous-completion response (no id)" $ do
let raw =
LBS.pack
"{\"is_running\":false,\"is_partial\":false,\"completion_time_in_millis\":1700000010000,\"took\":7,\"documents_found\":1,\"values_loaded\":1,\"start_time_in_millis\":1700000000000,\"expiration_time_in_millis\":1700000060000,\"columns\":[{\"name\":\"user\",\"type\":\"keyword\"}],\"values\":[[\"bob\"]]}"
Just result = decode raw
ES8Types.asyncESQLResultId result `shouldSatisfy` isNothing
ES8Types.asyncESQLResultIsRunning result `shouldBe` False
ES8Types.asyncESQLResultValues result `shouldBe` Just [[String "bob"]]
it "decodes a profile section when one is attached" $ do
let raw =
LBS.pack
"{\"is_running\":false,\"is_partial\":false,\"took\":1,\"columns\":[],\"values\":[],\"profile\":{\"query\":{\"took_millis\":1}}}"
Just result = decode raw
ES8Types.asyncESQLResultProfile result
`shouldSatisfy` (isJust :: Maybe ES8Types.EsqlProfile -> Bool)
-- The {"query": {...}} payload has no typed projections (ES|QL
-- marks profile as unstable) so it round-trips via esqlProfileOther.
let Just profile = ES8Types.asyncESQLResultProfile result
ES8Types.esqlProfileOther profile
`shouldBe` object ["query" .= object ["took_millis" .= (1 :: Int)]]
it "decodes an attached _clusters object (async get CCS metadata)" $ do
let raw =
LBS.pack
"{\"is_running\":false,\"is_partial\":false,\"took\":1,\"columns\":[],\"values\":[],\"_clusters\":{\"total\":2,\"successful\":2,\"running\":0,\"skipped\":0,\"partial\":0,\"failed\":0,\"details\":{\"remote_one\":{\"status\":\"successful\",\"indices\":\"logs-*\"}}}}"
Just result = decode raw
let Just clusters = ES8Types.asyncESQLResultClusters result
ES8Types.esqlClustersTotal clusters `shouldBe` 2
ES8Types.esqlClustersSuccessful clusters `shouldBe` 2
ES8Types.esqlClustersDetails clusters
`shouldBe` [ ( "remote_one",
ES8Types.EsqlClusterDetails
{ esqlClusterDetailsStatus = Just ES8Types.EsqlClusterStatusSuccessful,
esqlClusterDetailsIndices = Just "logs-*",
esqlClusterDetailsShards = Nothing,
esqlClusterDetailsFailures = Nothing,
esqlClusterDetailsOther =
object
[ "status" .= ("successful" :: Text),
"indices" .= ("logs-*" :: Text)
]
}
)
]
it "defaults is_running to False and is_partial to True when keys are absent" $ do
let raw = LBS.pack "{\"id\":\"Fm_==_RxD_123\"}"
Just result = decode raw
ES8Types.asyncESQLResultIsRunning result `shouldBe` False
ES8Types.asyncESQLResultIsPartial result `shouldBe` True
it "rejects malformed input (non-object)" $
decode (LBS.pack "[1,2,3]") `shouldSatisfy` (isNothing :: Maybe ES8Types.AsyncESQLResult -> Bool)
describe "endpoint shape" $ do
it "esqlAsync POSTs to /_query/async with no query string (params go in the body)" $ do
let req =
RequestsES8.esqlAsync $
ES8Types.mkAsyncESQLRequest $
minimalReq "FROM x"
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
it "esqlAsync embeds wait_for_completion_timeout in the body when set" $ do
let areq =
( ES8Types.mkAsyncESQLRequest $
minimalReq "FROM x"
)
{ ES8Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s"
}
req = RequestsES8.esqlAsync areq
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
let Just bodyBytes = bhRequestBody req
hasKey "wait_for_completion_timeout" bodyBytes `shouldBe` True
it "esqlAsync embeds the query field in the body" $ do
let req =
RequestsES8.esqlAsync $
ES8Types.mkAsyncESQLRequest $
minimalReq "FROM x | LIMIT 1"
let Just bodyBytes = bhRequestBody req
hasKey "query" bodyBytes `shouldBe` True
it "esqlAsyncWith with default options is byte-identical to esqlAsync (no query string)" $ do
let base = ES8Types.mkAsyncESQLRequest (minimalReq "FROM x")
r1 = RequestsES8.esqlAsync base
r2 = RequestsES8.esqlAsyncWith base ES8Types.defaultESQLQueryOptions
getRawEndpoint (bhRequestEndpoint r1) `shouldBe` getRawEndpoint (bhRequestEndpoint r2)
getRawEndpointQueries (bhRequestEndpoint r1)
`shouldBe` getRawEndpointQueries (bhRequestEndpoint r2)
bhRequestBody r1 `shouldBe` bhRequestBody r2
it "esqlAsyncWith emits delimiter/drop_null_columns/allow_partial_results as query params" $ do
let opts =
ES8Types.defaultESQLQueryOptions
{ ES8Types.esqloDelimiter = Just "|",
ES8Types.esqloDropNullColumns = Just True,
ES8Types.esqloAllowPartialResults = Just False
}
req =
RequestsES8.esqlAsyncWith
(ES8Types.mkAsyncESQLRequest (minimalReq "FROM x"))
opts
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async"]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("delimiter", Just "|")]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("drop_null_columns", Just "true")]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("allow_partial_results", Just "false")]
it "getAsyncESQL GETs /_query/async/{id} with no query when timeout is Nothing" $ do
let req = RequestsES8.getAsyncESQL (ES8Types.AsyncESQLId "Fm_==_RxD_123") Nothing
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async", "Fm_==_RxD_123"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
it "getAsyncESQL forwards wait_for_completion_timeout as a query string" $ do
let req = RequestsES8.getAsyncESQL (ES8Types.AsyncESQLId "abc") (Just "5s")
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async", "abc"]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldBe` [("wait_for_completion_timeout", Just "5s")]
it "getAsyncESQLWith with default options matches getAsyncESQL with no timeout" $ do
let r1 = RequestsES8.getAsyncESQL (ES8Types.AsyncESQLId "abc") Nothing
r2 = RequestsES8.getAsyncESQLWith (ES8Types.AsyncESQLId "abc") ES8Types.defaultGetAsyncESQLOptions
getRawEndpoint (bhRequestEndpoint r1) `shouldBe` getRawEndpoint (bhRequestEndpoint r2)
getRawEndpointQueries (bhRequestEndpoint r1)
`shouldBe` getRawEndpointQueries (bhRequestEndpoint r2)
it "getAsyncESQLWith forwards wait_for_completion_timeout from the options record" $ do
let opts = ES8Types.defaultGetAsyncESQLOptions {ES8Types.gaesqloWaitForCompletionTimeout = Just "5s"}
req = RequestsES8.getAsyncESQLWith (ES8Types.AsyncESQLId "abc") opts
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async", "abc"]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldBe` [("wait_for_completion_timeout", Just "5s")]
it "getAsyncESQLWith emits keep_alive and drop_null_columns as query params" $ do
let opts =
ES8Types.defaultGetAsyncESQLOptions
{ ES8Types.gaesqloKeepAlive = Just "10d",
ES8Types.gaesqloDropNullColumns = Just True
}
req = RequestsES8.getAsyncESQLWith (ES8Types.AsyncESQLId "abc") opts
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async", "abc"]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("keep_alive", Just "10d")]
getRawEndpointQueries (bhRequestEndpoint req)
`shouldContain` [("drop_null_columns", Just "true")]
it "deleteAsyncESQL DELETEs /_query/async/{id} with no query" $ do
let req = RequestsES8.deleteAsyncESQL (ES8Types.AsyncESQLId "FmRleEct")
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async", "FmRleEct"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
it "stopAsyncESQL POSTs /_query/async/{id}/stop with no query" $ do
let req = RequestsES8.stopAsyncESQL (ES8Types.AsyncESQLId "FmRleEct")
getRawEndpoint (bhRequestEndpoint req) `shouldBe` ["_query", "async", "FmRleEct", "stop"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
describe "live integration (requires ES8+ backend)" $
backendSpecific [ElasticSearch8, ElasticSearch9] $ do
it "submit→poll→delete round-trip returns rows from an indexed document" $
withTestEnv $ do
_ <- insertData
backend <- liftIO detectBackendType
let q = "FROM \"" <> unIndexName testIndex <> "\" | LIMIT 5"
-- wait_for_completion_timeout=5s + keep_on_completion=true so
-- the POST returns a completed result *and* stores it under
-- an id we can poll and delete.
submitted <-
if backend == Just ElasticSearch9
then
ClientES9.esqlAsync $
(ES9Types.mkAsyncESQLRequest (minimalES9Req q))
{ ES9Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s",
ES9Types.asyncESQLRequestKeepOnCompletion = Just True
}
else
ClientES8.esqlAsync $
(ES8Types.mkAsyncESQLRequest (minimalReq q))
{ ES8Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s",
ES8Types.asyncESQLRequestKeepOnCompletion = Just True
}
case submitted of
Left e ->
liftIO $ expectationFailure $ "esqlAsync failed: " <> show e
Right sub -> do
liftIO $ ES8Types.asyncESQLResultIsRunning sub `shouldBe` False
case ES8Types.asyncESQLResultId sub of
Nothing ->
-- Server completed synchronously but did not retain the
-- result (no keep_on_completion in this branch). Nothing
-- to poll or delete; assert the rows were returned.
liftIO $
ES8Types.asyncESQLResultValues sub
`shouldSatisfy` (isJust :: Maybe [[Value]] -> Bool)
Just asyncId -> do
polled <-
if backend == Just ElasticSearch9
then ClientES9.getAsyncESQL asyncId (Just "5s")
else ClientES8.getAsyncESQL asyncId (Just "5s")
liftIO $
case polled of
Left e ->
expectationFailure $ "getAsyncESQL failed: " <> show e
Right polledRes -> do
ES8Types.asyncESQLResultIsRunning polledRes `shouldBe` False
ES8Types.asyncESQLResultColumns polledRes `shouldSatisfy` isJust
del <-
if backend == Just ElasticSearch9
then ClientES9.deleteAsyncESQL asyncId
else ClientES8.deleteAsyncESQL asyncId
liftIO $ del `shouldBe` Acknowledged True
it "surfaces an EsError for a non-existent async ES|QL id" $
withTestEnv $ do
-- Submit a real async query, retaining its result under an id
-- (keep_on_completion=true). DELETE it once so the id is
-- validly-formatted but purged, then DELETE the same id again:
-- a genuinely non-existent (already-purged) id yields a 404,
-- whereas a malformed id would yield a 400. This exercises the
-- real 'non-existent' path the 'StatusDependant' handler decodes.
backend <- liftIO detectBackendType
submitted <-
if backend == Just ElasticSearch9
then
ClientES9.esqlAsync $
(ES9Types.mkAsyncESQLRequest (minimalES9Req "ROW x = 1"))
{ ES9Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s",
ES9Types.asyncESQLRequestKeepOnCompletion = Just True
}
else
ClientES8.esqlAsync $
(ES8Types.mkAsyncESQLRequest (minimalReq "ROW x = 1"))
{ ES8Types.asyncESQLRequestWaitForCompletionTimeout = Just "5s",
ES8Types.asyncESQLRequestKeepOnCompletion = Just True
}
case submitted of
Left e
| endpointMissing e ->
liftIO $ pendingWith "async ES|QL requires Elasticsearch 8+"
| otherwise ->
liftIO $ expectationFailure $ "esqlAsync failed: " <> show e
Right sub -> case ES8Types.asyncESQLResultId sub of
Nothing ->
liftIO $
pendingWith
"async submission did not retain an id (keep_on_completion=true)"
Just asyncId -> do
firstDel <-
if backend == Just ElasticSearch9
then ClientES9.deleteAsyncESQL asyncId
else ClientES8.deleteAsyncESQL asyncId
liftIO $ firstDel `shouldBe` Acknowledged True
result <-
tryEsError $
if backend == Just ElasticSearch9
then ClientES9.deleteAsyncESQL asyncId
else ClientES8.deleteAsyncESQL asyncId
liftIO $
case result of
Left err -> errorStatus err `shouldBe` Just 404
Right r ->
expectationFailure $
"expected EsError for purged async ES|QL id, got: " <> show r
describe "stop running async query (requires ES8.11+)" $
backendSpecific [ElasticSearch8, ElasticSearch9] $ do
it "submit→stop returns the final result with is_running=false" $
withTestEnv $ do
-- Submit a long-running async query (wait_for_completion_timeout=0ms
-- forces the server to defer; keep_alive=5m retains the result under
-- an id we can stop and then clean up).
backend <- liftIO detectBackendType
submitted <-
if backend == Just ElasticSearch9
then
ClientES9.esqlAsync $
(ES9Types.mkAsyncESQLRequest (minimalES9Req "ROW x = 1"))
{ ES9Types.asyncESQLRequestWaitForCompletionTimeout = Just "0ms",
ES9Types.asyncESQLRequestKeepAlive = Just "5m"
}
else
ClientES8.esqlAsync $
(ES8Types.mkAsyncESQLRequest (minimalReq "ROW x = 1"))
{ ES8Types.asyncESQLRequestWaitForCompletionTimeout = Just "0ms",
ES8Types.asyncESQLRequestKeepAlive = Just "5m"
}
case submitted of
Left e
| endpointMissing e ->
liftIO $ pendingWith "stopAsyncESQL requires Elasticsearch 8.11+"
| otherwise ->
liftIO $ expectationFailure $ "esqlAsync failed: " <> show e
Right sub -> case ES8Types.asyncESQLResultId sub of
Nothing ->
liftIO $
pendingWith
"Async submission did not return an id to stop (server completed synchronously)"
Just asyncId -> do
stopResp <-
if backend == Just ElasticSearch9
then ClientES9.stopAsyncESQL asyncId
else ClientES8.stopAsyncESQL asyncId
case stopResp of
Left e
| endpointMissing e ->
liftIO $ pendingWith "stopAsyncESQL requires Elasticsearch 8.11+"
| otherwise ->
liftIO $ expectationFailure $ "stopAsyncESQL failed: " <> show e
Right stopped -> liftIO $ do
ES8Types.asyncESQLResultIsRunning stopped `shouldBe` False
ES8Types.asyncESQLResultColumns stopped `shouldSatisfy` isJust
-- Clean up the retained async result regardless of outcome.
void $
tryEsError $
if backend == Just ElasticSearch9
then ClientES9.deleteAsyncESQL asyncId
else ClientES8.deleteAsyncESQL asyncId
-- | Detect the @no handler found for uri@ shape returned by Elasticsearch
-- when an endpoint isn't registered on this version. Mirrors the helper in
-- "Test.EsQueryViewsSpec".
endpointMissing :: EsError -> Bool
endpointMissing e =
"no handler found" `T.isInfixOf` T.toLower (errorMessage e)