bloodhound-1.0.0.0: tests/Test/IndexTransformsSpec.hs
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE TypeApplications #-}
module Test.IndexTransformsSpec (spec) where
import Control.Monad (void)
import Control.Monad.Catch (bracket_)
import Control.Monad.IO.Class (liftIO)
import Data.Aeson (Value (..), decode, encode, object, (.=))
import Data.Aeson.KeyMap qualified as KM
import Data.ByteString.Lazy.Char8 qualified as LBS
import Data.List qualified as L
import Data.Map.Strict qualified as Map
import Data.Maybe (isJust)
import Data.Text qualified as T
import Data.Time.Clock.POSIX (getPOSIXTime)
import Database.Bloodhound.OpenSearch1.Client qualified as OS1Client
import Database.Bloodhound.OpenSearch1.Requests qualified as OS1Requests
import Database.Bloodhound.OpenSearch1.Types qualified as OS1Types
import Database.Bloodhound.OpenSearch2.Client qualified as OS2Client
import Database.Bloodhound.OpenSearch2.Requests qualified as OS2Requests
import Database.Bloodhound.OpenSearch2.Types qualified as OS2Types
import Database.Bloodhound.OpenSearch3.Client qualified as OS3Client
import Database.Bloodhound.OpenSearch3.Requests qualified as OS3Requests
import Database.Bloodhound.OpenSearch3.Types qualified as OS3Types
import TestsUtils.Common
( os1OnlyIT,
os2OnlyIT,
os3OnlyIT,
withTestEnv,
)
import TestsUtils.Import
import Prelude
-- ---------------------------------------------------------------------------
-- Sample fixtures, drawn verbatim from the OS Index Transforms plugin docs
-- (<https://docs.opensearch.org/latest/im-plugin/index-transforms/transforms-apis/>).
-- ---------------------------------------------------------------------------
-- | A bare-bones transform request body — exactly the shape the docs show
-- for @PUT /_plugins/_transform/{transform_id}@ (an interval schedule, a
-- single @terms@ group, a @match_all@ data selection, and a page size).
sampleTransformJson :: LBS.ByteString
sampleTransformJson =
"{\
\ \"description\": \"A simple transform\",\
\ \"schedule\": {\
\ \"interval\": {\
\ \"period\": 1,\
\ \"unit\": \"Minutes\",\
\ \"start_time\": 1602100553\
\ }\
\ },\
\ \"metadata_id\": null,\
\ \"source_index\": \"bloodhound-test-source-os1\",\
\ \"target_index\": \"bloodhound-test-target-os1\",\
\ \"data_selection_query\": {\
\ \"match_all\": {}\
\ },\
\ \"page_size\": 1,\
\ \"groups\": [\
\ {\
\ \"terms\": {\
\ \"source_field\": \"category\",\
\ \"target_field\": \"category_term\"\
\ }\
\ }\
\ ]\
\}"
-- | The full server response envelope for Create \/ Get. Carries every
-- server-injected bookkeeping field the decoder must round-trip
-- (@schema_version@, @updated_at@, @enabled_at@, @roles@,
-- @transform_id@).
sampleDocumentResponseJson :: LBS.ByteString
sampleDocumentResponseJson =
"{\
\ \"_id\": \"my-transform-id\",\
\ \"_version\": 1,\
\ \"_seq_no\": 0,\
\ \"_primary_term\": 1,\
\ \"transform\": {\
\ \"transform_id\": \"my-transform-id\",\
\ \"schema_version\": 2,\
\ \"continuous\": false,\
\ \"schedule\": {\
\ \"interval\": {\
\ \"period\": 1,\
\ \"unit\": \"Minutes\",\
\ \"start_time\": 1602100553\
\ }\
\ },\
\ \"metadata_id\": null,\
\ \"updated_at\": 1602100553,\
\ \"enabled\": false,\
\ \"enabled_at\": null,\
\ \"description\": \"A simple transform\",\
\ \"source_index\": \"bloodhound-test-source-os1\",\
\ \"data_selection_query\": {\
\ \"match_all\": {\
\ \"boost\": 1.0\
\ }\
\ },\
\ \"target_index\": \"bloodhound-test-target-os1\",\
\ \"roles\": [],\
\ \"page_size\": 1,\
\ \"groups\": [\
\ {\
\ \"terms\": {\
\ \"source_field\": \"category\",\
\ \"target_field\": \"category_term\"\
\ }\
\ }\
\ ]\
\ }\
\}"
-- | The list endpoint envelope. Note the distinct @total_transforms@ and
-- @transforms@ keys (NOT the standard search @hits@ envelope).
sampleListResponseJson :: LBS.ByteString
sampleListResponseJson =
"{\
\ \"total_transforms\": 1,\
\ \"transforms\": [\
\ {\
\ \"_id\": \"my-transform-id\",\
\ \"_seq_no\": 0,\
\ \"_primary_term\": 1,\
\ \"transform\": {\
\ \"transform_id\": \"my-transform-id\",\
\ \"schedule\": {\
\ \"interval\": { \"period\": 1, \"unit\": \"Minutes\" }\
\ },\
\ \"source_index\": \"bloodhound-test-source-os1\",\
\ \"data_selection_query\": { \"match_all\": {} },\
\ \"target_index\": \"bloodhound-test-target-os1\",\
\ \"page_size\": 1,\
\ \"groups\": [\
\ { \"terms\": { \"source_field\": \"category\" } }\
\ ]\
\ }\
\ }\
\ ]\
\}"
-- | The @_explain@ response — keyed by transform_id at the top level.
sampleExplainResponseJson :: LBS.ByteString
sampleExplainResponseJson =
"{\
\ \"my-transform-id\": {\
\ \"metadata_id\": \"PaLcRj0gaQq-y_WO81N0Q\",\
\ \"transform_metadata\": {\
\ \"continuous_stats\": {\
\ \"last_timestamp\": 1633392680364,\
\ \"documents_behind\": { \"bloodhound-test-source-os1\": 0 }\
\ },\
\ \"transform_id\": \"my-transform-id\",\
\ \"last_updated_at\": 1633392680364,\
\ \"status\": \"finished\",\
\ \"failure_reason\": null,\
\ \"stats\": {\
\ \"pages_processed\": 1,\
\ \"documents_processed\": 10,\
\ \"documents_indexed\": 1,\
\ \"index_time_in_millis\": 5,\
\ \"search_time_in_millis\": 3\
\ }\
\ }\
\ }\
\}"
-- | The @_preview@ response — a list of opaque row documents.
samplePreviewResponseJson :: LBS.ByteString
samplePreviewResponseJson =
"{\
\ \"documents\": [\
\ {\
\ \"category_term\": \"alpha\",\
\ \"count\": 5\
\ },\
\ {\
\ \"category_term\": \"beta\",\
\ \"count\": 3\
\ }\
\ ]\
\}"
-- | The @terms@ group body for 'sampleTransformJson'.
sampleTermsGroup :: OS1Types.TermsGroup
sampleTermsGroup =
OS1Types.TermsGroup
{ OS1Types.termsGroupSourceField = "category",
OS1Types.termsGroupTargetField = Just "category_term",
OS1Types.termsGroupOther = Nothing
}
-- | Haskell-side 'OS1Types.Transform' mirror of 'sampleTransformJson'.
os1SampleTransform :: OS1Types.Transform
os1SampleTransform =
OS1Types.Transform
{ OS1Types.transformEnabled = Nothing,
OS1Types.transformContinuous = Nothing,
OS1Types.transformSchedule =
OS1Types.TransformScheduleInterval
OS1Types.TransformInterval
{ OS1Types.transformIntervalPeriod = 1,
OS1Types.transformIntervalUnit = OS1Types.TransformPeriodUnitMinutes,
OS1Types.transformIntervalStartTime = Just 1602100553
},
OS1Types.transformDescription = Just "A simple transform",
OS1Types.transformMetadataId = Nothing,
OS1Types.transformSourceIndex = "bloodhound-test-source-os1",
OS1Types.transformTargetIndex = "bloodhound-test-target-os1",
OS1Types.transformDataSelectionQuery =
object ["match_all" .= object []],
OS1Types.transformPageSize = 1,
OS1Types.transformGroups = Just [OS1Types.TransformGroupTerms sampleTermsGroup],
OS1Types.transformAggregations = Nothing
}
spec :: Spec
spec = do
-- =======================================================================
-- TransformId newtype
-- =======================================================================
describe "IndexTransforms TransformId JSON" $ do
it "encodes TransformId as a bare JSON string" $ do
encode (OS1Types.TransformId "abc-123")
`shouldBe` "\"abc-123\""
it "decodes a bare JSON string into TransformId" $ do
decode "\"abc-123\"" `shouldBe` Just (OS1Types.TransformId "abc-123")
it "round-trips TransformId through encode -> decode" $ do
let original = OS1Types.TransformId "round-trip-id"
(decode . encode) original `shouldBe` Just original
-- =======================================================================
-- TransformPeriodUnit enum
-- =======================================================================
describe "IndexTransforms TransformPeriodUnit JSON" $ do
let cases =
[ (OS1Types.TransformPeriodUnitMinutes, "Minutes"),
(OS1Types.TransformPeriodUnitHours, "Hours"),
(OS1Types.TransformPeriodUnitDays, "Days"),
(OS1Types.TransformPeriodUnitSeconds, "Seconds"),
(OS1Types.TransformPeriodUnitWeeks, "Weeks"),
(OS1Types.TransformPeriodUnitMonths, "Months")
]
forM_ cases $ \(u, wire) -> do
it ("encodes " <> show wire <> " to the documented wire string") $ do
encode u `shouldBe` "\"" <> LBS.fromStrict wire <> "\""
it ("decodes the documented wire string " <> show wire) $ do
(decode ("\"" <> LBS.fromStrict wire <> "\"") :: Maybe OS1Types.TransformPeriodUnit)
`shouldBe` Just u
it "decodes an unknown unit into TransformPeriodUnitCustom" $ do
(decode "\"Decades\"" :: Maybe OS1Types.TransformPeriodUnit)
`shouldBe` Just (OS1Types.TransformPeriodUnitCustom "Decades")
it "round-trips TransformPeriodUnit through ToJSON/FromJSON (known)" $ do
let u = OS1Types.TransformPeriodUnitMinutes
(decode . encode) u `shouldBe` Just u
-- =======================================================================
-- TransformSortDirection enum
-- =======================================================================
describe "IndexTransforms TransformSortDirection JSON" $ do
let cases =
[ (OS1Types.TransformSortDirectionAsc, "ASC"),
(OS1Types.TransformSortDirectionDesc, "DESC")
]
forM_ cases $ \(d, wire) -> do
it ("encodes " <> show wire <> " to the documented wire string") $ do
encode d `shouldBe` "\"" <> LBS.fromStrict wire <> "\""
it ("decodes the documented wire string " <> show wire) $ do
(decode ("\"" <> LBS.fromStrict wire <> "\"") :: Maybe OS1Types.TransformSortDirection)
`shouldBe` Just d
-- =======================================================================
-- TransformsListOptions -> query params
-- =======================================================================
describe "IndexTransforms TransformsListOptions params" $ do
it "defaultTransformsListOptions produces no query pairs" $ do
OS1Types.transformsListOptionsParams OS1Types.defaultTransformsListOptions
`shouldBe` []
it "renders all fields when fully populated" $ do
let opts =
OS1Types.defaultTransformsListOptions
{ OS1Types.transformsListOptionsFrom = Just 5,
OS1Types.transformsListOptionsSize = Just 20,
OS1Types.transformsListOptionsSearch = Just "foo",
OS1Types.transformsListOptionsSortField = Just "id",
OS1Types.transformsListOptionsSortDirection = Just OS1Types.TransformSortDirectionDesc
}
L.sortOn fst (OS1Types.transformsListOptionsParams opts)
`shouldBe` [ ("from", Just "5"),
("search", Just "foo"),
("size", Just "20"),
("sortDirection", Just "DESC"),
("sortField", Just "id")
]
it "omits Nothing fields" $ do
let opts =
OS1Types.defaultTransformsListOptions
{ OS1Types.transformsListOptionsSize = Just 10
}
OS1Types.transformsListOptionsParams opts
`shouldBe` [("size", Just "10")]
-- =======================================================================
-- Transform (request body) JSON
-- =======================================================================
describe "IndexTransforms Transform JSON" $ do
it "decodes the documented request body fixture" $ do
let decoded = decode sampleTransformJson :: Maybe OS1Types.Transform
decoded `shouldBe` Just os1SampleTransform
it "round-trips the sample Transform through encode -> decode" $ do
(decode . encode) os1SampleTransform `shouldBe` Just os1SampleTransform
it "encodes a Transform omitting nulls (no enabled/continuous/aggregations)" $ do
let encoded = encode os1SampleTransform
encodedStr = LBS.unpack encoded
encodedStr `shouldSatisfy` not . L.isInfixOf "\"enabled\""
encodedStr `shouldSatisfy` not . L.isInfixOf "\"continuous\""
encodedStr `shouldSatisfy` not . L.isInfixOf "\"aggregations\""
-- @schedule@, @source_index@, @target_index@, @data_selection_query@,
-- @page_size@, @groups@ are required and always appear.
encodedStr `shouldSatisfy` L.isInfixOf "\"schedule\""
encodedStr `shouldSatisfy` L.isInfixOf "\"source_index\""
encodedStr `shouldSatisfy` L.isInfixOf "\"page_size\""
-- =======================================================================
-- TransformGroup sum type
-- =======================================================================
describe "IndexTransforms TransformGroup JSON" $ do
it "decodes {\"terms\": {...}} into TransformGroupTerms" $ do
let raw = "{\"terms\": {\"source_field\": \"category\"}}"
expected =
OS1Types.TransformGroupTerms
OS1Types.TermsGroup
{ OS1Types.termsGroupSourceField = "category",
OS1Types.termsGroupTargetField = Nothing,
OS1Types.termsGroupOther = Nothing
}
(decode raw :: Maybe OS1Types.TransformGroup) `shouldBe` Just expected
it "decodes {\"histogram\": {...}} into TransformGroupHistogram" $ do
let raw = "{\"histogram\": {\"source_field\": \"price\", \"interval\": 10}}"
expected =
OS1Types.TransformGroupHistogram
OS1Types.HistogramGroup
{ OS1Types.histogramGroupSourceField = "price",
OS1Types.histogramGroupTargetField = Nothing,
OS1Types.histogramGroupInterval = Just 10,
OS1Types.histogramGroupOther = Nothing
}
(decode raw :: Maybe OS1Types.TransformGroup) `shouldBe` Just expected
it "decodes {\"date_histogram\": {...}} into TransformGroupDateHistogram" $ do
let raw = "{\"date_histogram\": {\"source_field\": \"timestamp\", \"interval\": \"1d\", \"format\": \"yyyy-MM-dd\"}}"
let decoded = decode raw :: Maybe OS1Types.TransformGroup
decoded `shouldSatisfy` isJust
case decoded of
Just (OS1Types.TransformGroupDateHistogram g) -> do
OS1Types.dateHistogramGroupSourceField g `shouldBe` "timestamp"
OS1Types.dateHistogramGroupInterval g `shouldBe` Just "1d"
OS1Types.dateHistogramGroupFormat g `shouldBe` Just "yyyy-MM-dd"
other -> expectationFailure ("unexpected decode: " <> show other)
it "round-trips each variant through encode -> decode" $ do
let g1 = OS1Types.TransformGroupTerms sampleTermsGroup
g2 =
OS1Types.TransformGroupHistogram
OS1Types.HistogramGroup
{ OS1Types.histogramGroupSourceField = "f",
OS1Types.histogramGroupTargetField = Nothing,
OS1Types.histogramGroupInterval = Nothing,
OS1Types.histogramGroupOther = Nothing
}
g3 =
OS1Types.TransformGroupDateHistogram
OS1Types.DateHistogramGroup
{ OS1Types.dateHistogramGroupSourceField = "t",
OS1Types.dateHistogramGroupTargetField = Nothing,
OS1Types.dateHistogramGroupInterval = Nothing,
OS1Types.dateHistogramGroupFormat = Nothing,
OS1Types.dateHistogramGroupTimeZone = Nothing,
OS1Types.dateHistogramGroupOther = Nothing
}
(decode . encode) g1 `shouldBe` Just g1
(decode . encode) g2 `shouldBe` Just g2
(decode . encode) g3 `shouldBe` Just g3
it "preserves unknown group fields through the Other escape hatch" $ do
let raw = "{\"percentiles\": {\"source_field\": \"x\"}}"
let decoded = decode raw :: Maybe OS1Types.TransformGroup
case decoded of
Just (OS1Types.TransformGroupOther _) -> pure ()
other -> expectationFailure ("expected TransformGroupOther, got " <> show other)
-- round-trip preserves the unknown wrapper key
(decode . encode =<< decoded) `shouldBe` decoded
-- =======================================================================
-- TransformDocumentResponse JSON (envelope + inner TransformResponse)
-- =======================================================================
describe "IndexTransforms TransformDocumentResponse JSON" $ do
it "decodes the documented response envelope (OS1)" $ do
let decoded = decode sampleDocumentResponseJson :: Maybe OS1Types.TransformDocumentResponse
decoded `shouldSatisfy` isJust
let Just r = decoded
OS1Types.transformDocumentResponseId r `shouldBe` "my-transform-id"
OS1Types.transformDocumentResponseVersion r `shouldBe` 1
OS1Types.transformDocumentResponseSeqNo r `shouldBe` 0
OS1Types.transformDocumentResponsePrimaryTerm r `shouldBe` 1
it "decodes the same response envelope identically across OS1/OS2/OS3" $ do
let one = decode sampleDocumentResponseJson :: Maybe OS1Types.TransformDocumentResponse
two = decode sampleDocumentResponseJson :: Maybe OS2Types.TransformDocumentResponse
three = decode sampleDocumentResponseJson :: Maybe OS3Types.TransformDocumentResponse
one `shouldSatisfy` isJust
two `shouldSatisfy` isJust
three `shouldSatisfy` isJust
it "round-trips the response envelope through encode -> decode" $ do
let decoded = decode sampleDocumentResponseJson :: Maybe OS1Types.TransformDocumentResponse
(decode . encode) decoded `shouldBe` decoded
it "rejects a response missing the transform key" $ do
let bad = "{ \"_id\": \"x\", \"_version\": 1, \"_seq_no\": 0, \"_primary_term\": 1 }"
(decode bad :: Maybe OS1Types.TransformDocumentResponse) `shouldBe` Nothing
-- =======================================================================
-- Delete response is the bulk envelope (live-verified on OS 3.7.0)
-- =======================================================================
describe "IndexTransforms deleteTransform response shape" $ do
-- Captured live against OS 3.7.0 (port 9205): DELETE returns the
-- bulk envelope because the transform is stored as a doc in
-- .opendistro-ism-config and the plugin deletes via the bulk
-- handler. BulkResponse (Common.Types.Bulk) matches the shape
-- byte-for-byte.
let deleteResponseJson =
"{\
\ \"took\": 11,\
\ \"errors\": false,\
\ \"items\": [\
\ {\
\ \"delete\": {\
\ \"_index\": \".opendistro-ism-config\",\
\ \"_id\": \"my-transform-id\",\
\ \"_version\": 2,\
\ \"result\": \"deleted\",\
\ \"forced_refresh\": true,\
\ \"_shards\": { \"total\": 2, \"successful\": 1, \"failed\": 0 },\
\ \"_seq_no\": 6404,\
\ \"_primary_term\": 2,\
\ \"status\": 200\
\ }\
\ }\
\ ]\
\}"
it "decodes the live bulk-delete envelope into BulkResponse" $ do
let decoded = decode deleteResponseJson :: Maybe BulkResponse
decoded `shouldSatisfy` isJust
let Just r = decoded
bulkTook r `shouldBe` 11
bulkErrors r `shouldBe` False
length (bulkActionItems r) `shouldBe` 1
-- =======================================================================
-- GetTransformsResponse (list envelope)
-- =======================================================================
describe "IndexTransforms GetTransformsResponse JSON" $ do
it "decodes the documented list envelope" $ do
let decoded = decode sampleListResponseJson :: Maybe OS1Types.GetTransformsResponse
decoded `shouldSatisfy` isJust
let Just r = decoded
OS1Types.getTransformsResponseTotalTransforms r `shouldBe` 1
length (OS1Types.getTransformsResponseTransforms r) `shouldBe` 1
it "decodes an empty transforms array" $ do
let empty = "{ \"total_transforms\": 0, \"transforms\": [] }"
let decoded = decode empty :: Maybe OS1Types.GetTransformsResponse
decoded `shouldSatisfy` isJust
let Just r = decoded
OS1Types.getTransformsResponseTotalTransforms r `shouldBe` 0
-- =======================================================================
-- TransformExplain (the _explain response)
-- =======================================================================
describe "IndexTransforms TransformExplain JSON" $ do
it "decodes the documented explain body" $ do
let decoded =
decode sampleExplainResponseJson ::
Maybe (Map.Map T.Text OS1Types.TransformExplain)
decoded `shouldSatisfy` isJust
let Just m = decoded
Map.size m `shouldBe` 1
let Just entry = Map.lookup "my-transform-id" m
OS1Types.transformExplainMetadataId entry `shouldBe` Just "PaLcRj0gaQq-y_WO81N0Q"
OS1Types.transformExplainTransformMetadata entry `shouldSatisfy` isJust
it "decodes a continuous_stats sub-object" $ do
let decoded =
decode sampleExplainResponseJson ::
Maybe (Map.Map T.Text OS1Types.TransformExplain)
let Just m = decoded
Just entry = Map.lookup "my-transform-id" m
Just md = OS1Types.transformExplainTransformMetadata entry
Just cs = OS1Types.transformMetadataContinuousStats md
OS1Types.transformContinuousStatsLastTimestamp cs `shouldBe` Just 1633392680364
-- =======================================================================
-- PreviewTransformResponse
-- =======================================================================
describe "IndexTransforms PreviewTransformResponse JSON" $ do
it "decodes the documented preview body" $ do
let decoded = decode samplePreviewResponseJson :: Maybe OS1Types.PreviewTransformResponse
decoded `shouldSatisfy` isJust
let Just r = decoded
length (OS1Types.previewTransformResponseDocuments r) `shouldBe` 2
it "decodes an empty documents array" $ do
let empty' = "{ \"documents\": [] }"
let decoded = decode empty' :: Maybe OS1Types.PreviewTransformResponse
decoded `shouldSatisfy` isJust
let Just r = decoded
length (OS1Types.previewTransformResponseDocuments r) `shouldBe` 0
-- =======================================================================
-- Lifecycle endpoint shapes
-- =======================================================================
describe "IndexTransforms lifecycle endpoint shape" $ do
let tid1 = OS1Types.TransformId "t1"
tid2 = OS2Types.TransformId "t2"
tid3 = OS3Types.TransformId "t3"
it "createTransform PUTs /_plugins/_transform/{id} with the wrapped body" $ do
let req = OS1Requests.createTransform tid1 os1SampleTransform
bhRequestMethod req `shouldBe` "PUT"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "t1"]
getRawEndpointQueries (bhRequestEndpoint req) `shouldBe` []
-- Body wraps under "transform" key
let Just body = bhRequestBody req
case decode body :: Maybe Value of
Just (Object o) -> "transform" `KM.member` o `shouldBe` True
_ -> expectationFailure "body should be a JSON object"
it "updateTransformWith emits if_seq_no / if_primary_term query params" $ do
let req = OS1Requests.updateTransformWith tid1 os1SampleTransform (Just (7, 4))
bhRequestMethod req `shouldBe` "PUT"
L.sortOn fst (getRawEndpointQueries (bhRequestEndpoint req))
`shouldBe` [("if_primary_term", Just "4"), ("if_seq_no", Just "7")]
it "getTransform GETs /_plugins/_transform/{id} with no body" $ do
let req = OS1Requests.getTransform tid1
bhRequestMethod req `shouldBe` "GET"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "t1"]
bhRequestBody req `shouldBe` Nothing
it "getTransforms GETs /_plugins/_transform; forwards list opts as queries" $ do
let reqNoOpts = OS1Requests.getTransforms Nothing
bhRequestMethod reqNoOpts `shouldBe` "GET"
getRawEndpoint (bhRequestEndpoint reqNoOpts)
`shouldBe` ["_plugins", "_transform"]
getRawEndpointQueries (bhRequestEndpoint reqNoOpts) `shouldBe` []
let opts =
OS1Types.defaultTransformsListOptions
{ OS1Types.transformsListOptionsSize = Just 5,
OS1Types.transformsListOptionsSearch = Just "abc"
}
reqWithOpts = OS1Requests.getTransforms (Just opts)
L.sortOn fst (getRawEndpointQueries (bhRequestEndpoint reqWithOpts))
`shouldBe` [("search", Just "abc"), ("size", Just "5")]
it "startTransform POSTs /_plugins/_transform/{id}/_start" $ do
let req = OS1Requests.startTransform tid1
bhRequestMethod req `shouldBe` "POST"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "t1", "_start"]
it "stopTransform POSTs /_plugins/_transform/{id}/_stop" $ do
let req = OS1Requests.stopTransform tid1
bhRequestMethod req `shouldBe` "POST"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "t1", "_stop"]
it "explainTransform GETs /_plugins/_transform/{id}/_explain" $ do
let req = OS1Requests.explainTransform tid1
bhRequestMethod req `shouldBe` "GET"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "t1", "_explain"]
it "previewTransform POSTs /_plugins/_transform/_preview with the wrapped body" $ do
let req = OS1Requests.previewTransform os1SampleTransform
bhRequestMethod req `shouldBe` "POST"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "_preview"]
-- Body wraps under "transform" key
let Just body = bhRequestBody req
LBS.unpack body `shouldSatisfy` L.isInfixOf "\"transform\""
it "deleteTransform DELETEs /_plugins/_transform/{id}" $ do
let req = OS1Requests.deleteTransform tid1
bhRequestMethod req `shouldBe` "DELETE"
getRawEndpoint (bhRequestEndpoint req)
`shouldBe` ["_plugins", "_transform", "t1"]
bhRequestBody req `shouldBe` Nothing
it "OS1, OS2, OS3 produce identical paths and methods for every endpoint" $ do
let same = "same-id"
one = OS1Types.TransformId same
two = OS2Types.TransformId same
three = OS3Types.TransformId same
getRawEndpoint (bhRequestEndpoint (OS1Requests.deleteTransform one))
`shouldBe` getRawEndpoint (bhRequestEndpoint (OS2Requests.deleteTransform two))
getRawEndpoint (bhRequestEndpoint (OS2Requests.deleteTransform two))
`shouldBe` getRawEndpoint (bhRequestEndpoint (OS3Requests.deleteTransform three))
bhRequestMethod (OS1Requests.startTransform one)
`shouldBe` bhRequestMethod (OS3Requests.startTransform three)
bhRequestMethod (OS1Requests.stopTransform one)
`shouldBe` bhRequestMethod (OS3Requests.stopTransform three)
-- =======================================================================
-- Live integration round-trip. Runs only against the matching
-- OS major version. The Transforms plugin ships with OS 1.0+.
-- =======================================================================
describe "IndexTransforms plugin live round-trip" $ do
os1It <- runIO os1OnlyIT
os1It "create -> get -> preview -> start -> explain -> stop -> delete (OpenSearch 1)" $
withTestEnv $ do
suffix <- liftIO $ show @Int . floor <$> getPOSIXTime
let sourceIdx = T.pack ("bloodhound_test_itf_src_os1_" <> suffix)
targetIdx = T.pack ("bloodhound_test_itf_tgt_os1_" <> suffix)
transformIdText = T.pack ("bloodhound_test_itf_t1_" <> suffix)
transformId = OS1Types.TransformId transformIdText
let Right sourceName = mkIndexName sourceIdx
Right targetName = mkIndexName targetIdx
bracket_
(transformSetupIndex sourceName targetName)
( do
void $ performBHRequest $ deleteIndex sourceName
void $ performBHRequest $ deleteIndex targetName
)
$ do
let transform =
os1SampleTransform
{ OS1Types.transformSourceIndex = sourceIdx,
OS1Types.transformTargetIndex = targetIdx,
OS1Types.transformDescription = Just "bloodhound live test"
}
createResp <- OS1Client.createTransform transformId transform
case createResp of
Left e ->
liftIO $
expectationFailure
("createTransform failed: " <> T.unpack (errorMessage e))
Right _ -> do
getResp <- OS1Client.getTransform transformId
liftIO $ expectationFailureOnLeft getResp "getTransform failed:"
previewResp <- OS1Client.previewTransform transform
liftIO $ expectationFailureOnLeft previewResp "previewTransform failed:"
startResp <- OS1Client.startTransform transformId
liftIO $ expectationFailureOnLeft startResp "startTransform failed:"
explainResp <- OS1Client.explainTransform transformId
liftIO $ expectationFailureOnLeft explainResp "explainTransform failed:"
stopResp <- OS1Client.stopTransform transformId
liftIO $ expectationFailureOnLeft stopResp "stopTransform failed:"
delResp <- OS1Client.deleteTransform transformId
liftIO $ expectationFailureOnLeft delResp "deleteTransform failed:"
os2It <- runIO os2OnlyIT
os2It "create -> get -> preview -> start -> explain -> stop -> delete (OpenSearch 2)" $
withTestEnv $ do
suffix <- liftIO $ show @Int . floor <$> getPOSIXTime
let sourceIdx = T.pack ("bloodhound_test_itf_src_os2_" <> suffix)
targetIdx = T.pack ("bloodhound_test_itf_tgt_os2_" <> suffix)
transformIdText = T.pack ("bloodhound_test_itf_t2_" <> suffix)
transformId = OS2Types.TransformId transformIdText
let Right sourceName = mkIndexName sourceIdx
Right targetName = mkIndexName targetIdx
bracket_
(transformSetupIndex sourceName targetName)
( do
void $ performBHRequest $ deleteIndex sourceName
void $ performBHRequest $ deleteIndex targetName
)
$ do
let transform =
OS2Types.Transform
{ OS2Types.transformEnabled = Nothing,
OS2Types.transformContinuous = Nothing,
OS2Types.transformSchedule =
OS2Types.TransformScheduleInterval
OS2Types.TransformInterval
{ OS2Types.transformIntervalPeriod = 1,
OS2Types.transformIntervalUnit = OS2Types.TransformPeriodUnitMinutes,
OS2Types.transformIntervalStartTime = Nothing
},
OS2Types.transformDescription = Just "bloodhound live test",
OS2Types.transformMetadataId = Nothing,
OS2Types.transformSourceIndex = sourceIdx,
OS2Types.transformTargetIndex = targetIdx,
OS2Types.transformDataSelectionQuery =
object ["match_all" .= object []],
OS2Types.transformPageSize = 1,
OS2Types.transformGroups =
Just
[ OS2Types.TransformGroupTerms
OS2Types.TermsGroup
{ OS2Types.termsGroupSourceField = "category",
OS2Types.termsGroupTargetField = Just "category_term",
OS2Types.termsGroupOther = Nothing
}
],
OS2Types.transformAggregations = Nothing
}
createResp <- OS2Client.createTransform transformId transform
case createResp of
Left e ->
liftIO $
expectationFailure
("createTransform failed: " <> T.unpack (errorMessage e))
Right _ -> do
getResp <- OS2Client.getTransform transformId
liftIO $ expectationFailureOnLeft getResp "getTransform failed:"
previewResp <- OS2Client.previewTransform transform
liftIO $ expectationFailureOnLeft previewResp "previewTransform failed:"
startResp <- OS2Client.startTransform transformId
liftIO $ expectationFailureOnLeft startResp "startTransform failed:"
explainResp <- OS2Client.explainTransform transformId
liftIO $ expectationFailureOnLeft explainResp "explainTransform failed:"
stopResp <- OS2Client.stopTransform transformId
liftIO $ expectationFailureOnLeft stopResp "stopTransform failed:"
delResp <- OS2Client.deleteTransform transformId
liftIO $ expectationFailureOnLeft delResp "deleteTransform failed:"
os3It <- runIO os3OnlyIT
os3It "create -> get -> preview -> start -> explain -> stop -> delete (OpenSearch 3)" $
withTestEnv $ do
suffix <- liftIO $ show @Int . floor <$> getPOSIXTime
let sourceIdx = T.pack ("bloodhound_test_itf_src_os3_" <> suffix)
targetIdx = T.pack ("bloodhound_test_itf_tgt_os3_" <> suffix)
transformIdText = T.pack ("bloodhound_test_itf_t3_" <> suffix)
transformId = OS3Types.TransformId transformIdText
let Right sourceName = mkIndexName sourceIdx
Right targetName = mkIndexName targetIdx
bracket_
(transformSetupIndex sourceName targetName)
( do
void $ performBHRequest $ deleteIndex sourceName
void $ performBHRequest $ deleteIndex targetName
)
$ do
let transform =
OS3Types.Transform
{ OS3Types.transformEnabled = Nothing,
OS3Types.transformContinuous = Nothing,
OS3Types.transformSchedule =
OS3Types.TransformScheduleInterval
OS3Types.TransformInterval
{ OS3Types.transformIntervalPeriod = 1,
OS3Types.transformIntervalUnit = OS3Types.TransformPeriodUnitMinutes,
OS3Types.transformIntervalStartTime = Nothing
},
OS3Types.transformDescription = Just "bloodhound live test",
OS3Types.transformMetadataId = Nothing,
OS3Types.transformSourceIndex = sourceIdx,
OS3Types.transformTargetIndex = targetIdx,
OS3Types.transformDataSelectionQuery =
object ["match_all" .= object []],
OS3Types.transformPageSize = 1,
OS3Types.transformGroups =
Just
[ OS3Types.TransformGroupTerms
OS3Types.TermsGroup
{ OS3Types.termsGroupSourceField = "category",
OS3Types.termsGroupTargetField = Just "category_term",
OS3Types.termsGroupOther = Nothing
}
],
OS3Types.transformAggregations = Nothing
}
createResp <- OS3Client.createTransform transformId transform
case createResp of
Left e ->
liftIO $
expectationFailure
("createTransform failed: " <> T.unpack (errorMessage e))
Right _ -> do
getResp <- OS3Client.getTransform transformId
liftIO $ expectationFailureOnLeft getResp "getTransform failed:"
previewResp <- OS3Client.previewTransform transform
liftIO $ expectationFailureOnLeft previewResp "previewTransform failed:"
startResp <- OS3Client.startTransform transformId
liftIO $ expectationFailureOnLeft startResp "startTransform failed:"
explainResp <- OS3Client.explainTransform transformId
liftIO $ expectationFailureOnLeft explainResp "explainTransform failed:"
stopResp <- OS3Client.stopTransform transformId
liftIO $ expectationFailureOnLeft stopResp "stopTransform failed:"
delResp <- OS3Client.deleteTransform transformId
liftIO $ expectationFailureOnLeft delResp "deleteTransform failed:"
-- | Create the source and target indexes, seed the source with a few
-- documents carrying a @category@ field, and refresh. The Transforms
-- plugin requires the source index to exist and to have at least one
-- document matching the @data_selection_query@ before it can preview
-- or run; we add three documents spanning two @category@ values so
-- the @terms@ group produces two buckets.
--
-- The @category@ field is mapped explicitly as @keyword@ — the
-- Transforms plugin refuses to create a @terms@ group on a field that
-- has no aggregation-friendly mapping (auto-mapped @text@ does not
-- qualify). The target index is created with the same setting; the
-- plugin writes buckets keyed by the group's @target_field@.
transformSetupIndex :: IndexName -> IndexName -> BH IO ()
transformSetupIndex sourceName targetName = do
void $
performBHRequest $
createIndex
(IndexSettings (ShardCount 1) (ReplicaCount 0) defaultIndexMappingsLimits)
sourceName
-- Map @category@ as keyword so the @terms@ group can aggregate on
-- it. The Transforms plugin rejects the create with
-- @Cannot find field [category] that can be grouped as [terms]@
-- otherwise.
void $
performBHRequest $
putMapping @Value sourceName $
object
[ "properties"
.= object
[ "category" .= object ["type" .= String "keyword"],
"value" .= object ["type" .= String "long"]
]
]
-- Target index may already need to exist for some plugin versions;
-- create it up front with the same settings so the start can write
-- to it.
void $
performBHRequest $
createIndex
(IndexSettings (ShardCount 1) (ReplicaCount 0) defaultIndexMappingsLimits)
targetName
let seedDoc cat value =
object ["category" .= String cat, "value" .= Number value]
void $
performBHRequest $
indexDocument
sourceName
defaultIndexDocumentSettings
(seedDoc "alpha" 1)
(DocId "1")
void $
performBHRequest $
indexDocument
sourceName
defaultIndexDocumentSettings
(seedDoc "alpha" 2)
(DocId "2")
void $
performBHRequest $
indexDocument
sourceName
defaultIndexDocumentSettings
(seedDoc "beta" 3)
(DocId "3")
void $ performBHRequest $ refreshIndex sourceName
pure ()
-- | Fail the test (with a prefix and the cluster error message) when a
-- 'ParsedEsResponse' is 'Left'. Used by the live lifecycle assertions.
expectationFailureOnLeft :: Either EsError a -> String -> IO ()
expectationFailureOnLeft r prefix =
case r of
Left e -> expectationFailure (prefix <> " " <> T.unpack (errorMessage e))
Right _ -> pure ()