TransformSpec.hs

{-# LANGUAGE OverloadedStrings #-}

module Test.TransformSpec (spec) where

import Data.Aeson
import Data.Aeson.KeyMap qualified as KM
import Data.ByteString.Lazy.Char8 qualified as LBS
import Data.List.NonEmpty qualified as NE
import TestsUtils.Import
import Prelude

-- | Construct an 'IndexName' from a literal known to be valid (all the
-- samples here are lowercase and within the documented rules).
idxName :: Text -> IndexName
idxName t = case mkIndexName t of
  Right n -> n
  Left e -> error ("invalid index name " <> show t <> ": " <> show e)

-- | A canonical pivot transform PUT body matching the ES 7.17 docs example.
-- The @pivot@ body is opaque on the Haskell side (group_by\/aggregations
-- DSL), so it round-trips verbatim.
samplePivotConfigBytes :: LBS.ByteString
samplePivotConfigBytes =
  "{\
  \  \"source\": {\"index\": \"kibana_sample_data_ecommerce\"},\
  \  \"dest\": {\"index\": \"kibana_sample_data_ecommerce_transform\"},\
  \  \"pivot\": {\
  \    \"group_by\": {\"customer_id\": {\"terms\": {\"field\": \"customer_id\"}}},\
  \    \"aggregations\": {\"order_count\": {\"value_count\": {\"field\": \"order_id\"}}}\
  \  },\
  \  \"description\": \"ecommerce transform\",\
  \  \"frequency\": \"1m\"\
  \}"

-- | A latest transform PUT body exercising the typed 'TransformLatest' path.
sampleLatestConfigBytes :: LBS.ByteString
sampleLatestConfigBytes =
  "{\
  \  \"source\": {\"index\": [\"logs-*\"]},\
  \  \"dest\": {\"index\": \"latest-logs\", \"pipeline\": \"logs-pipeline\"},\
  \  \"latest\": {\
  \    \"unique_key\": [\"host\", \"service\"],\
  \    \"sort\": \"@timestamp\"\
  \  }\
  \}"

-- | A continuous transform config carrying a typed @sync@ and @settings@,
-- plus an unknown sibling field (@_meta@-like) preserved in 'tcExtras'.
sampleContinuousConfigBytes :: LBS.ByteString
sampleContinuousConfigBytes =
  "{\
  \  \"source\": {\
  \    \"index\": \"events-*\",\
  \    \"query\": {\"match_all\": {}}\
  \  },\
  \  \"dest\": {\"index\": \"events-summary\"},\
  \  \"pivot\": {\"group_by\": {}, \"aggregations\": {}},\
  \  \"sync\": {\
  \    \"time\": {\
  \      \"field\": \"@timestamp\",\
  \      \"delay\": \"60s\"\
  \    }\
  \  },\
  \  \"settings\": {\
  \    \"max_page_search_size\": 500,\
  \    \"docs_per_second\": 200,\
  \    \"align_checkpoints\": true,\
  \    \"dates_as_epoch_millis\": false\
  \  },\
  \  \"created_by\": \"system\"\
  \}"

-- | A full @GET /_transform@ response with two entries (pivot + latest),
-- including the GET-only @id@ and @version@ fields.
sampleListResponseBytes :: LBS.ByteString
sampleListResponseBytes =
  "{\
  \  \"count\": 2,\
  \  \"transforms\": [\
  \    {\
  \      \"id\": \"ecommerce-transform\",\
  \      \"source\": {\"index\": \"ecommerce\"},\
  \      \"dest\": {\"index\": \"ecommerce-summary\"},\
  \      \"pivot\": {\"group_by\": {}, \"aggregations\": {}},\
  \      \"version\": \"7.17.0\"\
  \    },\
  \    {\
  \      \"id\": \"latest-transform\",\
  \      \"source\": {\"index\": \"logs\"},\
  \      \"dest\": {\"index\": \"latest-logs\"},\
  \      \"latest\": {\"unique_key\": [\"host\"], \"sort\": \"@timestamp\"}\
  \    }\
  \  ]\
  \}"

-- | A @GET /_transform/_stats@ response carrying the state enum and document
-- counters.
sampleStatsResponseBytes :: LBS.ByteString
sampleStatsResponseBytes =
  "{\
  \  \"count\": 1,\
  \  \"transforms\": [\
  \    {\
  \      \"id\": \"ecommerce-transform\",\
  \      \"state\": \"started\",\
  \      \"health\": \"green\",\
  \      \"checkpointing\": {\"last\": {\"checkpoint\": 1}},\
  \      \"documents_indexed\": 1000,\
  \      \"documents_processed\": 1200,\
  \      \"documents_deleted\": 0,\
  \      \"documents_failed\": 0\
  \    }\
  \  ]\
  \}"

-- | A @GET /_transform/{id}/_explain@ response.
sampleExplainResponseBytes :: LBS.ByteString
sampleExplainResponseBytes =
  "{\
  \  \"transforms\": [\
  \    {\
  \      \"id\": \"ecommerce-transform\",\
  \      \"state\": {\"task_state\": \"started\"},\
  \      \"checkpointing\": {\"last\": {\"checkpoint\": 1}}\
  \    }\
  \  ]\
  \}"

-- | A @POST /_transform/_preview@ response.
samplePreviewResponseBytes :: LBS.ByteString
samplePreviewResponseBytes =
  "{\
  \  \"preview\": [{\"order_count\": 42}],\
  \  \"generated_dest_index\": {\
  \    \"mappings\": {\"properties\": {\"order_count\": {\"type\": \"long\"}}}\
  \  }\
  \}"

spec :: Spec
spec = do
  describe "TransformState JSON" $ do
    it "decodes documented states" $ do
      decode "\"started\"" `shouldBe` (Just TransformStateStarted :: Maybe TransformState)
      decode "\"stopped\"" `shouldBe` (Just TransformStateStopped :: Maybe TransformState)
      decode "\"failed\"" `shouldBe` (Just TransformStateFailed :: Maybe TransformState)
      decode "\"indexing\"" `shouldBe` (Just TransformStateIndexing :: Maybe TransformState)

    it "round-trips a custom state through the escape hatch" $ do
      let Just (TransformStateCustom t) =
            decode "\"restarting\"" :: Maybe TransformState
      t `shouldBe` "restarting"
      encode (TransformStateCustom "restarting") `shouldBe` "\"restarting\""

  describe "TransformHealth JSON" $ do
    it "decodes documented health values" $ do
      decode "\"green\"" `shouldBe` (Just TransformHealthGreen :: Maybe TransformHealth)
      decode "\"yellow\"" `shouldBe` (Just TransformHealthYellow :: Maybe TransformHealth)
      decode "\"red\"" `shouldBe` (Just TransformHealthRed :: Maybe TransformHealth)

    it "round-trips a custom health through the escape hatch" $ do
      let Just (TransformHealthCustom t) =
            decode "\"unknown\"" :: Maybe TransformHealth
      t `shouldBe` "unknown"

  describe "TransformConfig JSON (PUT body)" $ do
    it "decodes a canonical pivot transform" $ do
      let Just decoded = decode samplePivotConfigBytes :: Maybe TransformConfig
      NE.toList (tsocIndex (tcSource decoded))
        `shouldBe` [IndexPattern "kibana_sample_data_ecommerce"]
      tdestIndex (tcDest decoded) `shouldBe` idxName "kibana_sample_data_ecommerce_transform"
      -- The pivot body is carried as an opaque Value.
      tcPivot decoded `shouldSatisfy` isJust
      tcLatest decoded `shouldBe` Nothing
      tcDescription decoded `shouldBe` Just "ecommerce transform"
      tcFrequency decoded `shouldBe` Just "1m"
      tcId decoded `shouldBe` Nothing

    it "decodes a latest transform with typed latest config" $ do
      let Just decoded = decode sampleLatestConfigBytes :: Maybe TransformConfig
      -- A bare-string-free array index list.
      NE.toList (tsocIndex (tcSource decoded)) `shouldBe` [IndexPattern "logs-*"]
      tdestPipeline (tcDest decoded) `shouldBe` Just "logs-pipeline"
      tcPivot decoded `shouldBe` Nothing
      let Just latest = tcLatest decoded
      NE.toList (tlUniqueKey latest)
        `shouldBe` [FieldName "host", FieldName "service"]
      tlSort latest `shouldBe` FieldName "@timestamp"

    it "preserves an opaque query, typed sync/settings and unknown fields in extras" $ do
      let Just decoded = decode sampleContinuousConfigBytes :: Maybe TransformConfig
      -- The opaque query survives on the source.
      tsocQuery (tcSource decoded) `shouldSatisfy` isJust
      -- The sync.time sub-object is typed.
      let Just sync = tcSync decoded
          Just syncTime = tsyTime sync
      tstField syncTime `shouldBe` FieldName "@timestamp"
      tstDelay syncTime `shouldBe` Just "60s"
      -- The settings scalars are typed.
      let Just settings = tcSettings decoded
      tsMaxPageSearchSize settings `shouldBe` Just 500
      tsDocsPerSecond settings `shouldBe` Just 200
      tsAlignCheckpoints settings `shouldBe` Just True
      tsDatesAsEpochMillis settings `shouldBe` Just False
      -- The unknown @created_by@ key survives in the config extras catch-all.
      KM.lookup "created_by" (tcExtras decoded) `shouldSatisfy` isJust

    it "round-trips a pivot config through encode . decode" $ do
      let Just decoded = decode samplePivotConfigBytes :: Maybe TransformConfig
          Just roundTripped = decode (encode decoded) :: Maybe TransformConfig
      NE.toList (tsocIndex (tcSource roundTripped))
        `shouldBe` [IndexPattern "kibana_sample_data_ecommerce"]
      tcDescription roundTripped `shouldBe` Just "ecommerce transform"
      tcPivot roundTripped `shouldSatisfy` isJust

    it "round-trips a continuous config with extras through encode . decode" $ do
      let Just decoded = decode sampleContinuousConfigBytes :: Maybe TransformConfig
          Just roundTripped = decode (encode decoded) :: Maybe TransformConfig
      -- The extras catch-all preserves the unknown sibling.
      KM.lookup "created_by" (tcExtras roundTripped) `shouldSatisfy` isJust
      -- The known @sync@ key is NOT duplicated into extras.
      KM.lookup "sync" (tcExtras roundTripped) `shouldBe` Nothing

    it "rejects a config missing required 'source'" $ do
      let decoded =
            decode
              "{\"dest\":{\"index\":\"d\"},\"pivot\":{}}" ::
              Maybe TransformConfig
      decoded `shouldBe` Nothing

    it "rejects a config missing required 'dest'" $ do
      let decoded =
            decode
              "{\"source\":{\"index\":\"s\"},\"pivot\":{}}" ::
              Maybe TransformConfig
      decoded `shouldBe` Nothing

    it "rejects a source with an empty 'index' array" $ do
      let decoded =
            decode
              "{\"source\":{\"index\":[]},\"dest\":{\"index\":\"d\"},\"pivot\":{}}" ::
              Maybe TransformConfig
      decoded `shouldBe` Nothing

    it "accepts a bare-string 'index' and normalises it to a singleton list" $ do
      let Just decoded =
            decode
              "{\"source\":{\"index\":\"single\"},\"dest\":{\"index\":\"d\"},\"pivot\":{}}" ::
              Maybe TransformConfig
      NE.toList (tsocIndex (tcSource decoded)) `shouldBe` [IndexPattern "single"]

  describe "TransformLatest JSON" $ do
    it "rejects a latest with an empty 'unique_key' array" $ do
      let decoded =
            decode
              "{\"unique_key\":[],\"sort\":\"ts\"}" ::
              Maybe TransformLatest
      decoded `shouldBe` Nothing

    it "accepts a bare-string 'unique_key'" $ do
      let Just decoded =
            decode
              "{\"unique_key\":\"host\",\"sort\":\"ts\"}" ::
              Maybe TransformLatest
      NE.toList (tlUniqueKey decoded) `shouldBe` [FieldName "host"]

  describe "TransformsResponse JSON (GET response)" $ do
    it "decodes the {count, transforms:[...]} envelope" $ do
      let Just decoded = decode sampleListResponseBytes :: Maybe TransformsResponse
      trCount decoded `shouldBe` Just 2
      length (trTransforms decoded) `shouldBe` 2
      let firstCfg = head (trTransforms decoded)
      tcId firstCfg `shouldBe` Just (TransformId "ecommerce-transform")
      tcPivot firstCfg `shouldSatisfy` isJust
      let secondCfg = trTransforms decoded !! 1
      tcId secondCfg `shouldBe` Just (TransformId "latest-transform")
      tcLatest secondCfg `shouldSatisfy` isJust

    it "re-encodes to the same envelope shape" $ do
      let Just decoded = decode sampleListResponseBytes :: Maybe TransformsResponse
          Just reparsed = decode (encode decoded) :: Maybe TransformsResponse
      trCount reparsed `shouldBe` Just 2
      length (trTransforms reparsed) `shouldBe` 2

  describe "TransformStatsResponse JSON" $ do
    it "decodes the state enum and document counters" $ do
      let Just decoded = decode sampleStatsResponseBytes :: Maybe TransformStatsResponse
      tstrCount decoded `shouldBe` Just 1
      length (tstrStats decoded) `shouldBe` 1
      let entry = head (tstrStats decoded)
      tstatId entry `shouldBe` Just (TransformId "ecommerce-transform")
      tstatState entry `shouldBe` Just TransformStateStarted
      tstatHealth entry `shouldBe` Just TransformHealthGreen
      tstatDocumentsIndexed entry `shouldBe` Just 1000
      tstatDocumentsProcessed entry `shouldBe` Just 1200
      -- The opaque checkpointing object survives.
      tstatCheckpointing entry `shouldSatisfy` isJust

    it "round-trips through encode . decode" $ do
      let Just decoded = decode sampleStatsResponseBytes :: Maybe TransformStatsResponse
          Just reparsed = decode (encode decoded) :: Maybe TransformStatsResponse
      tstatState (head (tstrStats reparsed)) `shouldBe` Just TransformStateStarted

  describe "TransformExplainResponse JSON" $ do
    it "decodes the transforms array carrying the id plus extras" $ do
      let Just decoded = decode sampleExplainResponseBytes :: Maybe TransformExplainResponse
      length (texrTransforms decoded) `shouldBe` 1
      let entry = head (texrTransforms decoded)
      teeId entry `shouldBe` Just (TransformId "ecommerce-transform")
      -- The version-dependent @state@ sub-object survives in extras.
      KM.lookup "state" (teeExtras entry) `shouldSatisfy` isJust

  describe "PreviewTransformResponse JSON" $ do
    it "decodes the preview array and generated dest index" $ do
      let Just decoded = decode samplePreviewResponseBytes :: Maybe PreviewTransformResponse
      prtvPreview decoded `shouldSatisfy` isJust
      prtvGeneratedDestIndex decoded `shouldSatisfy` isJust
      length <$> prtvPreview decoded `shouldBe` Just 1

  describe "PutTransformResponse / DeleteTransformResponse / StopTransformResponse JSON" $ do
    it "decodes a PUT response" $ do
      let Just decoded = decode "{\"id\":\"x\",\"acknowledged\":true}" :: Maybe PutTransformResponse
      ptrId decoded `shouldBe` Just (TransformId "x")
      ptrAcknowledged decoded `shouldBe` Just True

    it "decodes a DELETE response with deleted_tasks_count (ES 8+)" $ do
      let Just decoded =
            decode
              "{\"acknowledged\":true,\"deleted_tasks_count\":1}" ::
              Maybe DeleteTransformResponse
      dtrAcknowledged decoded `shouldBe` Just True
      dtrDeletedTasksCount decoded `shouldBe` Just 1

    it "decodes a STOP response with stopped flag (ES 8+)" $ do
      let Just decoded =
            decode
              "{\"acknowledged\":true,\"stopped\":true}" ::
              Maybe StopTransformResponse
      stopTAcknowledged decoded `shouldBe` Just True
      stopTStopped decoded `shouldBe` Just True

  describe "options params" $ do
    it "default options render no query string" $ do
      putTransformOptionsParams defaultPutTransformOptions `shouldBe` []
      getTransformsOptionsParams defaultGetTransformsOptions `shouldBe` []
      stopTransformOptionsParams defaultStopTransformOptions `shouldBe` []

    it "bool params render as true/false strings" $ do
      let opts =
            defaultPutTransformOptions
              { ptoDeferValidation = Just True
              }
      putTransformOptionsParams opts
        `shouldBe` [("defer_validation", Just "true")]

    it "from/size render via showText" $ do
      let opts =
            defaultGetTransformsOptions
              { gtoFrom = Just 10,
                gtoSize = Just 25
              }
      getTransformsOptionsParams opts
        `shouldBe` [("from", Just "10"), ("size", Just "25")]

    it "wait_for_completion passes the timeout string through" $ do
      let opts =
            defaultStopTransformOptions
              { sopoWaitForCompletion = Just "30s"
              }
      stopTransformOptionsParams opts
        `shouldBe` [("wait_for_completion", Just "30s")]

  describe "defaultTransformConfig" $ do
    it "builds a minimal config with no pivot/latest" $ do
      let cfg =
            defaultTransformConfig
              (defaultTransformSource (IndexPattern "src"))
              (defaultTransformDestination (idxName "dst"))
      NE.toList (tsocIndex (tcSource cfg)) `shouldBe` [IndexPattern "src"]
      tdestIndex (tcDest cfg) `shouldBe` idxName "dst"
      tcPivot cfg `shouldBe` Nothing
      tcLatest cfg `shouldBe` Nothing
      tcExtras cfg `shouldBe` KM.empty