legacy-seek-evidence.json

{
  "status": "Accepted causal historical reproduction and isolated scoped-lifetime differential; original failure retained",
  "fingerprint": "060dade4e8a9c764dabee54df3030fa6d857be3ea1404a2bc680a45ddadd5e4e",
  "runs": {
    "normal": {
      "log": "/tmp/rdkafka-ep14-legacy-fixtures-normal.log",
      "sha256": "1488d6d69c6e2f897b3ff2577846eaf1ab6a0d1d947a5fa75f2798fcc0375575",
      "passed": 54
    },
    "instrumented-haskell": {
      "log": "/tmp/rdkafka-ep14-legacy-fixtures-hpc.log",
      "sha256": "d5b7cb13f31913f5820f2905e79e9d0bcc8699552222b8eaec3a82c2d0defb91",
      "passed": 54
    },
    "instrumented-native": {
      "log": "/tmp/rdkafka-ep14-legacy-fixtures-asan.log",
      "sha256": "2e4ea314d564e9484263e06f21fd22c19dc3c688ef23088cf43170227dc43fd8",
      "passed": 54
    }
  },
  "mutations": "dist-mutations/20261024T230850Z-legacy-seek-88557-0/results.json",
  "compiled_mutations_caught": 2,
  "controls": 1,
  "inherited_positive_cases_retained": 10,
  "historical_failure": {
    "log": "dist-coverage/failures/ep8-sync-seek-state.log",
    "sha256": "2c45980cc04be4e3579487dd66c991d48769255ac109ae257ff61581abf0cc2b",
    "observed": "Sync consumer previously read fourteen records and reported assignment partition zero. Several blocking metadata/group/time calls intervened before seek returned State. The other four seek cases were skipped by the group dependency.",
    "native_diagnosis": "At native release 2.15.0 rd_kafka_seek waits for the fetcher result. rd_kafka_toppar_seek returns State for a fetch state below OFFSET_QUERY, excluding STOPPING, which returns PrevInProgress. Thus the assignment snapshot did not establish the later active-fetcher precondition.",
    "limit": "The saved log alone does not identify the native transition. The new native-only reproduction shows duplicate topic/partition identity; parent lifetime and late finalizer ordering are under investigation. No max-poll or broker event is asserted as the cause."
  },
  "fixture_change": "All five inherited positive offset scenarios run in both modes on fresh run-owned topics and unique grouped consumers. Each confirms exact record zero immediately before seeking, then verifies exact next identity/offset/value or documented EOF/timeout. Unassignment/State/reassignment and repeated replay remain separate tests. State is not suppressed or retried into success.",
  "native_source": {
    "project": "mori://confluentinc/librdkafka",
    "release": "2.15.0",
    "commit": "9a94e11452cdeb0a844db44ee5dd01ccbe17d3ab",
    "paths": [
      "src/rdkafka.c:4727",
      "src/rdkafka_partition.c:1950",
      "src/rdkafka_partition.h:301"
    ],
    "source_artifact_uri_pending": true
  },
  "rejected_hypothesis": {
    "log": "/tmp/rdkafka-ep14-stale-assignment.log",
    "sha256": "1f400cce426bb905b24efc6fbb5dc71258bab1e474e9d1dad5affc01dd0626a2",
    "description": "A deliberately short max-poll interval produced MAXPOLL and a stale assignment query, but seek still succeeded. This does not reproduce or explain the historical trigger."
  },
  "required_checks": "/tmp/rdkafka-ep14-legacy-fixtures-check.log",
  "adr_checks": "/tmp/rdkafka-ep14-legacy-fixtures-adr.log",
  "leak_instrumentation": false,
  "scope_decision": {
    "at": "2026-10-25",
    "user_choice": "Keep the historical investigation required",
    "completion_requirement": "Identify a reproducible supported transition that accounts for the saved condition, with concrete native/operation evidence; passing reruns and an unknown-cause limitation do not close this requirement."
  },
  "continued_forensics": {
    "historical_checkout": "fc5892d",
    "checkout_receipt": "dist-compatibility/seek-historical-fc5892d/forensic-receipt.json",
    "archive_crosscheck": "Historical mutation archive retains byte-identical ConsumerTest.hs, TestEnv.hs, Consumer.hs and ConsumerProperties.hs relative to fc5892d. The native/admin source hashes also match the preserved subject manifest. The whole failed-run fingerprint differs, so no exact whole-candidate reconstruction is claimed.",
    "traced_replay": {
      "tests": 182,
      "seconds": 100.61,
      "result": "pass; sync fetcher remains active through metadata/group/time queries and first seek is served in active state"
    },
    "forced_gc_replay": {
      "tests": 182,
      "seconds": 100.79,
      "options": [
        "-G1",
        "-A32k"
      ],
      "result": "pass with explicit performGC before every inherited consumer operation; not a reconstruction"
    },
    "broker_log": "Read-only retained primary broker logs establish the exact historical topic was created at 2026-10-24 07:33:31.078 UTC and deleted at 07:34:47.721 UTC after the suite teardown. No intervening deletion of that exact topic occurs. Default INFO logs contain no matching group/fetch-state transition, so they cannot identify the inactive-fetcher cause.",
    "accepted_cause": "A managed legacy-seek topic may outlive its native client. Its late C finalizer accesses the freed parent; when that address is reused by the next consumer, TAILQ_REMOVE writes through the old topic link into the new client topic-list head. Metadata/name lookup then constructs a second topic/partition in fetch state none while the real assigned partition remains active. Legacy seek resolves that second partition and returns State.",
    "artifacts": {
      "rdkafka-mp3-historical-trace7.log": {
        "path": "dist-compatibility/expanded-recovery-checkpoint/rdkafka-mp3-historical-trace7.log",
        "sha256": "445c86226e5c49b469933b1be9fbd18a3958d350bbaff3f976991054893c5c12"
      },
      "rdkafka-mp3-historical-gc7.log": {
        "path": "dist-compatibility/expanded-recovery-checkpoint/rdkafka-mp3-historical-gc7.log",
        "sha256": "9c0521941eae17709a7b4f504344facdf99da7c3842fb7b7c990c84dcfdc20b6"
      },
      "rdkafka-mp3-historical-broker7.log": {
        "path": "dist-compatibility/expanded-recovery-checkpoint/rdkafka-mp3-historical-broker7.log",
        "sha256": "f20d9eda2095d6564ca1dafa6640ed07d4dde43417efd3f531d22e7ab5d78634"
      }
    }
  },
  "native_trace_reproduction": {
    "status": "Native State reproduced; exact object divergence under investigation",
    "tests_failed": 5,
    "real_failure_count": 1,
    "dependency_skips": 4,
    "seconds": 99.16,
    "native_project": "mori://confluentinc/librdkafka",
    "native_revision": "9a94e11452cdeb0a844db44ee5dd01ccbe17d3ab",
    "native_change": "C-only fetch-state transitions and seek-entry trace, with stop-state backtraces; no behavior/layout change. ConsumerTest restored to original with no Haskell debug or forced GC.",
    "observed": "Consumer 73 delivers records on an active partition; first seek later enters with state none. No active-to-none transition is logged; the original active partition reaches stopping/stopped only during final teardown. Distinguish object identity before attributing the result to lost assignment or a stopped fetcher.",
    "artifacts": {
      "rdkafka-mp3-historical-native-trace9.log": {
        "path": "dist-compatibility/expanded-source9/rdkafka-mp3-historical-native-trace9.log",
        "sha256": "5c787df34cea6516047d39e72a4a327864bf83ec560fef3f8f829b0c51a08979"
      }
    },
    "identity_probe": {
      "observed": "The reproduced failing sync consumer delivers on rkt 0xba7a2c500 / partition 0xba7328500. During all-topics metadata a second rkt 0xba7a2ca00 / partition 0xba7328f00 is created for the same topic. Seek uses the second partition in state none. The original remains active until teardown. A separate native lifetime trace aborts when the preceding async consumer topic finalizer accesses its already destroyed parent mutex; parent-address reuse is under investigation.",
      "artifacts": {
        "historical-identities-pass.log": {
          "path": "dist-compatibility/expanded-source9/historical-identities-pass.log",
          "sha256": "7db8174078eac3b4efc9e1018186f2632c8e495cf9e4dc847da22054d433b798"
        },
        "historical-identities-state.log": {
          "path": "dist-compatibility/expanded-source9/historical-identities-state.log",
          "sha256": "e1717a0b11937edd5dacd526a0008ddeae4db60d3bb323db72d0e286d6b55ebb"
        },
        "historical-late-topic-finalizer-abort.log": {
          "path": "dist-compatibility/expanded-source9/historical-late-topic-finalizer-abort.log",
          "sha256": "bd5a9d023a1be7917028f3b0ba55a995932d81203837fba8dd818bad306a2f69"
        }
      },
      "limit": "Do not yet assert that the precise old saved event was a late finalizer. The observed duplicate object and separate invalid-parent destructor motivate a direct address/order probe."
    }
  },
  "causal_acceptance": {
    "historical_subject": "fc5892d core Consumer/Metadata/native API sources with original ConsumerTest and GHC coverage; no Haskell debug/forced GC in the reproducing run. Core source bytes match the original preserved mutation archive; whole failed-candidate fingerprint was not reconstructable.",
    "native_trace": "Tagged librdkafka 2.15.0 with C-only pointer/state/finalizer logging; no native behavior or opaque layout change.",
    "timeline": [
      "Async consumer 72 uses parent 0xbff0b8000, topic 0xbfee2de00 and active partition 0xbfee2e800.",
      "Sync consumer 73 reuses native parent address 0xbff0b8000 and delivers fourteen records using a different topic 0xbfee2e800 and partition 0xbfee2d900.",
      "During watermark lookup two delayed application-reference releases finalize old topic 0xbfee2de00. Its parent now reports consumer-73; the stack is runSomeFinalizers -> rd_kafka_topic_destroy_final.",
      "Native by-name lookup now reports topic count zero, while the original assigned partition has not stopped.",
      "Metadata/topic lookup constructs topic 0xbfee2de00 again with a new never-started partition 0xbfee2ca00. Seek enters that partition in state none and returns State; four dependent cases skip.",
      "A separate explicit client-lifetime trace, with no address reuse, records destruction/free of parent consumer 72 before its topic finalizer and aborts on the destroyed parent mutex. This corroborates the invalid lifetime independently of topic-list corruption."
    ],
    "mechanism_source": {
      "project": "mori://confluentinc/librdkafka",
      "version": "2.15.0",
      "commit": "9a94e11452cdeb0a844db44ee5dd01ccbe17d3ab",
      "project_relative_paths": [
        "src/rdkafka_topic.c:rd_kafka_topic_destroy_final (TAILQ_REMOVE)",
        "src/rdkafka_topic.c:rd_kafka_topic_find_fl",
        "src/rdkafka.c:rd_kafka_seek",
        "src/rdkafka_partition.c:rd_kafka_toppar_seek"
      ],
      "artifact_uri_pending": true
    },
    "minimal_differential": {
      "change": "Only legacy seek in the historical checkout is changed to retain its parent with withForeignPtr and bracket/finalize every acquired topic before returning, mirroring the already implemented current lifetime fix. No offsets, errors, waits, topic fixtures or callback mode is changed.",
      "passed": 182,
      "seconds": 99.41,
      "finalizer_order": "Both historical consumer topics finalize while their original parent is still alive; no State and no duplicate topic object."
    },
    "current_repair": {
      "commit": "2d8be3574b395cbd995b23fff09e06da03c95ae8",
      "source": "src/Kafka/Consumer.hs",
      "source_sha256": "170ea8259f8b0279db807c48799f323a268e7636efcbc929ade003cd4e4bb70a",
      "behavior": "Legacy seek retains the native client and explicitly finalizes scoped topics; current native-child Haskell finalizers also retain the parent. Existing ten fresh positive scenarios, native State propagation/recovery tests, full primary normal/mapped runs and current native diagnostic cohorts remain accepted."
    },
    "scope_limit": "This is a causal reproduction of the saved failure condition with the preserved historical core, not a claim that the untraced original process event can be observed retrospectively. The stopped-fetcher and max-poll explanations are not asserted.",
    "artifacts": {
      "historical-causal-identity.log": {
        "path": "dist-compatibility/expanded-source9/historical-causal-identity.log",
        "sha256": "7d71250fd32bf8b5773099504fe6e5a8c57ca3aa18fb435ec8c0e091964a3314"
      },
      "historical-trace-v4.dylib": {
        "path": "dist-compatibility/expanded-source9/historical-trace-v4.dylib",
        "sha256": "d43912f91b6100e52da2985ed25f812e6c4f313e6325784222c43010402c75d5"
      },
      "historical-scoped-binary": {
        "path": "dist-compatibility/expanded-source9/historical-scoped-binary",
        "sha256": "159997f6bd528ebcf7a61790ed0e3b9b0a548edfacc9250d9096de08c985eed8"
      },
      "historical-scoped-replay.log": {
        "path": "dist-compatibility/expanded-source9/historical-scoped-replay.log",
        "sha256": "d467c5510499003864488f66fb7272f327b85c5713caabf698dfdae6ed601f33"
      },
      "historical-Consumer-original.hs": {
        "path": "dist-compatibility/expanded-source9/historical-Consumer-original.hs",
        "sha256": "302dc7b476e90ec031a6a28f4dd96a0307028fa8b67889724f276ff7a77d734a"
      },
      "historical-Consumer-scoped.hs": {
        "path": "dist-compatibility/expanded-source9/historical-Consumer-scoped.hs",
        "sha256": "7c61dc0a193e3ca6250312176c6964ed21663bef609fe4b7b1eb821fbbefeb42"
      },
      "historical-client-destruction-order.log": {
        "path": "dist-compatibility/expanded-source9/historical-client-destruction-order.log",
        "sha256": "949173d4205d89c5831e0015cfe616b16656d2061f9a38e90ebaf4a2b75ec2ab"
      },
      "historical-trace-v5.dylib": {
        "path": "dist-compatibility/expanded-source9/historical-trace-v5.dylib",
        "sha256": "b1819973bc434e9a8ccb6af66697a3a53446d9d256e4ac2f1e2af87d62b8b673"
      },
      "historical-trace-v5-rdkafka.c": {
        "path": "dist-compatibility/expanded-source9/historical-trace-v5-rdkafka.c",
        "sha256": "3bbb03d7743debdcc065a173056523c221ffaedd15b38f989927568467c69a48"
      },
      "historical-trace-v5-rdkafka_topic.c": {
        "path": "dist-compatibility/expanded-source9/historical-trace-v5-rdkafka_topic.c",
        "sha256": "7bad08c3ec3c35ff819b27c0e73e0849d72999f3865a176939242443d69b27dd"
      },
      "historical-trace-v5-rdkafka_partition.c": {
        "path": "dist-compatibility/expanded-source9/historical-trace-v5-rdkafka_partition.c",
        "sha256": "c3a522c87cfc323f93ef5531f48988e738bec4dc50ded6aea4e382b98329b264"
      },
      "historical-native-trace-v5.diff": {
        "path": "dist-compatibility/expanded-source9/historical-native-trace-v5.diff",
        "sha256": "59a800395bd40922e7796458e37a08b94d65f37ac94e344fec19e693c322e101"
      }
    },
    "native_v5_sha256": "b1819973bc434e9a8ccb6af66697a3a53446d9d256e4ac2f1e2af87d62b8b673"
  }
}