diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.3.0.0 — 2026-11-14 + +### Breaking Changes + +- Add the `analyses` route group to `KansokuAPI` and the `hosts` route to `Users`, changing the named route record types. +- Extend the retention route with `period`, `startEvent`, `returnEvent`, and separate start and return filter parameters. + +### New Features + +- Add `Kansoku.Analysis.Api` and `Kansoku.Analysis.Types` for revisioned, account-owned opaque analysis documents with keyset pagination. +- Add `RetentionPeriod` for exact UTC day, Monday-week, and calendar-month retention cohorts. +- Add `UserHost` and `UserHosts` for exact per-host counts over a user's retained history, and allow filtering user event history by host. +- Add analysis problem types, including `409` revision-conflict and limit responses, to the problem catalog and generated OpenAPI. + +### Bug Fixes + +- Document analysis collection validation problems separately from Relay pagination errors. +- Keep the analysis document capacity independent of the configured ingest body limit. + ## 0.2.0.0 — 2026-11-12 ### Breaking Changes diff --git a/kansoku-api.cabal b/kansoku-api.cabal --- a/kansoku-api.cabal +++ b/kansoku-api.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: kansoku-api -version: 0.2.0.0 +version: 0.3.0.0 synopsis: Servant API and wire DTO types for Kansoku. description: Shared Servant route types, analytics event types, metric DTOs, validation, @@ -32,6 +32,8 @@ import: common hs-source-dirs: src exposed-modules: + Kansoku.Analysis.Api + Kansoku.Analysis.Types Kansoku.Api Kansoku.Api.OpenApi Kansoku.Api.Response diff --git a/src/Kansoku/Analysis/Api.hs b/src/Kansoku/Analysis/Api.hs new file mode 100644 --- /dev/null +++ b/src/Kansoku/Analysis/Api.hs @@ -0,0 +1,60 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE UndecidableInstances #-} + +module Kansoku.Analysis.Api where + +import Data.SOP (I (..), NS (..)) +import Data.UUID (UUID) +import GHC.TypeLits (Symbol) +import Kansoku.Analysis.Types +import Kansoku.Api.Response +import Kansoku.Auth.Api (MetricsAuth) +import Kansoku.Cohort.Api (PageReply) +import Kansoku.Prelude +import Relay.Pagination.Servant (RelayPage) +import Servant.API +import Servant.API.MultiVerb + +data AnalysisResult a + = AnalysisOk a + | AnalysisBadRequest ProblemDetails + | AnalysisNotFound ProblemDetails + | AnalysisConflict ProblemDetails + | AnalysisUnavailable ProblemDetails + deriving stock (Generic, Eq, Show) + +type AnalysisResponses (desc :: Symbol) a = + '[ RespondAs JSON 200 desc a, + RespondAs ProblemJSON 400 "Invalid analysis" ProblemDetails, + RespondAs ProblemJSON 404 "Analysis not found" ProblemDetails, + RespondAs ProblemJSON 409 "Revision conflict or account limit reached" ProblemDetails, + RespondAs ProblemJSON 503 "Analysis store unavailable" ProblemDetails + ] + +instance AsUnion (AnalysisResponses desc a) (AnalysisResult a) where + toUnion = \case + AnalysisOk x -> Z (I x) + AnalysisBadRequest x -> S (Z (I x)) + AnalysisNotFound x -> S (S (Z (I x))) + AnalysisConflict x -> S (S (S (Z (I x)))) + AnalysisUnavailable x -> S (S (S (S (Z (I x))))) + fromUnion = \case + Z (I x) -> AnalysisOk x + S (Z (I x)) -> AnalysisBadRequest x + S (S (Z (I x))) -> AnalysisNotFound x + S (S (S (Z (I x)))) -> AnalysisConflict x + S (S (S (S (Z (I x))))) -> AnalysisUnavailable x + S (S (S (S (S impossible)))) -> case impossible of {} + +type AtAnalysis sub = "analyses" :> MetricsAuth :> Capture "id" UUID :> sub + +type AnalysisReply method a = MultiVerb method '[JSON, ProblemJSON] (AnalysisResponses "Saved analysis result" a) (AnalysisResult a) + +data Analyses mode = Analyses + { createAnalysis :: mode :- "analyses" :> MetricsAuth :> Description "Save opaque client state without executing or interpreting it. Names are trimmed, 1..120 Unicode characters; descriptions retain whitespace, maximum 500 characters. Metadata rejects zero characters. documentVersion is a positive integer; document must be an object, maximum 32768 compact Aeson UTF-8 bytes. Raw analysis request bodies have a separate 5 MiB transport limit, independent of the ingest body setting. Duplicate names are allowed. The positive KANSOKU_ANALYSES_MAX_PER_ACCOUNT limit defaults to 1000 and must match on all replicas. A full account returns non-retryable 409 analysis_limit_reached." :> ReqBody '[JSON] AnalysisInput :> AnalysisReply 'POST SavedAnalysis, + getAnalysis :: mode :- AtAnalysis (AnalysisReply 'GET SavedAnalysis), + listAnalyses :: mode :- "analyses" :> MetricsAuth :> Description "Current-read pages ordered by updatedAt descending, id ascending. Default 20, maximum 100. Both directions keep canonical order. Cursors preserve microseconds, survive deletion/replacement of their source row, and must be reused with the same kind filter. No cross-request snapshot; refresh from the first page after edits. Invalid kind returns a 400 problem document; invalid pagination uses the existing Relay JSON error." :> QueryParam "kind" AnalysisKind :> RelayPage 20 100 :> PageReply SavedAnalysis, + replaceAnalysis :: mode :- AtAnalysis (Description "Full replacement with required positive revision from the last read. Stale revision returns 409 analysis_revision_conflict. Every successful replacement increments revision, preserves createdAt and advances updatedAt, including identical content. The same field and byte limits as create apply." :> ReqBody '[JSON] AnalysisReplace :> AnalysisReply 'PUT SavedAnalysis), + deleteAnalysis :: mode :- AtAnalysis (Description "Delete with the required positive revision from the last read. Stale revision returns 409; missing or foreign IDs return 404, including repeated deletion." :> QueryParam' '[Required, Strict] "revision" Revision :> AnalysisReply 'DELETE Bool) + } + deriving stock (Generic) diff --git a/src/Kansoku/Analysis/Types.hs b/src/Kansoku/Analysis/Types.hs new file mode 100644 --- /dev/null +++ b/src/Kansoku/Analysis/Types.hs @@ -0,0 +1,85 @@ +-- | Opaque, account-owned client documents. Kansoku never executes these values. +module Kansoku.Analysis.Types where + +import Data.Aeson qualified as A +import Data.Int (Int64) +import Data.Text qualified as Text +import Data.UUID (UUID) +import Kansoku.Prelude +import Servant.API (FromHttpApiData (..), ToHttpApiData (..)) +import Text.Read (readMaybe) + +data AnalysisKind = Trends | Funnel | Retention + deriving stock (Generic, Eq, Show) + +kindText :: AnalysisKind -> Text +kindText = \case Trends -> "trends"; Funnel -> "funnel"; Retention -> "retention" + +instance FromHttpApiData AnalysisKind where + parseUrlPiece = \case + "trends" -> Right Trends + "funnel" -> Right Funnel + "retention" -> Right Retention + _ -> Left "kind: expected trends, funnel, or retention" + +instance ToHttpApiData AnalysisKind where toUrlPiece = kindText + +instance ToJSON AnalysisKind where toJSON = A.String . kindText + +instance FromJSON AnalysisKind where + parseJSON = A.withText "kind" (either (fail . Text.unpack) pure . parseUrlPiece) + +-- | PostgreSQL bigint, always positive. Increment is checked by the store. +newtype Revision = Revision {unRevision :: Int64} + deriving stock (Generic, Eq, Show) + deriving newtype (ToJSON) + +instance FromJSON Revision where + parseJSON value = do + n <- parseJSON value + if n > 0 then pure (Revision n) else fail "revision: expected a positive bigint" + +instance FromHttpApiData Revision where + parseUrlPiece raw = case readMaybe (Text.unpack raw) of + Just n | n > 0 -> Right (Revision n) + _ -> Left "revision: expected a positive bigint" + +instance ToHttpApiData Revision where toUrlPiece = Text.pack . show . unRevision + +data AnalysisInput = AnalysisInput + { name :: !Text, + description :: !(Maybe Text), + kind :: !AnalysisKind, + documentVersion :: !Integer, + document :: !A.Value + } + deriving stock (Generic, Eq, Show) + deriving anyclass (FromJSON, ToJSON) + +data AnalysisReplace = AnalysisReplace + { name :: !Text, + description :: !(Maybe Text), + kind :: !AnalysisKind, + documentVersion :: !Integer, + document :: !A.Value, + revision :: !Revision + } + deriving stock (Generic, Eq, Show) + deriving anyclass (FromJSON, ToJSON) + +replacementInput :: AnalysisReplace -> AnalysisInput +replacementInput AnalysisReplace {name, description, kind, documentVersion, document} = AnalysisInput {name, description, kind, documentVersion, document} + +data SavedAnalysis = SavedAnalysis + { id :: !UUID, + name :: !Text, + description :: !(Maybe Text), + kind :: !AnalysisKind, + documentVersion :: !Integer, + document :: !A.Value, + revision :: !Revision, + createdAt :: !UTCTime, + updatedAt :: !UTCTime + } + deriving stock (Generic, Eq, Show) + deriving anyclass (FromJSON, ToJSON) diff --git a/src/Kansoku/Api.hs b/src/Kansoku/Api.hs --- a/src/Kansoku/Api.hs +++ b/src/Kansoku/Api.hs @@ -6,6 +6,7 @@ ) where +import Kansoku.Analysis.Api (Analyses) import Kansoku.Cohort.Api (Cohorts) import Kansoku.Event.Contract (SchemaCatalog) import Kansoku.Ingest.Api (IngestRoutes) @@ -21,6 +22,7 @@ { ingest :: mode :- NamedRoutes IngestRoutes, metrics :: mode :- NamedRoutes MetricsRoutes, metricDefinitions :: mode :- NamedRoutes MetricDefinitions, + analyses :: mode :- NamedRoutes Analyses, users :: mode :- NamedRoutes Users, cohorts :: mode :- NamedRoutes Cohorts, schemas :: mode :- SchemaAPI diff --git a/src/Kansoku/Api/OpenApi.hs b/src/Kansoku/Api/OpenApi.hs --- a/src/Kansoku/Api/OpenApi.hs +++ b/src/Kansoku/Api/OpenApi.hs @@ -10,9 +10,11 @@ import Data.Aeson qualified as A import Data.Char (toLower, toUpper) +import Data.Int (Int64) import Data.OpenApi qualified as O import Data.OpenApi.Declare (runDeclare) import Data.Text qualified as Text +import Kansoku.Analysis.Types import Kansoku.Api (kansokuAPI) import Kansoku.Api.Response (ProblemDetails, problemFieldName) import Kansoku.Cohort.Types @@ -35,7 +37,7 @@ VersionedEventSchema, ) import Kansoku.Ingest.Api (BatchAck, BeaconEnvelope, IngestAck, IngestEvent, RejectedEvent) -import Kansoku.Metrics.Api (AcquisitionMeasure, AggregateBy, CohortMode, ExperimentMeasure, FunnelAttribution, FunnelCounting, FunnelMeasure, FunnelMode, MetricPoint, MetricResult, OrganizationAttribution, PathBy, RetentionMode, SessionMeasure, TimeGrain, UserWindow) +import Kansoku.Metrics.Api (AcquisitionMeasure, AggregateBy, CohortMode, ExperimentMeasure, FunnelAttribution, FunnelCounting, FunnelMeasure, FunnelMode, MetricPoint, MetricResult, OrganizationAttribution, PathBy, RetentionMode, RetentionPeriod, SessionMeasure, TimeGrain, UserWindow) import Kansoku.Metrics.Definition import Kansoku.Metrics.Filter (FilterOp, MetricFilter) import Kansoku.Metrics.Limits (maxMetricPoints) @@ -197,6 +199,9 @@ instance O.ToParamSchema UserWindow where toParamSchema _ = O.toParamSchema (Proxy @Text) & O.enum_ ?~ ["calendar", "rolling7", "rolling30"] +instance O.ToParamSchema RetentionPeriod where + toParamSchema _ = O.toParamSchema (Proxy @Text) & O.enum_ ?~ ["day", "week", "month"] + instance O.ToParamSchema RetentionMode where toParamSchema _ = O.toParamSchema (Proxy @Text) & O.enum_ ?~ ["on", "on_or_after"] @@ -275,7 +280,7 @@ & withUnsupportedBodyMedia & withRetryAfterHeader & withOperationIds - & withUserPageProblems + & withSelectionPageProblems authSecuritySchemes :: O.SecurityDefinitions authSecuritySchemes = @@ -498,6 +503,16 @@ named <- O.genericDeclareNamedSchema O.defaultSchemaOptions proxy pure (nullableUserFields ["firstActivityAt", "lastActivityAt"] named & O.schema . O.required .~ ["userId", "traits", "firstActivityAt", "lastActivityAt"]) +instance O.ToSchema UserHosts where + declareNamedSchema proxy = do + named <- O.genericDeclareNamedSchema O.defaultSchemaOptions proxy + pure (named & O.schema . O.properties . ix "hosts" %~ (\case O.Inline schema -> O.Inline (schema & O.maxItems ?~ 1000); reference -> reference)) + +instance O.ToSchema UserHost where + declareNamedSchema proxy = do + named <- O.genericDeclareNamedSchema O.defaultSchemaOptions proxy + pure (nullableUserFields ["host"] named & O.schema . O.required .~ ["host", "eventCount", "lastOccurredAt"]) + instance O.ToSchema UserActivityCounts instance O.ToSchema A.Value where @@ -542,27 +557,73 @@ where addNullable field result = result & O.schema . O.properties . at field ?~ O.Inline (mempty & O.type_ ?~ O.OpenApiTypeArray [O.OpenApiString, O.OpenApiNull]) --- User selections also fail before Relay parsing, with the shared problem body. -withUserPageProblems :: O.OpenApi -> O.OpenApi -withUserPageProblems = +-- User selections and analysis kinds can fail before Relay parsing, with a problem body. +withSelectionPageProblems :: O.OpenApi -> O.OpenApi +withSelectionPageProblems = O.paths %~ imap ( \path item -> - if path `elem` ["/users", "/users/{userId}/events"] + if path `elem` ["/users", "/users/{userId}/events", "/analyses"] then item & O.get . _Just %~ ( \operation -> case operation ^. at 400 of - Just (O.Inline response) -> pageBounds operation & at 400 ?~ O.Inline (response & O.content . at "application/problem+json" ?~ (mempty & O.schema ?~ problemSchema)) - _ -> pageBounds operation + Just (O.Inline response) -> bounds path operation & at 400 ?~ O.Inline (response & O.content . at "application/problem+json" ?~ (mempty & O.schema ?~ problemSchema)) + _ -> bounds path operation ) else item ) where (_, problemSchema) = runDeclare (O.declareSchemaRef (Proxy @ProblemDetails)) mempty + bounds "/analyses" operation = operation + bounds _ operation = pageBounds operation + pageBounds operation = operation & O.parameters %~ map bound bound (O.Inline parameter) | parameter ^. O.name `elem` ["first", "last"] = O.Inline (parameter & O.schema . _Just %~ (\case O.Inline schema -> O.Inline (schema & O.minimum_ ?~ 1); reference -> reference)) bound parameter = parameter + +instance O.ToSchema AnalysisKind where + declareNamedSchema = O.genericDeclareNamedSchema lowerEnumOptions + +instance O.ToParamSchema AnalysisKind where + toParamSchema = O.genericToParamSchema lowerEnumOptions + +instance O.ToSchema Revision where + declareNamedSchema _ = pure (O.NamedSchema (Just "Revision") (O.toSchema (Proxy @Int64) & O.minimum_ ?~ 1)) + +instance O.ToParamSchema Revision where + toParamSchema _ = O.toParamSchema (Proxy @Int64) & O.minimum_ ?~ 1 + +instance O.ToSchema AnalysisInput where + declareNamedSchema proxy = analysisEnvelopeSchema <$> O.genericDeclareNamedSchema O.defaultSchemaOptions proxy + +instance O.ToSchema AnalysisReplace where + declareNamedSchema proxy = analysisEnvelopeSchema <$> O.genericDeclareNamedSchema O.defaultSchemaOptions proxy + +instance O.ToSchema SavedAnalysis where + declareNamedSchema proxy = do + named <- O.genericDeclareNamedSchema O.defaultSchemaOptions proxy + pure (analysisEnvelopeSchema named & O.schema . O.required %~ ("description" :)) + +analysisEnvelopeSchema :: O.NamedSchema -> O.NamedSchema +analysisEnvelopeSchema named = + named + & O.schema + . O.properties + . ix "name" + .~ O.Inline (mempty & O.type_ ?~ O.OpenApiTypeSingle O.OpenApiString & O.minLength ?~ 1 & O.maxLength ?~ 120 & O.description ?~ "Trimmed name; Unicode character count; no zero characters") + & O.schema + . O.properties + . ix "document" + .~ O.Inline (mempty & O.type_ ?~ O.OpenApiTypeSingle O.OpenApiObject & O.additionalProperties ?~ O.AdditionalPropertiesAllowed True & O.description ?~ "Opaque JSON object, never executed or interpreted. Maximum 32768 bytes of compact Aeson UTF-8 encoding; nested values, including escaped zero characters, are preserved.") + & O.schema + . O.properties + . ix "documentVersion" + .~ O.Inline (mempty & O.type_ ?~ O.OpenApiTypeSingle O.OpenApiInteger & O.minimum_ ?~ 1 & O.description ?~ "Positive integral client document version") + & O.schema + . O.properties + . ix "description" + %~ (\case O.Inline schema -> O.Inline (schema & O.type_ ?~ O.OpenApiTypeArray [O.OpenApiString, O.OpenApiNull] & O.maxLength ?~ 500 & O.description ?~ "Optional or null; whitespace retained; maximum 500 Unicode characters; no zero characters"); reference -> reference) diff --git a/src/Kansoku/Api/Response.hs b/src/Kansoku/Api/Response.hs --- a/src/Kansoku/Api/Response.hs +++ b/src/Kansoku/Api/Response.hs @@ -27,7 +27,7 @@ import GHC.TypeLits (Symbol) import Kansoku.Prelude import Network.HTTP.Media ((//)) -import Network.HTTP.Types.Status (status400, status401, status403, status404, status405, status406, status413, status415, status429, status500, status503, statusMessage) +import Network.HTTP.Types.Status (status400, status401, status403, status404, status405, status406, status409, status413, status415, status429, status500, status503, statusMessage) import Servant.API (Accept (..), Header, JSON, MimeRender (..), MimeUnrender (..)) import Servant.API.MultiVerb (AsUnion (..), RespondAs, WithHeaders) @@ -171,6 +171,7 @@ 404 -> status404 405 -> status405 406 -> status406 + 409 -> status409 413 -> status413 415 -> status415 429 -> status429 @@ -197,7 +198,12 @@ -- from 'problem'; human-readable detail is supplied by the failure producer. problemCatalog :: [(Text, Int)] problemCatalog = - [ ("invalid_cohort", 400), + [ ("invalid_analysis", 400), + ("analysis_not_found", 404), + ("analysis_revision_conflict", 409), + ("analysis_limit_reached", 409), + ("analysis_store_unavailable", 503), + ("invalid_cohort", 400), ("cohort_not_found", 404), ("cohort_store_unavailable", 503), ("invalid_date_range", 400), diff --git a/src/Kansoku/Metrics/Api.hs b/src/Kansoku/Metrics/Api.hs --- a/src/Kansoku/Metrics/Api.hs +++ b/src/Kansoku/Metrics/Api.hs @@ -8,6 +8,7 @@ SessionMeasure (..), UserWindow (..), RetentionMode (..), + RetentionPeriod (..), CohortMode (..), PathBy (..), FunnelMode (..), @@ -171,6 +172,21 @@ toUrlPiece Rolling7 = "rolling7" toUrlPiece Rolling30 = "rolling30" +-- | UTC calendar unit for retention cohorts and offsets, independent of output grain. +data RetentionPeriod = RetentionDay | RetentionWeek | RetentionMonth + deriving stock (Generic, Eq, Show) + +instance FromHttpApiData RetentionPeriod where + parseUrlPiece "day" = Right RetentionDay + parseUrlPiece "week" = Right RetentionWeek + parseUrlPiece "month" = Right RetentionMonth + parseUrlPiece _ = Left "period must be day, week or month" + +instance ToHttpApiData RetentionPeriod where + toUrlPiece RetentionDay = "day" + toUrlPiece RetentionWeek = "week" + toUrlPiece RetentionMonth = "month" + data RetentionMode = RetentionOn | RetentionOnOrAfter deriving stock (Generic, Eq, Show) @@ -361,8 +377,8 @@ :- MetricsAuth :> "metrics" :> "retention" - :> Description "user_id is filter-only: eq/neq accept one nonempty opaque string; in accepts 1..20 raw string IDs before deduplication (1024 UTF-8 bytes each). Only identified subjects match, including neq; anonymous/missing subjects fail each term. Resolved event identity includes pre-login history; sessions/acquisition select the first resolved user and retain the whole session. Supported only on events, page-views, users (calendar/rolling7/rolling30), sessions, acquisition, revenue, feature-adoption and custom. Other routes return invalid_metric; groupBy=user_id returns invalid_group_by. Saved simple defaults AND with requests; ratio component selection stays side-local. Existing route grains, populations, budgets and 10000 complete-point cap apply; selecting one user is not a keyed lookup. Exact canonical retention over closed UTC days. mode=on (default) counts the exact offset day; on_or_after counts any day from offset through the largest requested offset, including unrequested intermediate days. cohort=window (default) means first matching activity in range; lifetime means first matching activity in retained history, not first ever. Date bounds select the observed activity interval and cohort dates. Offset 0 always emits cohort size. Omit immature on offsets; on_or_after omits nonzero offsets unless the full maximum horizon is closed by to. to cannot exceed the current UTC day. Day grain only; filters/grouping define identical populations throughout lookback. Bot rows are excluded unless includeBots=true. Up to two dimensions, eight bounded JSON filters, 32 distinct offsets 1..365, range at most 366 days and 10000 complete points." - :> MetricArguments (QueryParam "days" Text :> QueryParam "mode" RetentionMode :> QueryParam "cohort" CohortMode :> QueryParam "includeBots" Bool :> OrganizationOptions MetricResponse), + :> Description "user_id is filter-only: eq/neq accept one nonempty opaque string; in accepts 1..20 raw string IDs before deduplication (1024 UTF-8 bytes each). Only identified subjects match, including neq; anonymous/missing subjects fail each term. Resolved event identity includes pre-login history; sessions/acquisition select the first resolved user and retain the whole session. Supported only on events, page-views, users (calendar/rolling7/rolling30), sessions, acquisition, revenue, feature-adoption and custom. Other routes return invalid_metric; groupBy=user_id returns invalid_group_by. Saved simple defaults AND with requests; ratio component selection stays side-local. Existing route grains, populations, budgets and 10000 complete-point cap apply; selecting one user is not a keyed lookup. Exact canonical retention over closed UTC periods. period=day (default), week (UTC Monday), or calendar month controls cohorts and days offset units independently of timeGrain (omit or day only). from must be Monday for week or day one for month; misalignment is invalid_retention_period. to may be unaligned but cannot exceed the current UTC day. Cohort labels are the period first day, YYYY-MM-DD. days is ascending distinct positive offsets: at most 32 in 1..365 days, 1..52 weeks, 1..12 months; invalid_retention_days or too_many_retention_days rejects overflow. Defaults are [1,7,14,30], [1,2,4,8], [1,3,6,12] respectively. Offset 0 always emits observed cohort size, including partial final cohorts. mode=on (default) counts once in the full target period [cohort+N,cohort+N+1); omit immature offsets. on_or_after counts once from target start through the largest requested period, including unrequested intermediate periods; omit all nonzero offsets unless that full maximum horizon is closed by to. Mature missing returns are zero. cohort=window (default) means first matching activity in range; lifetime means first matching activity in retained history using start selection, not first ever. startEvent and returnEvent each independently default to any matching activity. Supplied names are trimmed single 1..128 lowercase ASCII letters, digits, underscore, dot, colon or hyphen; invalid_retention_event rejects empty/invalid names, valid unmatched names give empty starts or zero returns. Repeatable startFilter and returnFilter use the shared filter grammar and apply only on that side; shared filter applies to both. Across all three lists one eight-filter/16384 decoded UTF-8 byte budget applies (invalid_filter). Side filters accept event dimensions and governed scalar properties, including organization IDs; cohort, cohort:id, user:*, org:*, user_id and session:* terms (also inside OR) are invalid_retention_filter. Shared current traits and saved membership retain their existing contract; user_id and session selectors remain unsupported on retention. Start activity labels each group; only return activity with equal dimension values counts, including nullable groups and event_name. Groups are separate populations and need not be disjoint. Bot/profile-only exclusions apply on both sides; includeBots=true includes bot rows. Up to two dimensions, range at most 366 days, 10000 complete points; metric_result_too_large and query_resource_limit return complete errors." + :> MetricArguments (QueryParam "days" Text :> QueryParam "mode" RetentionMode :> QueryParam "cohort" CohortMode :> QueryParam "includeBots" Bool :> OrganizationOptions (QueryParam "period" RetentionPeriod :> QueryParam "startEvent" Text :> QueryParam "returnEvent" Text :> QueryParams "startFilter" MetricFilter :> QueryParams "returnFilter" MetricFilter :> MetricResponse)), acquisition :: mode :- MetricsAuth :> "metrics" :> "acquisition" :> AcquisitionQuery, featureAdoption :: mode :- MetricsAuth :> "metrics" :> "feature-adoption" :> OrganizationMetricQuery, conversion :: mode :- MetricsAuth :> "metrics" :> "conversion" :> Description "user_id is filter-only: eq/neq accept one nonempty opaque string; in accepts 1..20 raw string IDs before deduplication (1024 UTF-8 bytes each). Only identified subjects match, including neq; anonymous/missing subjects fail each term. Resolved event identity includes pre-login history; sessions/acquisition select the first resolved user and retain the whole session. Supported only on events, page-views, users (calendar/rolling7/rolling30), sessions, acquisition, revenue, feature-adoption and custom. Other routes return invalid_metric; groupBy=user_id returns invalid_group_by. Saved simple defaults AND with requests; ratio component selection stays side-local. Existing route grains, populations, budgets and 10000 complete-point cap apply; selecting one user is not a keyed lookup. Exact ordered signup conversion: starters are the denominator; only starters with a later signup_completed in the millisecond/event-ID timeline within range (window capped at 90 days) convert. Zero denominator returns zero; ratio stays 0..1. Group ownership is the earliest eligible start. timeGrain and prop grouping are rejected." :> OrganizationMetricQuery, diff --git a/src/Kansoku/Metrics/Limits.hs b/src/Kansoku/Metrics/Limits.hs --- a/src/Kansoku/Metrics/Limits.hs +++ b/src/Kansoku/Metrics/Limits.hs @@ -7,6 +7,8 @@ maxFunnelSteps, maxRetentionDays, maxRetentionOffset, + maxRetentionWeekOffset, + maxRetentionMonthOffset, maxMetricPoints, maxPathSteps, maxPathTop, @@ -38,3 +40,7 @@ maxSelectedUserIds :: Int maxSelectedUserIds = 20 + +maxRetentionWeekOffset, maxRetentionMonthOffset :: Int +maxRetentionWeekOffset = 52 +maxRetentionMonthOffset = 12 diff --git a/src/Kansoku/Users/Api.hs b/src/Kansoku/Users/Api.hs --- a/src/Kansoku/Users/Api.hs +++ b/src/Kansoku/Users/Api.hs @@ -19,7 +19,8 @@ data Users mode = Users { activityCounts :: mode :- "users" :> "activity-counts" :> MetricsAuth :> Description "Exact distinct identified and anonymous activity subjects. Required paired UTC dates [from,to), 1..366 days; includeBots defaults false. Excludes identify/identify_once/alias/group and empty subjects. Sum equals whole-range metrics/users with timeGrain=none. No directory filters." :> RequiredDates (QueryParam "includeBots" Bool :> MultiVerb 'GET '[JSON, ProblemJSON] (OkResponses "Exact activity counts" UserActivityCounts) (KansokuResult UserActivityCounts)), - events :: mode :- "users" :> MetricsAuth :> Capture "userId" Text :> "events" :> Description "Resolved retained history including pre-login and identity events. Paired optional UTC dates [from,to), 1..366 days; includeBots defaults false. One event_name eq/in filter, 1..100 names of 1..128 characters. Occurrence/receipt/event ID descending. Exact-version declared scalar properties only, excluding identifiers, alias payloads and context. Identity/group properties require governed catalogs. Relay page size 1..100, default 20; both directions, current reads without snapshot. Unknown/foreign/erased user: user_not_found 404. Nonpagination errors use problem+json; Relay pagination errors use JSON." :> Dates (QueryParam "includeBots" Bool :> QueryParams "filter" MetricFilter :> RelayPage 20 100 :> PageReply UserEvent), + events :: mode :- "users" :> MetricsAuth :> Capture "userId" Text :> "events" :> Description "Resolved retained history including pre-login and identity events. Paired optional UTC dates [from,to), 1..366 days; includeBots defaults false. At most one event_name eq/in term (1..100 names of 1..128 characters) and one host eq/in term (1..100 canonical lowercase hosts of 1..253 characters without scheme, port, path, whitespace or trailing dot), ANDed. Host matches node host; null hosts never match a host term. Occurrence/receipt/event ID descending. Exact-version declared scalar properties only, excluding identifiers, alias payloads and context. Identity/group properties require governed catalogs. Relay page size 1..100, default 20; both directions, current reads without snapshot. Unknown/foreign/erased user: user_not_found 404. Nonpagination errors use problem+json; Relay pagination errors use JSON." :> Dates (QueryParam "includeBots" Bool :> QueryParams "filter" MetricFilter :> RelayPage 20 100 :> PageReply UserEvent), + hosts :: mode :- "users" :> MetricsAuth :> Capture "userId" Text :> "hosts" :> Description "Complete exact host counts for resolved retained history, including identity and pre-login events. Optional paired UTC dates [from,to), 1..366 days; includeBots defaults false. At most one event_name eq/in filter (1..100 names of 1..128 characters); host filters rejected. Same selection as history. Non-null hosts ordered by eventCount descending then host ascending; one null-host entry last, zero counts omitted. Sum of selected entries equals filtered history size for the same parameters on settled data; current reads without cross-request snapshot. At most 1000 entries including null, otherwise user_hosts_too_large 400 without partial output. Unknown/foreign/erased user: user_not_found 404; query failure/deadline: clickhouse_unavailable 503." :> Dates (QueryParam "includeBots" Bool :> QueryParams "filter" MetricFilter :> MultiVerb 'GET '[JSON, ProblemJSON] (OkResponses "User hosts" UserHosts) (KansokuResult UserHosts)), getProfile :: mode :- "users" :> MetricsAuth :> Capture "userId" Text :> Description "Current identified-user profile. RetainedEventCount includes all event kinds and bots; activity extrema exclude bots and identify/identify_once/alias/group. Traits include winning observedAt and set/set_once provenance. Evidence counts never expose values and fingerprint validity is 180 days; resolvedAnonymousCount counts original anonymous subjects stitched to this user. Organization is current explicit membership with governed traits or null. Unknown, foreign and erased users share user_not_found 404. One deadline and owned connection, eventual cross-table consistency. Analytical inference, never authorization." :> MultiVerb 'GET '[JSON, ProblemJSON] (OkResponses "User profile" UserProfile) (KansokuResult UserProfile), directory :: mode :- "users" :> MetricsAuth :> Description "Identified-user Relay directory. Optional paired UTC dates [from,to), 1..366 days, select activity but extrema span all retained history. search is literal case-insensitive substring, 1..256 code points. order is lastActivity (default) or firstActivity: timestamp descending, null last, userId ascending. includeBots defaults false; identify/identify_once/alias/group are not activity. Without dates profile-only users have null extrema. Up to eight typed user trait filters and one cohort eq/neq UUID, AND only, 16384 decoded bytes. Page size 1..100, default 20, first/after or last/before. Keep selections fixed; current reads require refresh/deduplication after concurrent changes. Exhaustion is a complete error; unknown/repeated scalar parameters are rejected. Relay pagination errors use JSON, other errors problem+json." :> Dates (QueryParam "search" Text :> QueryParam "order" Text :> QueryParam "includeBots" Bool :> QueryParams "filter" MetricFilter :> RelayPage 20 100 :> PageReply UserSummary) } diff --git a/src/Kansoku/Users/Types.hs b/src/Kansoku/Users/Types.hs --- a/src/Kansoku/Users/Types.hs +++ b/src/Kansoku/Users/Types.hs @@ -74,3 +74,27 @@ <*> o Data.Aeson..: "isPageView" <*> o Data.Aeson..: "properties" <*> pure "" + +data UserHost = UserHost + { host :: !(Maybe Text), + eventCount :: !Integer, + lastOccurredAt :: !Text + } + deriving stock (Generic, Eq, Show) + +instance ToJSON UserHost where + toJSON UserHost {host, eventCount, lastOccurredAt} = + Data.Aeson.object ["host" Data.Aeson..= host, "eventCount" Data.Aeson..= eventCount, "lastOccurredAt" Data.Aeson..= lastOccurredAt] + +instance FromJSON UserHost where + parseJSON = Data.Aeson.withObject "user host" $ \o -> + UserHost <$> o Data.Aeson..: "host" <*> o Data.Aeson..: "eventCount" <*> o Data.Aeson..: "lastOccurredAt" + +newtype UserHosts = UserHosts {hosts :: [UserHost]} + deriving stock (Generic, Eq, Show) + +instance ToJSON UserHosts where + toJSON UserHosts {hosts} = Data.Aeson.object ["hosts" Data.Aeson..= hosts] + +instance FromJSON UserHosts where + parseJSON = Data.Aeson.withObject "user hosts" $ \o -> UserHosts <$> o Data.Aeson..: "hosts"