feat(stats): metric hours (#71552)

src/sentry/api/endpoints/organization_stats_v2.py

@@ -169,8 +169,9 @@ def get(self, request: Request, organization) -> Response:
 
             if features.has("organizations:custom-metrics", organization):
                 if (
-                    request.GET.get("category") == "metrics"
-                    or request.GET.get("category") == "metricSecond"
+                    request.GET.get("category") == "metricSecond"  # TODO(metrics): remove this
+                    or request.GET.get("category") == "metricHour"
+                    or request.GET.get("category") == "metricOutcomes"
                 ):
                     # TODO(metrics): align project resolution
                     result = run_metrics_outcomes_query(

src/sentry/sentry_metrics/querying/data/transformation/stats.py

@@ -1,19 +1,28 @@
 from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
 from typing import Any
 
 from sentry.sentry_metrics.querying.data.execution import QueryResult
 from sentry.sentry_metrics.querying.data.transformation.base import QueryResultsTransformer
+from sentry.utils.outcomes import Outcome
 
 
-class MetricsStatsTransformer(QueryResultsTransformer[Mapping[str, Any]]):
+@dataclass(frozen=True)
+class MetricsOutcomesResult:
+    series: Sequence[Mapping[str, Any]]
+    totals: Sequence[Mapping[str, Any]]
+
+
+class MetricsOutcomesTransformer(QueryResultsTransformer[Mapping[str, Any]]):
     def transform_result(self, result: Sequence[Mapping[str, Any]]) -> Sequence[Mapping[str, Any]]:
         ret_val = []
 
         for item in result:
             ret_val_item = {}
             for key in item:
                 if key == "outcome.id":
-                    ret_val_item["outcome"] = int(item[key])
+                    outcome = int(item[key])
+                    ret_val_item["outcome"] = Outcome(outcome).api_name()
                 elif key in "aggregate_value":
                     ret_val_item["quantity"] = item[key]
                 else:

src/sentry/sentry_metrics/querying/stats.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from datetime import datetime
+
+from django.http import QueryDict
+
+from sentry.api.utils import get_date_range_from_params
+from sentry.models.environment import Environment
+from sentry.models.organization import Organization
+from sentry.models.project import Project
+from sentry.sentry_metrics.querying.data import MQLQuery, run_queries
+from sentry.sentry_metrics.querying.data.transformation.stats import MetricsOutcomesTransformer
+from sentry.snuba.referrer import Referrer
+from sentry.snuba.sessions_v2 import InvalidField
+from sentry.utils.dates import parse_stats_period
+
+METRIC_OUTCOME_AGGREGATE = "sum(c:metric_stats/volume@none)"
+METRIC_HOURS_AGGREGATE = "max(g:metric_stats/cardinality@none){cardinality.window:60}"
+METRIC_CARDINALITY_AGGREGATE = "max(g:metric_stats/cardinality@none){cardinality.window:3600}"
+
+
+def _get_mql_string(aggregate: str, group_by: Sequence[str]) -> str:
+
+    # TODO(metrics): add support for reason tag
+    group_by_tags = []
+    if "outcome" in group_by:
+        group_by_tags.append("outcome.id")
+    if "project" in group_by:
+        group_by_tags.append("project_id")
+
+    if group_by_tags:
+        return f"{aggregate} by ({', '.join(group_by_tags)})"
+
+    return aggregate
+
+
+def _run_metrics_outcomes_query(
+    start: datetime,
+    end: datetime,
+    group_by: Sequence[str],
+    interval,
+    organization: Organization,
+    projects: Sequence[Project],
+    environments: Sequence[Environment],
+):
+
+    mql_string = _get_mql_string(METRIC_OUTCOME_AGGREGATE, group_by)
+
+    rows = run_queries(
+        mql_queries=[MQLQuery(mql_string)],
+        start=start,
+        end=end,
+        interval=int(3600 if interval is None else interval.total_seconds()),
+        organization=organization,
+        projects=projects,
+        environments=environments,
+        referrer=Referrer.OUTCOMES_TIMESERIES.value,
+    ).apply_transformer(MetricsOutcomesTransformer())
+
+    return rows
+
+
+def _run_estimated_metrics_hours_query(
+    start: datetime,
+    end: datetime,
+    group_by: Sequence[str],
+    organization: Organization,
+    projects: Sequence[Project],
+    environments: Sequence[Environment],
+):
+    mql_string = _get_mql_string(METRIC_HOURS_AGGREGATE, group_by)
+
+    rows = run_queries(
+        mql_queries=[MQLQuery(mql_string)],
+        start=start,
+        end=end,
+        # metrics hours queries have to be bucketed by 1h
+        interval=int(3600),
+        organization=organization,
+        projects=projects,
+        environments=environments,
+        referrer=Referrer.OUTCOMES_TIMESERIES.value,
+    ).apply_transformer(MetricsOutcomesTransformer())
+
+    return rows
+
+
+def _run_metrics_cardinality_query(
+    start: datetime,
+    end: datetime,
+    group_by: Sequence[str],
+    interval,
+    organization: Organization,
+    projects: Sequence[Project],
+    environments: Sequence[Environment],
+):
+
+    if not interval:
+        interval = 3600
+    elif interval.total_seconds() < 3600:
+        interval = 3600
+    else:
+        interval = interval.total_seconds()
+
+    mql_string = _get_mql_string(METRIC_CARDINALITY_AGGREGATE, group_by)
+
+    rows = run_queries(
+        mql_queries=[MQLQuery(mql_string)],
+        start=start,
+        end=end,
+        interval=interval,
+        organization=organization,
+        projects=projects,
+        environments=environments,
+        referrer=Referrer.OUTCOMES_TIMESERIES.value,
+    ).apply_transformer(MetricsOutcomesTransformer())
+
+    return rows
+
+
+def run_metric_stats_query(
+    query: QueryDict,
+    organization: Organization,
+    projects: Sequence[Project],
+    environments: Sequence[Environment],
+):
+
+    start, end = get_date_range_from_params(query)
+    group_by = query.getlist("groupBy", [])
+    interval = parse_stats_period(query.get("interval", "1h"))
+
+    category = query.get("category")
+    # TODO(metrics): remove  metricsSeconds after FE is updated
+    if category == "metricSecond" or category == "metricOutcomes":
+        return _run_metrics_outcomes_query(
+            start=start,
+            end=end,
+            group_by=group_by,
+            interval=interval,
+            environments=environments,
+            organization=organization,
+            projects=projects,
+        )
+    elif category == "metricHour":
+        return _run_estimated_metrics_hours_query(
+            start=start,
+            end=end,
+            group_by=group_by,
+            environments=environments,
+            organization=organization,
+            projects=projects,
+        )
+
+    raise InvalidField(f'Invalid category: "{category}"')

src/sentry/snuba/outcomes.py

@@ -13,20 +13,19 @@
 from snuba_sdk.function import Function
 from snuba_sdk.query import Query
 
-from sentry.api.utils import get_date_range_from_params
 from sentry.constants import DataCategory
+from sentry.models.environment import Environment
+from sentry.models.organization import Organization
+from sentry.models.project import Project
 from sentry.release_health.base import AllowedResolution
 from sentry.search.utils import InvalidQuery
-from sentry.sentry_metrics.querying.data import MQLQuery, run_queries
-from sentry.sentry_metrics.querying.data.transformation.stats import MetricsStatsTransformer
-from sentry.snuba.referrer import Referrer
+from sentry.sentry_metrics.querying.stats import run_metric_stats_query
 from sentry.snuba.sessions_v2 import (
     InvalidField,
     SimpleGroupBy,
     get_constrained_date_range,
     massage_sessions_result,
 )
-from sentry.utils.dates import parse_stats_period
 from sentry.utils.outcomes import Outcome
 from sentry.utils.snuba import raw_snql_query
 
@@ -467,49 +466,21 @@ def massage_outcomes_result(
     return result
 
 
-def _get_outcomes_mql_string(query: QueryDict) -> str:
-    # metric_stats/volume counts the metric outcomes
-    aggregate = "sum(c:metric_stats/volume@none)"
-
-    # TODO(metrics): add support for reason tag
-    group_by = []
-    if "outcome" in query.getlist("groupBy", []):
-        group_by.append("outcome.id")
-    if "project" in query.getlist("groupBy", []):
-        group_by.append("project_id")
-
-    if group_by:
-        return f"{aggregate} by ({', '.join(group_by)})"
-
-    return aggregate
-
-
 def run_metrics_outcomes_query(
-    query: QueryDict, organization, projects, environments
+    query: QueryDict,
+    organization: Organization,
+    projects: Sequence[Project],
+    environments: Sequence[Environment],
 ) -> dict[str, list]:
-    start, end = get_date_range_from_params(query)
-    interval = parse_stats_period(query.get("interval", "1h"))
-
-    mql_string = _get_outcomes_mql_string(query)
-
-    rows = run_queries(
-        mql_queries=[MQLQuery(mql_string)],
-        start=start,
-        end=end,
-        interval=int(3600 if interval is None else interval.total_seconds()),
-        organization=organization,
-        projects=projects,
-        environments=environments,
-        referrer=Referrer.OUTCOMES_TIMESERIES.value,
-    ).apply_transformer(MetricsStatsTransformer())
-
-    # Dummy query definition to pass to the _format_rows, as it expects a QueryDefinition object
+    rows = run_metric_stats_query(
+        query=query, organization=organization, projects=projects, environments=environments
+    )
+    # Dummy query definition to pass to the massage_outcomes_result, as it expects a QueryDefinition object
     # TODO(metrics): add a `metrics` category or refactor _format_rows
     copied = query.copy()
     copied["category"] = "error"
     query_def = QueryDefinition.from_query_dict(copied, {"organization_id": organization.id})
 
-    series = _format_rows(rows["series"], query_def)
-    totals = _format_rows(rows["totals"], query_def)
-
-    return massage_outcomes_result(query_def, totals, series)
+    return massage_outcomes_result(
+        query=query_def, result_totals=rows["totals"], result_timeseries=rows["series"]
+    )

tests/snuba/api/endpoints/test_organization_stats_v2.py

@@ -950,13 +950,46 @@ def setUp(self):
             value=1,
         )
 
+        self.store_metric(
+            org_id=self.org.id,
+            project_id=self.project.id,
+            type="gauge",
+            name="g:metric_stats/cardinality@none",
+            timestamp=self.ts(self.now - timedelta(hours=1)),
+            use_case_id=UseCaseID.METRIC_STATS,
+            tags={"mri": "", "cardinality.window": "60"},
+            value=1,
+        )
+
+        self.store_metric(
+            org_id=self.org.id,
+            project_id=self.project2.id,
+            type="gauge",
+            name="g:metric_stats/cardinality@none",
+            timestamp=self.ts(self.now - timedelta(hours=1)),
+            use_case_id=UseCaseID.METRIC_STATS,
+            tags={"mri": "", "cardinality.window": "60"},
+            value=2,
+        )
+
+        self.store_metric(
+            org_id=self.org.id,
+            project_id=self.project2.id,
+            type="gauge",
+            name="g:metric_stats/cardinality@none",
+            timestamp=self.ts(self.now - timedelta(hours=1)),
+            use_case_id=UseCaseID.METRIC_STATS,
+            tags={"mri": "", "cardinality.window": "60"},
+            value=3,
+        )
+
     @freeze_time("2021-03-14T12:27:28.303Z")
     @with_feature("organizations:custom-metrics")
     def test_metrics_category(self):
         response = self.do_request(
             {
                 "project": [-1],
-                "category": ["metrics"],
+                "category": ["metricOutcomes"],
                 "statsPeriod": "1d",
                 "interval": "1d",
                 "field": ["sum(quantity)"],
@@ -979,7 +1012,7 @@ def test_metrics_group_by_project(self):
         response = self.do_request(
             {
                 "project": [-1],
-                "category": ["metrics"],
+                "category": ["metricOutcomes"],
                 "groupBy": ["project"],
                 "statsPeriod": "1d",
                 "interval": "1d",
@@ -1012,7 +1045,7 @@ def test_metrics_multiple_group_by(self):
         response = self.do_request(
             {
                 "project": [-1],
-                "category": ["metrics"],
+                "category": ["metricOutcomes"],
                 "groupBy": ["project", "outcome"],
                 "statsPeriod": "1d",
                 "interval": "1d",
@@ -1043,3 +1076,36 @@ def test_metrics_multiple_group_by(self):
             "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
             "start": "2021-03-13T00:00:00Z",
         }
+
+    @freeze_time("2021-03-14T12:27:28.303Z")
+    @with_feature("organizations:custom-metrics")
+    def test_metric_hour(self):
+        response = self.do_request(
+            {
+                "project": [-1],
+                "category": ["metricHour"],
+                "groupBy": ["project"],
+                "statsPeriod": "1h",
+                "interval": "1h",
+                "field": ["sum(quantity)"],
+            },
+            status_code=200,
+        )
+
+        assert result_sorted(response.data) == {
+            "end": "2021-03-14T13:00:00Z",
+            "groups": [
+                {
+                    "by": {"project": self.project.id},
+                    "series": {"sum(quantity)": [1, 0]},
+                    "totals": {"sum(quantity)": 1},
+                },
+                {
+                    "by": {"project": self.project2.id},
+                    "series": {"sum(quantity)": [3, 0]},
+                    "totals": {"sum(quantity)": 3},
+                },
+            ],
+            "intervals": ["2021-03-14T11:00:00Z", "2021-03-14T12:00:00Z"],
+            "start": "2021-03-14T11:00:00Z",
+        }