From 892d829cbe78ab46cca5d34b90f480d187fb5599 Mon Sep 17 00:00:00 2001
From: Paul D'Ambra <paul@posthog.com>
Date: Thu, 22 Jun 2023 12:41:40 +0100
Subject: [PATCH] chore: log message too large error (#16182)

* chore: log message too large error

* send data to loki not sentry, they rhyme
---
 .run/PostHog.run.xml   |  2 +-
 posthog/api/capture.py | 28 +++++++++++-----------------
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/.run/PostHog.run.xml b/.run/PostHog.run.xml
index d637d92aabe..ed075e33cff 100644
--- a/.run/PostHog.run.xml
+++ b/.run/PostHog.run.xml
@@ -47,4 +47,4 @@
     <option name="customRunCommand" value="" />
     <method v="2" />
   </configuration>
-</component>
\ No newline at end of file
+</component>
diff --git a/posthog/api/capture.py b/posthog/api/capture.py
index f19ef5d5115..4ab681b04be 100644
--- a/posthog/api/capture.py
+++ b/posthog/api/capture.py
@@ -12,10 +12,9 @@ from django.conf import settings
 from django.http import JsonResponse
 from django.utils import timezone
 from django.views.decorators.csrf import csrf_exempt
-from kafka.errors import KafkaError
+from kafka.errors import KafkaError, MessageSizeTooLargeError
 from kafka.producer.future import FutureRecordMetadata
-from prometheus_client import Counter, Histogram
-from prometheus_client.utils import INF
+from prometheus_client import Counter
 from rest_framework import status
 from sentry_sdk import configure_scope
 from sentry_sdk.api import capture_exception, start_span
@@ -88,19 +87,6 @@ TOKEN_SHAPE_INVALID_COUNTER = Counter(
     labelnames=["reason"],
 )
 
-REPLAY_INGESTION_COUNTER = Counter(
-    "capture_replay_ingestion_total",
-    "Events processed for replay ingestion.",
-    labelnames=["method"],
-)
-
-REPLAY_INGESTION_BATCH_COMPRESSION_RATIO_HISTOGRAM = Histogram(
-    "session_recordings_chunks_length",
-    "We chunk session recordings to fit them into kafka, how often do we chunk and by how much?",
-    buckets=(0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.1, 2, 5, 10, INF),
-    labelnames=["method"],
-)
-
 # This is a heuristic of ids we have seen used as anonymous. As they frequently
 # have significantly more traffic than non-anonymous distinct_ids, and likely
 # don't refer to the same underlying person we prefer to partition them randomly
@@ -440,7 +426,15 @@ def get_event(request):
                 # errors, and set Retry-After header accordingly.
                 # TODO: return 400 error for non-retriable errors that require the
                 # client to change their request.
-                logger.error("kafka_produce_failure", exc_info=exc)
+
+                logger.error(
+                    "kafka_produce_failure",
+                    exc_info=exc,
+                    name=exc.__class__.__name__,
+                    # data could be large, so we don't always want to include it,
+                    # but we do want to include it for some errors to aid debugging
+                    data=data if isinstance(exc, MessageSizeTooLargeError) else None,
+                )
                 return cors_response(
                     request,
                     generate_exception_response(