Sentry SDK Crash Detection POC

fixtures/sdk_crash_detection/crash_event.py

@@ -0,0 +1,269 @@
+from typing import Any, Collection, Dict, Mapping, Sequence
+
+IN_APP_FRAME = {
+    "function": "LoginViewController.viewDidAppear",
+    "raw_function": "LoginViewController.viewDidAppear(Bool)",
+    "symbol": "$s8Sentry9LoginViewControllerC13viewDidAppearyySbF",
+    "package": "SentryApp",
+    "filename": "LoginViewController.swift",
+    "abs_path": "/Users/sentry/git/iOS/Sentry/LoggedOut/LoginViewController.swift",
+    "lineno": 196,
+    "in_app": True,
+    "image_addr": "0x100260000",
+    "instruction_addr": "0x102b16630",
+    "symbol_addr": "0x100260000",
+}
+
+
+def get_sentry_frame(function: str, in_app: bool = False) -> Mapping[str, Any]:
+    return {
+        "function": function,
+        "package": "Sentry",
+        "in_app": in_app,
+        "image_addr": "0x100304000",
+    }
+
+
+def get_frames(function: str, sentry_frame_in_app: bool = False) -> Sequence[Mapping[str, Any]]:
+    frames = [
+        get_sentry_frame(function, sentry_frame_in_app),
+        {
+            "function": "LoginViewController.viewDidAppear",
+            "symbol": "$s8Sentry9LoginViewControllerC13viewDidAppearyySbF",
+            "package": "SentryApp",
+            "in_app": True,
+            "filename": "LoginViewController.swift",
+            "image_addr": "0x100260000",
+        },
+        IN_APP_FRAME,
+        {
+            "function": "-[UIViewController _setViewAppearState:isAnimating:]",
+            "symbol": "-[UIViewController _setViewAppearState:isAnimating:]",
+            "package": "UIKitCore",
+            "in_app": False,
+            "image_addr": "0x1a4e8f000",
+        },
+        {
+            "function": "-[UIViewController __viewDidAppear:]",
+            "symbol": "-[UIViewController __viewDidAppear:]",
+            "package": "UIKitCore",
+            "in_app": False,
+            "image_addr": "0x1a4e8f000",
+        },
+        {
+            "function": "-[UIViewController _endAppearanceTransition:]",
+            "symbol": "-[UIViewController _endAppearanceTransition:]",
+            "package": "UIKitCore",
+            "in_app": False,
+            "image_addr": "0x1a4e8f000",
+        },
+        {
+            "function": "-[UINavigationController navigationTransitionView:didEndTransition:fromView:toView:]",
+            "symbol": "-[UINavigationController navigationTransitionView:didEndTransition:fromView:toView:]",
+            "package": "UIKitCore",
+            "in_app": False,
+            "image_addr": "0x1a4e8f000",
+        },
+        {
+            "function": "__49-[UINavigationController _startCustomTransition:]_block_invoke",
+            "symbol": "__49-[UINavigationController _startCustomTransition:]_block_invoke",
+            "package": "UIKitCore",
+            "in_app": False,
+            "image_addr": "0x1a4e8f000",
+        },
+    ]
+
+    # The frames have to be ordered from caller to callee, or oldest to youngest.
+    # The last frame is the one creating the exception.
+    # As we usually look at stacktraces from youngest to oldest, we reverse the order.
+    return frames[::-1]
+
+
+def get_crash_event(handled=False, function="-[Sentry]", **kwargs) -> Dict[str, Collection[str]]:
+    return get_crash_event_with_frames(get_frames(function), handled=handled, **kwargs)
+
+
+def get_crash_event_with_frames(
+    frames: Sequence[Mapping[str, Any]], handled=False, **kwargs
+) -> Dict[str, Collection[str]]:
+    result = {
+        "event_id": "80e3496eff734ab0ac993167aaa0d1cd",
+        "release": "5.222.5",
+        "type": "error",
+        "level": "fatal",
+        "platform": "cocoa",
+        "tags": {"level": "fatal"},
+        "exception": {
+            "values": [
+                {
+                    "stacktrace": {
+                        "frames": frames,
+                    },
+                    "type": "SIGABRT",
+                    "mechanism": {"handled": handled},
+                }
+            ]
+        },
+        "breadcrumbs": {
+            "values": [
+                {
+                    "timestamp": 1675900265.0,
+                    "type": "debug",
+                    "category": "started",
+                    "level": "info",
+                    "message": "Breadcrumb Tracking",
+                },
+            ]
+        },
+        "contexts": {
+            "app": {
+                "app_start_time": "2023-02-08T23:51:05Z",
+                "device_app_hash": "8854fe9e3d4e4a66493baee798bfae0228efabf1",
+                "build_type": "app store",
+                "app_identifier": "com.some.company.io",
+                "app_name": "SomeCompany",
+                "app_version": "5.222.5",
+                "app_build": "21036",
+                "app_id": "397D4F75-6C01-32D1-BF46-62098979E470",
+                "type": "app",
+            },
+            "device": {
+                "family": "iOS",
+                "model": "iPhone14,8",
+                "model_id": "D28AP",
+                "arch": "arm64e",
+                "memory_size": 5944508416,
+                "free_memory": 102154240,
+                "usable_memory": 4125687808,
+                "storage_size": 127854202880,
+                "boot_time": "2023-02-01T05:21:23Z",
+                "timezone": "PST",
+                "type": "device",
+            },
+            "os": {
+                "name": "iOS",
+                "version": "16.3",
+                "build": "20D47",
+                "kernel_version": "Darwin Kernel Version 22.3.0: Wed Jan  4 21:25:19 PST 2023; root:xnu-8792.82.2~1/RELEASE_ARM64_T8110",
+                "rooted": False,
+                "type": "os",
+            },
+        },
+        "debug_meta": {
+            "images": [
+                {
+                    "code_file": "/private/var/containers/Bundle/Application/895DA2DE-5FE3-44A0-8C0F-900519EA5516/iOS-Swift.app/iOS-Swift",
+                    "debug_id": "aa8a3697-c88a-36f9-a687-3d3596568c8d",
+                    "arch": "arm64",
+                    "image_addr": "0x100260000",
+                    "image_size": 180224,
+                    "image_vmaddr": "0x100000000",
+                    "type": "macho",
+                },
+                {
+                    "code_file": "/private/var/containers/Bundle/Application/9EB557CD-D653-4F51-BFCE-AECE691D4347/iOS-Swift.app/Frameworks/Sentry.framework/Sentry",
+                    "debug_id": "e2623c4d-79c5-3cdf-90ab-2cf44e026bdd",
+                    "arch": "arm64",
+                    "image_addr": "0x100304000",
+                    "image_size": 802816,
+                    "type": "macho",
+                },
+                {
+                    "code_file": "/System/Library/PrivateFrameworks/UIKitCore.framework/UIKitCore",
+                    "debug_id": "b0858d8e-7220-37bf-873f-ecc2b0a358c3",
+                    "arch": "arm64e",
+                    "image_addr": "0x1a4e8f000",
+                    "image_size": 25309184,
+                    "image_vmaddr": "0x188ff7000",
+                    "type": "macho",
+                },
+                {
+                    "code_file": "/System/Library/Frameworks/CFNetwork.framework/CFNetwork",
+                    "debug_id": "b2273be9-538a-3f56-b9c7-801f39550f58",
+                    "arch": "arm64e",
+                    "image_addr": "0x1a3e32000",
+                    "image_size": 3977216,
+                    "image_vmaddr": "0x187f9a000",
+                    "in_app": False,
+                    "type": "macho",
+                },
+            ]
+        },
+        "environment": "test-app",
+        "sdk": {
+            "name": "sentry.cocoa",
+            "version": "8.1.0",
+            "integrations": [
+                "Crash",
+                "PerformanceTracking",
+                "MetricKit",
+                "WatchdogTerminationTracking",
+                "ViewHierarchy",
+                "NetworkTracking",
+                "ANRTracking",
+                "AutoBreadcrumbTracking",
+                "FramesTracking",
+                "AppStartTracking",
+                "Screenshot",
+                "FileIOTracking",
+                "UIEventTracking",
+                "AutoSessionTracking",
+                "CoreDataTracking",
+                "PreWarmedAppStartTracing",
+            ],
+        },
+        "threads": {
+            "values": [
+                {
+                    "id": 0,
+                    "stacktrace": {
+                        "frames": [
+                            {
+                                "function": "<redacted>",
+                                "in_app": False,
+                                "data": {"symbolicator_status": "unknown_image"},
+                                "image_addr": "0x0",
+                                "instruction_addr": "0x1129be52e",
+                                "symbol_addr": "0x0",
+                            },
+                            {
+                                "function": "<redacted>",
+                                "in_app": False,
+                                "data": {"symbolicator_status": "unknown_image"},
+                                "image_addr": "0x0",
+                                "instruction_addr": "0x104405f21",
+                                "symbol_addr": "0x0",
+                            },
+                        ],
+                    },
+                    "raw_stacktrace": {
+                        "frames": [
+                            {
+                                "function": "<redacted>",
+                                "in_app": False,
+                                "image_addr": "0x0",
+                                "instruction_addr": "0x1129be52e",
+                                "symbol_addr": "0x0",
+                            },
+                            {
+                                "function": "<redacted>",
+                                "in_app": False,
+                                "image_addr": "0x0",
+                                "instruction_addr": "0x104405f21",
+                                "symbol_addr": "0x0",
+                            },
+                        ],
+                    },
+                    "crashed": True,
+                }
+            ]
+        },
+        "user": {
+            "id": "803F5C87-0F8B-41C7-8499-27BD71A92738",
+            "ip_address": "192.168.0.1",
+            "geo": {"country_code": "US", "region": "United States"},
+        },
+        "logger": "my.logger.name",
+    }
+    result.update(kwargs)
+    return result

src/sentry/conf/server.py

@@ -1568,6 +1568,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "organizations:slack-escape-messages": False,
     # If true, allow to create/use org auth tokens
     "organizations:org-auth-tokens": False,
+    # Enable detecting SDK crashes during event processing
+    "organizations:sdk-crash-detection": False,
     # Enables commenting on PRs from the Sentry comment bot.
     "organizations:pr-comment-bot": False,
     # Adds additional filters and a new section to issue alert rules.
@@ -3446,3 +3448,6 @@ class TopicDefinition(TypedDict):
 # The Redis cluster to use for monitoring the health of
 # Celery queues.
 SENTRY_QUEUE_MONITORING_REDIS_CLUSTER = "default"
+
+# The project ID for SDK Crash Monitoring to save the detected SDK crashed to.
+SDK_CRASH_DETECTION_PROJECT_ID: Optional[int] = None

src/sentry/features/__init__.py

@@ -179,6 +179,7 @@
 default_manager.add("organizations:set-grouping-config", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:slack-escape-messages", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:slack-overage-notifications", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
+default_manager.add("organizations:sdk-crash-detection", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:starfish-view", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:starfish-test-endpoint", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:streamline-targeting-context", OrganizationFeature, FeatureHandlerStrategy.REMOTE)

src/sentry/tasks/post_process.py

@@ -1056,6 +1056,30 @@ def fire_error_processed(job: PostProcessJob):
         )
 
 
+def sdk_crash_monitoring(job: PostProcessJob):
+    from sentry.utils.sdk_crashes.sdk_crash_detection import sdk_crash_detection
+
+    if job["is_reprocessed"]:
+        return
+
+    event = job["event"]
+
+    if not features.has("organizations:sdk-crash-detection", event.project.organization):
+        return
+
+    if settings.SDK_CRASH_DETECTION_PROJECT_ID is None:
+        logger.warning(
+            "SDK crash detection is enabled but SDK_CRASH_DETECTION_PROJECT_ID is not set."
+        )
+        return None
+
+    with metrics.timer("post_process.sdk_crash_monitoring.duration"):
+        with sentry_sdk.start_span(op="tasks.post_process_group.sdk_crash_monitoring"):
+            sdk_crash_detection.detect_sdk_crash(
+                event=event, event_project_id=settings.SDK_CRASH_DETECTION_PROJECT_ID
+            )
+
+
 def plugin_post_process_group(plugin_slug, event, **kwargs):
     """
     Fires post processing hooks for a group.
@@ -1091,6 +1115,7 @@ def plugin_post_process_group(plugin_slug, event, **kwargs):
         process_similarity,
         update_existing_attachments,
         fire_error_processed,
+        sdk_crash_monitoring,
     ],
     GroupCategory.PERFORMANCE: [
         process_snoozes,

src/sentry/utils/sdk_crashes/cocoa_sdk_crash_detector.py

@@ -0,0 +1,45 @@
+from typing import Any, Mapping, Sequence
+
+from sentry.utils.glob import glob_match
+from sentry.utils.sdk_crashes.sdk_crash_detector import SDKCrashDetector
+
+
+class CocoaSDKCrashDetector(SDKCrashDetector):
+    def is_sdk_crash(self, frames: Sequence[Mapping[str, Any]]) -> bool:
+        if not frames:
+            return False
+
+        # The frames are ordered from caller to callee, or oldest to youngest.
+        # The last frame is the one creating the exception.
+        # Therefore, we must iterate in reverse order.
+        for frame in reversed(frames):
+            if self.is_sdk_frame(frame):
+                return True
+
+            if frame.get("in_app") is True:
+                return False
+
+        return False
+
+    def is_sdk_frame(self, frame: Mapping[str, Any]) -> bool:
+        function = frame.get("function")
+
+        if function is not None:
+            # [SentrySDK crash] is a testing function causing a crash.
+            # Therefore, we don't want to mark it a as a SDK crash.
+            if "SentrySDK crash" in function:
+                return False
+
+            function_matchers = ["*sentrycrash*", "**[[]Sentry*"]
+            for matcher in function_matchers:
+                if glob_match(frame.get("function"), matcher, ignorecase=True):
+                    return True
+
+        filename = frame.get("filename")
+        if filename is not None:
+            filenameMatchers = ["Sentry**"]
+            for matcher in filenameMatchers:
+                if glob_match(filename, matcher, ignorecase=True):
+                    return True
+
+        return False