fix: CircleHitbox should properly detect when ray is outside (#3100)

Previously some rays that originated outside of the `CircleHitbox` were
counted as inside, this solves that and adds an example of how to use
the `isInsideHitbox` functionality (thanks to @wurzelsand).


Closes #3063

examples/lib/stories/collision_detection/collision_detection.dart

@@ -9,6 +9,7 @@ import 'package:examples/stories/collision_detection/quadtree_example.dart';
 import 'package:examples/stories/collision_detection/raycast_example.dart';
 import 'package:examples/stories/collision_detection/raycast_light_example.dart';
 import 'package:examples/stories/collision_detection/raycast_max_distance_example.dart';
+import 'package:examples/stories/collision_detection/rays_in_shape_example.dart';
 import 'package:examples/stories/collision_detection/raytrace_example.dart';
 import 'package:flame/game.dart';
 import 'package:flutter/widgets.dart';
@@ -76,5 +77,11 @@ void addCollisionDetectionStories(Dashbook dashbook) {
       codeLink:
           baseLink('collision_detection/raycast_max_distance_example.dart'),
       info: RaycastMaxDistanceExample.description,
+    )
+    ..add(
+      'Ray inside/outside shapes',
+      (_) => GameWidget(game: RaysInShapeExample()),
+      codeLink: baseLink('collision_detection/rays_in_shape_example.dart'),
+      info: RaysInShapeExample.description,
     );
 }

examples/lib/stories/collision_detection/rays_in_shape_example.dart

@@ -0,0 +1,158 @@
+import 'dart:async';
+import 'dart:math';
+
+import 'package:flame/collisions.dart';
+import 'package:flame/components.dart';
+import 'package:flame/events.dart';
+import 'package:flame/extensions.dart';
+import 'package:flame/game.dart';
+import 'package:flame/geometry.dart';
+import 'package:flutter/material.dart';
+
+const playArea = Rect.fromLTRB(-100, -100, 100, 100);
+
+class RaysInShapeExample extends FlameGame {
+  static const description = '''
+In this example we showcase the raytrace functionality where you can see whether
+the rays are inside the shapes or not. Click to change the shape that the rays
+are casted against. The rays originates from small circles, and if the circle is
+inside the shape it will be red, otherwise green. And if the ray doesn't hit any
+shape it will be gray.
+''';
+
+  RaysInShapeExample()
+      : super(
+          world: RaysInShapeWorld(),
+          camera: CameraComponent.withFixedResolution(
+            width: playArea.width,
+            height: playArea.height,
+          ),
+        );
+}
+
+final whiteStroke = Paint()
+  ..color = const Color(0xffffffff)
+  ..style = PaintingStyle.stroke;
+
+final lightStroke = Paint()
+  ..color = const Color(0x50ffffff)
+  ..style = PaintingStyle.stroke;
+
+final greenStroke = Paint()
+  ..color = const Color(0xff00ff00)
+  ..style = PaintingStyle.stroke;
+
+final redStroke = Paint()
+  ..color = const Color(0xffff0000)
+  ..style = PaintingStyle.stroke;
+
+class RaysInShapeWorld extends World
+    with
+        HasGameReference<RaysInShapeExample>,
+        HasCollisionDetection,
+        TapCallbacks {
+  final _rng = Random();
+  List<Ray2> _rays = [];
+
+  List<Ray2> randomRays(int count) => List<Ray2>.generate(
+        count,
+        (index) => Ray2(
+          origin: (Vector2.random(_rng)) * playArea.size.width -
+              playArea.size.toVector2() / 2,
+          direction: (Vector2.random(_rng) - Vector2(0.5, 0.5)).normalized(),
+        ),
+      );
+
+  int _componentIndex = 0;
+
+  final _components = [
+    CircleComponent(
+      radius: 60,
+      anchor: Anchor.center,
+      position: Vector2.zero(),
+      paint: whiteStroke,
+      children: [CircleHitbox()],
+    ),
+    RectangleComponent(
+      size: Vector2(100, 100),
+      anchor: Anchor.center,
+      position: Vector2.zero(),
+      paint: whiteStroke,
+      children: [RectangleHitbox()],
+    ),
+    PositionComponent(
+      position: Vector2.zero(),
+      children: [
+        PolygonHitbox.relative(
+          [
+            Vector2(-0.7, -1),
+            Vector2(1, -0.4),
+            Vector2(0.3, 1),
+            Vector2(-1, 0.6),
+          ],
+          parentSize: Vector2(100, 100),
+          anchor: Anchor.center,
+          position: Vector2.zero(),
+        )
+          ..paint = whiteStroke
+          ..renderShape = true,
+      ],
+    ),
+  ];
+
+  @override
+  FutureOr<void> onLoad() {
+    super.onLoad();
+    add(_components[_componentIndex]);
+    _rays = randomRays(200);
+  }
+
+  @override
+  void onTapUp(TapUpEvent event) {
+    super.onTapUp(event);
+    remove(_components[_componentIndex]);
+    _componentIndex = (_componentIndex + 1) % _components.length;
+    add(_components[_componentIndex]);
+    _recording.clear();
+    _rays = randomRays(200);
+  }
+
+  final Map<Ray2, RaycastResult<ShapeHitbox>?> _recording = {};
+
+  @override
+  void update(double dt) {
+    super.update(dt);
+
+    for (final ray in _rays) {
+      final result = collisionDetection.raycast(ray);
+      _recording.addAll({ray: result});
+    }
+  }
+
+  @override
+  void render(Canvas canvas) {
+    super.render(canvas);
+    for (final ray in _recording.keys) {
+      final result = _recording[ray];
+      if (result == null) {
+        canvas.drawLine(
+          ray.origin.toOffset(),
+          (ray.origin + ray.direction.scaled(10)).toOffset(),
+          lightStroke,
+        );
+        canvas.drawCircle(ray.origin.toOffset(), 1, lightStroke);
+      } else {
+        canvas.drawLine(
+          ray.origin.toOffset(),
+          result.intersectionPoint!.toOffset(),
+          lightStroke,
+        );
+        canvas.drawCircle(
+          ray.origin.toOffset(),
+          1,
+          result.isInsideHitbox ? redStroke : greenStroke,
+        );
+      }
+    }
+  }
+}

packages/flame/lib/src/collisions/hitboxes/circle_hitbox.dart

@@ -73,7 +73,8 @@ class CircleHitbox extends CircleComponent with ShapeHitbox {
         ..y *= (ray.direction.y.sign * _temporaryLineSegment.to.y.sign);
     }
 
-    if (_temporaryLineSegment.to.length2 < radius * radius) {
+    if (ray.origin.distanceToSquared(_temporaryAbsoluteCenter) <
+        radius * radius) {
       _temporaryLineSegment.to.scaleTo(2 * radius);
       isInsideHitbox = true;
     }

packages/flame/test/collisions/collision_detection_test.dart

@@ -1459,6 +1459,25 @@ void main() {
             closeToVector(Vector2(0, 1)),
           );
         },
+        'ray from slightly outside of the CircleHitbox should not be counted '
+            'as inside': (collisionSystem) async {
+          final game = collisionSystem as FlameGame;
+          final world = game.world;
+          final positionComponent = PositionComponent(
+            position: Vector2.zero(),
+            anchor: Anchor.center,
+            size: Vector2.all(120),
+          )..add(CircleHitbox());
+          await world.ensureAdd(positionComponent);
+          await game.ready();
+          final ray = Ray2(
+            origin: Vector2(-38.06044293218409, -48.5986651724067),
+            direction: Vector2(0.927474693393028, -0.3738859359691247),
+          );
+          final result = collisionSystem.collisionDetection.raycast(ray);
+          expect(result?.hitbox?.parent, positionComponent);
+          expect(result?.isInsideHitbox, isFalse);
+        },
       });
     });