checkpoint

pkgs/sdk_triage_bot/lib/src/common.dart

@@ -37,9 +37,9 @@ String get geminiKey {
   return token;
 }
 
-/// Don't return more than 5k of text for an issue body.
+/// Don't return more than 10k of text for an issue body.
 String trimmedBody(String body) {
-  const textLimit = 5 * 1024;
+  const textLimit = 10 * 1024;
 
   return body.length > textLimit ? body = body.substring(0, textLimit) : body;
 }

pkgs/sdk_triage_bot/lib/src/gemini.dart

@@ -44,7 +44,7 @@ class GeminiService {
   /// On failures, this will throw a `GenerativeAIException`.
   Future<List<String>> classify(String prompt) async {
     final result = await _query(_classifyModel, prompt);
-    final labels = result.split(',').map((l) => l.trim()).toList();
+    final labels = result.split(',').map((l) => l.trim()).toList()..sort();
     return labels;
   }
 

pkgs/sdk_triage_bot/lib/src/prompts.dart

@@ -2,6 +2,8 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// TODO(devoncarew): Add additional prompt instructions for `area-pkg` issues.
+
 String assignAreaPrompt({
   required String title,
   required String body,
@@ -45,6 +47,10 @@ If the issue is clearly a bug report, then also apply the label 'type-bug'.
 If the issue is mostly a question,  then also apply the label 'type-question'.
 Otherwise don't apply a 'type-' label.
 
+If the issue title starts with "[breaking change]" then apply the
+`breaking-change-request` label but don't assign an area label. IMPORTANT: only
+do this if the issue title starts with "[breaking change]".
+
 If the issue was largely unchanged from our default issue template, then apply the
 'needs-info' label and don't assign an area label. These issues will generally
 have a title of "Create an issue" and the body will start with
@@ -54,10 +60,6 @@ If the issue title is "Analyzer Feedback from IntelliJ", these are generally not
 well qualified. For these issues, apply the 'needs-info' label but don't assign
 an area label.
 
-If the issue title starts with "[breaking change] " then it doesn't need to be
-triaged into a specific area; apply the `breaking-change-request` label but
-don't assign an area label.
-
 Return the labels as comma separated text.
 
 Here are a series of few-shot examples:
@@ -89,6 +91,61 @@ body: ## Version information
 OUTPUT: needs-info
 </EXAMPLE>
 
+<EXAMPLE>
+INPUT: title: Support likely() and unlikely() hints for AOT code optimization
+
+body: ```dart
+// Tell the compiler which branches are going to be taken most of the time.
+
+if (unlikely(n == 0)) {
+  // This branch is known to be taken rarely.
+} else {
+  // This branch is expected to be in the hot path.
+}
+
+final result = likely(s == null) ? commonPath() : notTakenOften();
+```
+
+Please add support for the `likely()` and `unlikely()` optimization hints within branching conditions. The AOT compiler can use these hints to generate faster code in a hot path that contains multiple branches.
+
+OUTPUT: area-vm, type-enhancement, type-performance
+</EXAMPLE>
+
+<EXAMPLE>
+INPUT: title: Analyzer doesn't notice incorrect return type of generic method
+
+body: dart analyze gives no errors on the follow code:
+
+```dart
+void main() {
+  method(getB());
+}
+
+void method(String b) => print(b);
+
+B getB<B extends A>() {
+  return A() as B;
+}
+
+class A {}
+```
+I would have suspected it to say something along the line of **The argument type 'A' can't be assigned to the parameter type 'String'.**
+
+OUTPUT: area-analyzer, type-enhancement
+</EXAMPLE>
+
+<EXAMPLE>
+INPUT: title: DDC async function stepping improvements
+
+body: Tracking issue to monitor progress on improving debugger stepping through async function bodies.
+
+The new DDC async semantics expand async function bodies into complex state machines. The normal JS stepping semantics don't map cleanly to steps through Dart code given this lowering. There are a couple potential approaches to fix this:
+1) Add more logic to the Dart debugger to perform custom stepping behavior when stepping through async code.
+2) Modify the async lowering in such a way that stepping more closely resembles stepping through Dart. For example, rather than returning multiple times, the state machine function might be able to yield. Stepping over a yield might allow the debugger to stay within the function body.
+
+OUTPUT: area-web
+</EXAMPLE>
+
 The issue to triage follows:
 
 title: $title
@@ -106,20 +163,18 @@ String summarizeIssuePrompt({
 }) {
   const needsMoreInfo = '''
 Our classification model determined that we'll need more information to triage
-this issue. Please gently prompt the user to provide more information.
+this issue. Thank them for their contribution and gently prompt them to provide
+more information.
 ''';
 
-  final needsInfoVerbiage = needsInfo ? needsMoreInfo : '';
-  final responseLimit = needsInfo
-      ? '2-3 sentences, 50 words or less'
-      : '1-2 sentences, 24 words or less';
+  final responseLimit = needsInfo ? '' : ' (1-2 sentences, 24 words or less)';
 
   return '''
-You are a software engineer on the Dart team at Google. You are responsible for
-triaging incoming issues from users. For each issue, briefly summarize the issue
-($responseLimit).
+You are a software engineer on the Dart team at Google.
+You are responsible for triaging incoming issues from users.
+For each issue, briefly summarize the issue $responseLimit.
 
-$needsInfoVerbiage
+${needsInfo ? needsMoreInfo : ''}
 
 The issue to triage follows:
 

pkgs/sdk_triage_bot/lib/triage.dart

@@ -131,9 +131,9 @@ ${trimmedBody(comment.body ?? '')}
   // create github comment
   await githubService.createComment(sdkSlug, issueNumber, comment);
 
-  final allRepoLabels = (await githubService.getAllLabels(sdkSlug)).toSet();
-  final labelAdditions = newLabels.toSet().intersection(allRepoLabels).toList()
-    ..sort();
+  final allRepoLabels = await githubService.getAllLabels(sdkSlug);
+  final labelAdditions =
+      filterLegalLabels(newLabels, allRepoLabels: allRepoLabels);
   if (labelAdditions.isNotEmpty) {
     labelAdditions.add('triage-automation');
   }
@@ -149,7 +149,9 @@ ${trimmedBody(comment.body ?? '')}
   logger.log('Triaged ${issue.htmlUrl}');
 }
 
-List<String> filterExistingLabels(
-    List<String> allLabels, List<String> newLabels) {
-  return newLabels.toSet().intersection(allLabels.toSet()).toList();
+List<String> filterLegalLabels(
+  List<String> labels, {
+  required List<String> allRepoLabels,
+}) {
+  return labels.toSet().intersection(allRepoLabels.toSet()).toList()..sort();
 }

pkgs/sdk_triage_bot/tool/bench.dart

@@ -51,7 +51,7 @@ void main(List<String> args) async {
         await githubService.fetchIssue(sdkSlug, expectation.issueNumber);
     final bodyTrimmed = trimmedBody(issue.body);
 
-    print('#${issue.number}');
+    print('#${issue.number}: ${expectation.expectedLabels.join(', ')}');
 
     try {
       final labels = await geminiService.classify(
@@ -60,12 +60,7 @@ void main(List<String> args) async {
       if (expectation.satisfiedBy(labels)) {
         predicted++;
       } else {
-        var title = issue.title.length > 100
-            ? '${issue.title.substring(0, 100)}...'
-            : issue.title;
-        print('  "$title"');
-        print('     labeled: ${expectation.expectedLabels.join(', ')}');
-        print('  prediction: ${labels.join(', ')}');
+        stderr.writeln('   bot: ${labels.join(', ')}');
       }
     } on GenerativeAIException catch (e) {
       // Failures here can include things like gemini safety issues, ...
@@ -114,8 +109,22 @@ class ClassificationResults {
   }
 
   bool satisfiedBy(List<String> labels) {
-    final filtered = labels.where((l) => !l.startsWith('type-')).toSet();
-    final expected = expectedLabels.where((l) => !l.startsWith('type-'));
-    return expected.every(filtered.contains);
+    // handle needs-info
+    if (expectedLabels.contains('needs-info')) {
+      return labels.contains('needs-info');
+    }
+
+    // handle breaking-change-request
+    if (expectedLabels.contains('breaking-change-request')) {
+      return labels.contains('breaking-change-request');
+    }
+
+    for (final label in expectedLabels.where((l) => l.startsWith('area-'))) {
+      if (!labels.contains(label)) {
+        return false;
+      }
+    }
+
+    return true;
   }
 }

pkgs/sdk_triage_bot/tool/bench.md

@@ -20,7 +20,6 @@ materially improve the classification performance.
 | #56354 | `area-web`, `type-bug`                          |
 | #56353 | `area-dart2wasm`                                |
 | #56350 | `area-analyzer`, `type-enhancement`             |
-| #56348 | `area-intellij`                                 |
 | #56347 | `area-dart-cli`, `type-bug`                     |
 | #56346 | `area-pkg`, `pkg-json`, `type-enhancement`      |
 | #56345 | `area-analyzer`, `type-enhancement`             |
@@ -44,7 +43,7 @@ materially improve the classification performance.
 | #56316 | `area-web`                                      |
 | #56315 | `area-web`                                      |
 | #56314 | `area-web`, `type-bug`                          |
-| #56308 | `area-vm`                                       |
+| #56308 | `area-vm` `breaking-change-request`             |
 | #56306 | `area-vm`, `type-bug`                           |
 | #56305 | `area-front-end`, `type-bug`, `type-question`   |
 | #56304 | `area-core-library`, `type-enhancement`         |
@@ -55,13 +54,12 @@ materially improve the classification performance.
 | #56283 | `area-dart2wasm`                                |
 | #56256 | `area-front-end`, `type-bug`                    |
 | #56254 | `area-pkg`, `pkg-vm-service`, `type-bug`        |
-| #56246 | `area-intellij`                                 |
-| #56240 | `area-intellij`                                 |
+| #56240 | `needs-info`                                    |
 | #56229 | `area-infrastructure`                           |
 | #56227 | `area-native-interop`                           |
 | #56220 | `area-infrastructure`, `type-code-health`       |
 | #56217 | `area-meta`                                     |
-| #56216 | `area-intellij`                                 |
+| #56216 | `needs-info`                                    |
 | #56214 | `area-native-interop`                           |
 | #56208 | `area-google3`, `type-enhancement`              |
 | #56207 | `area-google3`                                  |
@@ -108,3 +106,4 @@ We need more information from the user before we can triage these issues.
 
 ## Results
 2024-08-27: 55.6% using gemini-1.5-flash-latest
+2024-08-30: 64.8% using gemini-1.5-flash-latest