diff --git a/pkgs/sdk_triage_bot/lib/src/common.dart b/pkgs/sdk_triage_bot/lib/src/common.dart index e9b34c1a..5cc83e46 100644 --- a/pkgs/sdk_triage_bot/lib/src/common.dart +++ b/pkgs/sdk_triage_bot/lib/src/common.dart @@ -37,9 +37,9 @@ String get geminiKey { return token; } -/// Don't return more than 5k of text for an issue body. +/// Don't return more than 10k of text for an issue body. String trimmedBody(String body) { - const textLimit = 5 * 1024; + const textLimit = 10 * 1024; return body.length > textLimit ? body = body.substring(0, textLimit) : body; } diff --git a/pkgs/sdk_triage_bot/lib/src/gemini.dart b/pkgs/sdk_triage_bot/lib/src/gemini.dart index 64e73784..1b8195e3 100644 --- a/pkgs/sdk_triage_bot/lib/src/gemini.dart +++ b/pkgs/sdk_triage_bot/lib/src/gemini.dart @@ -44,7 +44,7 @@ class GeminiService { /// On failures, this will throw a `GenerativeAIException`. Future> classify(String prompt) async { final result = await _query(_classifyModel, prompt); - final labels = result.split(',').map((l) => l.trim()).toList(); + final labels = result.split(',').map((l) => l.trim()).toList()..sort(); return labels; } diff --git a/pkgs/sdk_triage_bot/lib/src/prompts.dart b/pkgs/sdk_triage_bot/lib/src/prompts.dart index 95137b06..36157c9a 100644 --- a/pkgs/sdk_triage_bot/lib/src/prompts.dart +++ b/pkgs/sdk_triage_bot/lib/src/prompts.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// TODO(devoncarew): Add additional prompt instructions for `area-pkg` issues. + String assignAreaPrompt({ required String title, required String body, @@ -45,6 +47,10 @@ If the issue is clearly a bug report, then also apply the label 'type-bug'. If the issue is mostly a question, then also apply the label 'type-question'. Otherwise don't apply a 'type-' label. +If the issue title starts with "[breaking change]" then apply the +`breaking-change-request` label but don't assign an area label. IMPORTANT: only +do this if the issue title starts with "[breaking change]". + If the issue was largely unchanged from our default issue template, then apply the 'needs-info' label and don't assign an area label. These issues will generally have a title of "Create an issue" and the body will start with @@ -54,10 +60,6 @@ If the issue title is "Analyzer Feedback from IntelliJ", these are generally not well qualified. For these issues, apply the 'needs-info' label but don't assign an area label. -If the issue title starts with "[breaking change] " then it doesn't need to be -triaged into a specific area; apply the `breaking-change-request` label but -don't assign an area label. - Return the labels as comma separated text. Here are a series of few-shot examples: @@ -89,6 +91,61 @@ body: ## Version information OUTPUT: needs-info + +INPUT: title: Support likely() and unlikely() hints for AOT code optimization + +body: ```dart +// Tell the compiler which branches are going to be taken most of the time. + +if (unlikely(n == 0)) { + // This branch is known to be taken rarely. +} else { + // This branch is expected to be in the hot path. +} + +final result = likely(s == null) ? commonPath() : notTakenOften(); +``` + +Please add support for the `likely()` and `unlikely()` optimization hints within branching conditions. The AOT compiler can use these hints to generate faster code in a hot path that contains multiple branches. + +OUTPUT: area-vm, type-enhancement, type-performance + + + +INPUT: title: Analyzer doesn't notice incorrect return type of generic method + +body: dart analyze gives no errors on the follow code: + +```dart +void main() { + method(getB()); +} + +void method(String b) => print(b); + +B getB() { + return A() as B; +} + +class A {} +``` +I would have suspected it to say something along the line of **The argument type 'A' can't be assigned to the parameter type 'String'.** + +OUTPUT: area-analyzer, type-enhancement + + + +INPUT: title: DDC async function stepping improvements + +body: Tracking issue to monitor progress on improving debugger stepping through async function bodies. + +The new DDC async semantics expand async function bodies into complex state machines. The normal JS stepping semantics don't map cleanly to steps through Dart code given this lowering. There are a couple potential approaches to fix this: +1) Add more logic to the Dart debugger to perform custom stepping behavior when stepping through async code. +2) Modify the async lowering in such a way that stepping more closely resembles stepping through Dart. For example, rather than returning multiple times, the state machine function might be able to yield. Stepping over a yield might allow the debugger to stay within the function body. + +OUTPUT: area-web + + The issue to triage follows: title: $title @@ -106,20 +163,18 @@ String summarizeIssuePrompt({ }) { const needsMoreInfo = ''' Our classification model determined that we'll need more information to triage -this issue. Please gently prompt the user to provide more information. +this issue. Thank them for their contribution and gently prompt them to provide +more information. '''; - final needsInfoVerbiage = needsInfo ? needsMoreInfo : ''; - final responseLimit = needsInfo - ? '2-3 sentences, 50 words or less' - : '1-2 sentences, 24 words or less'; + final responseLimit = needsInfo ? '' : ' (1-2 sentences, 24 words or less)'; return ''' -You are a software engineer on the Dart team at Google. You are responsible for -triaging incoming issues from users. For each issue, briefly summarize the issue -($responseLimit). +You are a software engineer on the Dart team at Google. +You are responsible for triaging incoming issues from users. +For each issue, briefly summarize the issue $responseLimit. -$needsInfoVerbiage +${needsInfo ? needsMoreInfo : ''} The issue to triage follows: diff --git a/pkgs/sdk_triage_bot/lib/triage.dart b/pkgs/sdk_triage_bot/lib/triage.dart index 9a3de9fa..e7561e01 100644 --- a/pkgs/sdk_triage_bot/lib/triage.dart +++ b/pkgs/sdk_triage_bot/lib/triage.dart @@ -131,9 +131,9 @@ ${trimmedBody(comment.body ?? '')} // create github comment await githubService.createComment(sdkSlug, issueNumber, comment); - final allRepoLabels = (await githubService.getAllLabels(sdkSlug)).toSet(); - final labelAdditions = newLabels.toSet().intersection(allRepoLabels).toList() - ..sort(); + final allRepoLabels = await githubService.getAllLabels(sdkSlug); + final labelAdditions = + filterLegalLabels(newLabels, allRepoLabels: allRepoLabels); if (labelAdditions.isNotEmpty) { labelAdditions.add('triage-automation'); } @@ -149,7 +149,9 @@ ${trimmedBody(comment.body ?? '')} logger.log('Triaged ${issue.htmlUrl}'); } -List filterExistingLabels( - List allLabels, List newLabels) { - return newLabels.toSet().intersection(allLabels.toSet()).toList(); +List filterLegalLabels( + List labels, { + required List allRepoLabels, +}) { + return labels.toSet().intersection(allRepoLabels.toSet()).toList()..sort(); } diff --git a/pkgs/sdk_triage_bot/todo.txt b/pkgs/sdk_triage_bot/todo.txt deleted file mode 100644 index 54a7dd1c..00000000 --- a/pkgs/sdk_triage_bot/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ - -- special case `area-pkg` diff --git a/pkgs/sdk_triage_bot/tool/bench.dart b/pkgs/sdk_triage_bot/tool/bench.dart index 62d597ab..c08c8b1c 100644 --- a/pkgs/sdk_triage_bot/tool/bench.dart +++ b/pkgs/sdk_triage_bot/tool/bench.dart @@ -51,7 +51,7 @@ void main(List args) async { await githubService.fetchIssue(sdkSlug, expectation.issueNumber); final bodyTrimmed = trimmedBody(issue.body); - print('#${issue.number}'); + print('#${issue.number}: ${expectation.expectedLabels.join(', ')}'); try { final labels = await geminiService.classify( @@ -60,12 +60,7 @@ void main(List args) async { if (expectation.satisfiedBy(labels)) { predicted++; } else { - var title = issue.title.length > 100 - ? '${issue.title.substring(0, 100)}...' - : issue.title; - print(' "$title"'); - print(' labeled: ${expectation.expectedLabels.join(', ')}'); - print(' prediction: ${labels.join(', ')}'); + stderr.writeln(' bot: ${labels.join(', ')}'); } } on GenerativeAIException catch (e) { // Failures here can include things like gemini safety issues, ... @@ -114,8 +109,22 @@ class ClassificationResults { } bool satisfiedBy(List labels) { - final filtered = labels.where((l) => !l.startsWith('type-')).toSet(); - final expected = expectedLabels.where((l) => !l.startsWith('type-')); - return expected.every(filtered.contains); + // handle needs-info + if (expectedLabels.contains('needs-info')) { + return labels.contains('needs-info'); + } + + // handle breaking-change-request + if (expectedLabels.contains('breaking-change-request')) { + return labels.contains('breaking-change-request'); + } + + for (final label in expectedLabels.where((l) => l.startsWith('area-'))) { + if (!labels.contains(label)) { + return false; + } + } + + return true; } } diff --git a/pkgs/sdk_triage_bot/tool/bench.md b/pkgs/sdk_triage_bot/tool/bench.md index c5d189a5..1179b474 100644 --- a/pkgs/sdk_triage_bot/tool/bench.md +++ b/pkgs/sdk_triage_bot/tool/bench.md @@ -20,7 +20,6 @@ materially improve the classification performance. | #56354 | `area-web`, `type-bug` | | #56353 | `area-dart2wasm` | | #56350 | `area-analyzer`, `type-enhancement` | -| #56348 | `area-intellij` | | #56347 | `area-dart-cli`, `type-bug` | | #56346 | `area-pkg`, `pkg-json`, `type-enhancement` | | #56345 | `area-analyzer`, `type-enhancement` | @@ -44,7 +43,7 @@ materially improve the classification performance. | #56316 | `area-web` | | #56315 | `area-web` | | #56314 | `area-web`, `type-bug` | -| #56308 | `area-vm` | +| #56308 | `area-vm` `breaking-change-request` | | #56306 | `area-vm`, `type-bug` | | #56305 | `area-front-end`, `type-bug`, `type-question` | | #56304 | `area-core-library`, `type-enhancement` | @@ -55,13 +54,12 @@ materially improve the classification performance. | #56283 | `area-dart2wasm` | | #56256 | `area-front-end`, `type-bug` | | #56254 | `area-pkg`, `pkg-vm-service`, `type-bug` | -| #56246 | `area-intellij` | -| #56240 | `area-intellij` | +| #56240 | `needs-info` | | #56229 | `area-infrastructure` | | #56227 | `area-native-interop` | | #56220 | `area-infrastructure`, `type-code-health` | | #56217 | `area-meta` | -| #56216 | `area-intellij` | +| #56216 | `needs-info` | | #56214 | `area-native-interop` | | #56208 | `area-google3`, `type-enhancement` | | #56207 | `area-google3` | @@ -108,3 +106,4 @@ We need more information from the user before we can triage these issues. ## Results 2024-08-27: 55.6% using gemini-1.5-flash-latest +2024-08-30: 64.8% using gemini-1.5-flash-latest