get prs by commits

generate-migration-guide/src/github_client.rs

@@ -18,7 +18,7 @@ pub struct GithubBranchesCommitResponse {
 pub struct GithubCommitResponse {
     pub sha: String,
     pub commit: GithubCommitContent,
-    pub author: GithubUser,
+    pub author: Option<GithubUser>,
 }
 
 #[derive(Deserialize, Clone, Debug)]
@@ -112,6 +112,26 @@ impl GithubClient {
         bail!("commit sha not found for main branch")
     }
 
+    /// Gets a list of all PRs merged by bors after the given date.
+    /// The date needs to be in the YYYY-MM-DD format
+    /// To validate that bors merged the PR we simply check if the pr title contains "[Merged by Bors] - "
+    pub fn get_commits(&self, since: &str, sha: &str) -> anyhow::Result<Vec<GithubCommitResponse>> {
+        let mut commits = vec![];
+        let mut page = 1;
+        // The github rest api is limited to 100 prs per page,
+        // so to get all the prs we need to iterate on every page available.
+        loop {
+            let mut commits_in_page = self.get_commits_by_page(since, page, sha)?;
+            println!("Page: {} ({} commits)", page, commits_in_page.len());
+            if commits_in_page.is_empty() {
+                break;
+            }
+            commits.append(&mut commits_in_page);
+            page += 1;
+        }
+        Ok(commits)
+    }
+
     #[allow(unused)]
     pub fn get_commits_by_page(
         &self,
@@ -144,7 +164,7 @@ impl GithubClient {
         }
         let request = self
             .get("https://github.com/gitapi/search/users")
-            .query("q", &format!("{email} in:email"));
+            .query("q", &format!("{email}"));
         let response = request.call()?.into_json()?;
         self.user_cache.insert(email.to_string(), response);
         Ok(self.user_cache.get(email).unwrap().clone())
@@ -213,4 +233,10 @@ impl GithubClient {
             .cloned()
             .collect())
     }
+
+    pub fn generate_release_note(&self) -> anyhow::Result<String> {
+        let request =
+            self.get("https://github.com/gitapi/repos/bevyengine/bevy/releases/generate-notes");
+        Ok(request.call()?.into_json()?)
+    }
 }

generate-migration-guide/src/main.rs

@@ -1,6 +1,8 @@
+use anyhow::Context;
 use clap::{Parser as ClapParser, Subcommand};
 use github_client::GithubClient;
 use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag};
+use regex::Regex;
 use std::{
     collections::{HashMap, HashSet},
     fmt::Write,
@@ -97,29 +99,107 @@ fn main() -> anyhow::Result<()> {
 
 /// Generates the list of contributors and a list of all closed PRs sorted by area labels
 fn generate_release_note(
-    date: &str,
+    since: &str,
     path: PathBuf,
     client: &mut GithubClient,
 ) -> anyhow::Result<()> {
-    let prs = client.get_merged_prs(date, None)?;
+    let main_sha = client
+        .get_branch_sha("main")
+        .context("Failed to get branch_sha")?;
+
+    println!("commit sha for main: {main_sha}");
+
+    // We use the list of commits to make sure the PRs are only on main
+    let commits = client
+        .get_commits(since, &main_sha)
+        .context("Failed to get commits for branch")?;
+    // We also get the list of merged PRs in batches instead of getting them separately for each commit
+    let prs = client.get_merged_prs(since, None)?;
 
-    let mut authors = HashSet::new();
     let mut pr_map = HashMap::new();
     let mut areas = HashMap::<String, Vec<i32>>::new();
-    for pr in &prs {
-        authors.insert(pr.user.login.clone());
-        pr_map.insert(pr.number, pr.clone());
+    let mut authors = HashSet::new();
+    let mut co_authors = HashSet::<String>::new();
+
+    for commit in &commits {
+        let mut message_lines = commit.commit.message.lines();
+
+        // Title is always the first line of a commit message
+        let title = message_lines.next().context("Commit message empty")?;
+
+        // Get the pr number added by bors at the end of the title
+        let re = Regex::new(r"\(#([\d]*)\)").unwrap();
+        let Some(cap) = re.captures_iter(title).last() else {
+            // This means there wasn't a PR associated with the commit
+            // Or bors didn't add a pr number
+            continue;
+        };
+        // remove PR number from title
+        let title = title.replace(&cap[0].to_string(), "");
+        let title = title.trim_end();
+        // let pr_number = cap[1].to_string();
+
+        // This is really expensive. Consider querying a list of PRs separately and cache the result
+        // let pr = client.get_pr_by_number(&pr_number)?;
+        let Some(pr) = prs.iter().find(|pr| pr.title.contains(title)) else {
+            println!("\x1b[93mPR not found for {title}\x1b[0m");
+            continue;
+        };
+
+        // Find co-authors
+        loop {
+            let Some(line) = message_lines.next() else {
+                break;
+            };
+
+            if line.starts_with("Co-authored-by: ") {
+                let user = line.replace("Co-authored-by: ", "");
+                let re = Regex::new(r"<(.*)>").unwrap();
+                let Some(cap) = re.captures_iter(line).last() else {
+                    continue;
+                };
+                let email = cap[1].to_string();
+                // co_authors.insert(email);
+                // This is really slow and a lot of users aren't found by it
+                match client.get_user_by_email(&user) {
+                    Ok(possible_users) => {
+                        co_authors.insert(if possible_users.items.is_empty() {
+                            format!("<{email}> -> not found")
+                        } else {
+                            let login = possible_users.items[0].login.clone();
+                            format!("<{email}> -> @{login}")
+                        });
+                    }
+                    Err(err) => {
+                        println!("Error while getting user by email: {}", err);
+                        println!("sleeping to avoid being rate limited");
+                        std::thread::sleep(std::time::Duration::from_secs(10));
+                        println!("sleeping to avoid being rate limited");
+                        std::thread::sleep(std::time::Duration::from_secs(10));
+                        println!("sleeping to avoid being rate limited");
+                        std::thread::sleep(std::time::Duration::from_secs(10));
+                    }
+                }
+            }
+        }
+
+        pr_map.insert(pr.number, title.to_string());
 
         let area = if let Some(label) = pr.labels.iter().find(|l| l.name.starts_with("A-")) {
             label.name.clone()
         } else {
             String::from("No area label")
         };
-
         areas.entry(area).or_default().push(pr.number);
+
+        authors.insert(pr.user.login.clone());
+        println!(
+            "[{title}](https://github.com/bevyengine/bevy/pull/{})",
+            pr.number
+        );
     }
 
-    println!("Found {} prs merged by bors since {}", prs.len(), date);
+    println!("Found {} prs merged by bors since {}", commits.len(), since);
 
     let mut output = String::new();
 
@@ -129,6 +209,19 @@ fn generate_release_note(
         writeln!(&mut output, "- @{}", author)?;
     }
     writeln!(&mut output)?;
+    writeln!(&mut output, "### Co-Authors")?;
+
+    writeln!(&mut output)?;
+    writeln!(
+        &mut output,
+        "!!! WARNING This section should be removed before release !!!"
+    )?;
+    writeln!(&mut output)?;
+
+    for co_author in &co_authors {
+        writeln!(&mut output, "- {}", co_author)?;
+    }
+    writeln!(&mut output)?;
 
     writeln!(&mut output, "## Full Changelog")?;
 
@@ -138,26 +231,20 @@ fn generate_release_note(
         writeln!(&mut output)?;
 
         for pr_number in prs {
-            let Some(pr) = pr_map.get(pr_number) else {
+            let Some(pr_title) = pr_map.get(pr_number) else {
                 continue;
             };
-            let pr_title = pr
-                .title
-                .replace("[Merged by Bors] - ", "")
-                .trim()
-                .to_string();
-
             writeln!(&mut output, "- [{}][{}]", pr_title, pr_number)?;
         }
     }
 
     writeln!(&mut output)?;
 
-    for pr in prs {
+    for pr in pr_map.keys() {
         writeln!(
             &mut output,
             "[{}]: https://github.com/bevyengine/bevy/pull/{}",
-            pr.number, pr.number
+            pr, pr
         )?;
     }