From 8a6b3e689cfa56ba4dd9b2b940e912aa18d30d8b Mon Sep 17 00:00:00 2001
From: Sam Johnson <sam@durosoft.com>
Date: Wed, 3 Apr 2024 01:46:01 -0400
Subject: [PATCH] multibyte parsing test

---
 src/parsing.rs | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/parsing.rs b/src/parsing.rs
index cdd398e..4a4aedf 100644
--- a/src/parsing.rs
+++ b/src/parsing.rs
@@ -151,7 +151,8 @@ impl ParseStream {
         let text: IndexedString = value.to_string().to_lowercase().into();
         let remaining_lower = self.remaining().to_lowercase();
         if remaining_lower.starts_with(&text) {
-            return Ok(Exact::new(self.consume(text.len())?));
+            let span = self.consume(text.len())?;
+            return Ok(Exact::new(span));
         }
         let prefix = common_prefix(&text, &remaining_lower);
         let expected = &text.slice(prefix.len()..);
@@ -520,7 +521,7 @@ impl Peekable for &str {
     }
 
     fn peek_value(value: Self, stream: &ParseStream) -> bool {
-        stream.remaining().starts_with(value)
+        stream.remaining().to_string().starts_with(value)
     }
 }
 
@@ -713,3 +714,15 @@ fn test_regex_parsing() {
         .unwrap_err();
     assert!(parsed.to_string().contains("expected match for"));
 }
+
+#[test]
+fn test_multibyte_parsing() {
+    let mut stream = ParseStream::from("你好, 世界");
+    assert!(stream.peek_value("你"));
+    let parsed = stream.parse_istr("你好").unwrap();
+    println!("parsed: |{}|", parsed.to_string());
+    println!("remaining: |{}|", stream.remaining());
+    assert_eq!(parsed.to_string(), "你好");
+    assert_ne!(stream.source().len(), stream.source().byte_len());
+    assert!(stream.peek_value(","));
+}