From 6773e27b5e5a435260236140fd03185f173f7d01 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Fri, 5 May 2023 10:30:29 -0400 Subject: [PATCH 01/14] Add support for interpolated strings --- Cargo.lock | 72 ++++--- Cargo.toml | 2 +- src/ast_converter.rs | 155 +++++++++++++- src/generator/dense.rs | 1 + src/generator/readable.rs | 3 + src/generator/token_based.rs | 3 + src/generator/utils.rs | 1 + src/nodes/expressions/binary.rs | 1 + src/nodes/expressions/interpolated_string.rs | 204 +++++++++++++++++++ src/nodes/expressions/mod.rs | 9 + src/nodes/expressions/prefix.rs | 16 +- src/parser.rs | 134 ++++++++++++ src/process/evaluator/mod.rs | 7 + src/process/node_processor.rs | 1 + src/process/visitors.rs | 4 + src/rules/remove_comments.rs | 5 + src/rules/remove_compound_assign.rs | 1 + src/rules/remove_spaces.rs | 5 + 18 files changed, 578 insertions(+), 46 deletions(-) create mode 100644 src/nodes/expressions/interpolated_string.rs diff --git a/Cargo.lock b/Cargo.lock index c6daa1f..0104ca0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,9 +115,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytecount" -version = "0.5.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0fdd54b507df8f22012890aadd099979befdba27713c767993f8380112ca7c" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" [[package]] name = "byteorder" @@ -357,7 +357,7 @@ dependencies = [ "json5", "log", "node-sys", - "paste 1.0.6", + "paste", "pretty_assertions", "rand", "rand_distr", @@ -505,18 +505,19 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "full_moon" -version = "0.16.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cce7c865934d6b2b9eb7dc4413a24cb6fbc67f21ae002427b5e36c835508a69" +checksum = "7b9a9bf5e42aec08f4b59be1438d66b01ab0a0f51dca309626e219697b60871c" dependencies = [ "bytecount", "cfg-if 1.0.0", "derive_more", "full_moon_derive", "logos", - "paste 0.1.18", + "paste", "serde", "smol_str", + "stacker", ] [[package]] @@ -908,31 +909,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "paste" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" -dependencies = [ - "paste-impl", - "proc-macro-hack", -] - [[package]] name = "paste" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" -[[package]] -name = "paste-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" -dependencies = [ - "proc-macro-hack", -] - [[package]] name = "pest" version = "2.1.3" @@ -1079,12 +1061,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - [[package]] name = "proc-macro2" version = "1.0.47" @@ -1094,6 +1070,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "quote" version = "1.0.13" @@ -1290,18 +1275,18 @@ checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" [[package]] name = "serde" -version = "1.0.132" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.132" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" dependencies = [ "proc-macro2", "quote", @@ -1354,9 +1339,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "smol_str" -version = "0.1.21" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61d15c83e300cce35b7c8cd39ff567c1ef42dde6d4a1a38dbdbf9a59902261bd" +checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" dependencies = [ "serde", ] @@ -1367,6 +1352,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "libc", + "psm", + "winapi", +] + [[package]] name = "strsim" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index 814f604..0cf8e27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ clap = { version = "4.1.1", features = ["derive"] } durationfmt = "0.1.1" env_logger = "0.9.0" log = "0.4" -full_moon = { version = "0.16.2", features = ["roblox"] } +full_moon = { version = "0.18.1", features = ["roblox", "stacker"] } serde = { version = "1.0", features = ["derive"] } json5 = "0.4" elsa = "1.7.0" diff --git a/src/ast_converter.rs b/src/ast_converter.rs index af03af9..6100347 100644 --- a/src/ast_converter.rs +++ b/src/ast_converter.rs @@ -2,7 +2,7 @@ use std::{fmt, str::FromStr}; use full_moon::{ ast, - tokenizer::{self, Symbol, TokenType}, + tokenizer::{self, InterpolatedStringKind, Symbol, TokenType}, }; use crate::nodes::*; @@ -322,6 +322,116 @@ impl<'a> AstConverter<'a> { self.expressions.push(value.into()); } + ConvertWork::MakeInterpolatedString { + interpolated_string, + } => { + let mut segments = Vec::new(); + let mut segments_iter = interpolated_string.segments().peekable(); + + while let Some(segment) = segments_iter.next() { + let literal = &segment.literal; + if let Some(segment) = self.convert_string_interpolation_segment(literal)? { + segments.push(segment.into()); + } + + let expression = self.pop_expression()?; + let mut value_segment = ValueSegment::new(expression); + + if self.hold_token_data { + let opening_brace = Token::new_with_line( + literal.end_position().bytes().saturating_sub(1), + literal.end_position().bytes(), + literal.end_position().line(), + ); + + let next_literal = segments_iter + .peek() + .map(|next_segment| &next_segment.literal) + .unwrap_or(interpolated_string.last_string()); + + let start_position = next_literal.start_position().bytes(); + let closing_brace = Token::new_with_line( + start_position, + start_position + 1, + next_literal.start_position().line(), + ); + + value_segment.set_tokens(ValueSegmentTokens { + opening_brace, + closing_brace, + }); + } + + segments.push(value_segment.into()); + } + + if let Some(segment) = self + .convert_string_interpolation_segment(interpolated_string.last_string())? + { + segments.push(segment.into()); + } + + let mut value = InterpolatedStringExpression::new(segments); + + if self.hold_token_data { + let last = interpolated_string.last_string(); + let first = interpolated_string + .segments() + .next() + .map(|segment| &segment.literal) + .unwrap_or(last); + + let (opening_tick, closing_tick) = match first.token_type() { + TokenType::InterpolatedString { literal: _, kind } => match kind { + InterpolatedStringKind::Begin | InterpolatedStringKind::Simple => { + let start_position = first.start_position().bytes(); + let mut start_token = Token::new_with_line( + start_position, + start_position + 1, + first.start_position().line(), + ); + let end_position = last.end_position().bytes(); + let mut end_token = Token::new_with_line( + end_position.saturating_sub(1), + end_position, + last.end_position().line(), + ); + + for trivia_token in first.leading_trivia() { + start_token.push_leading_trivia( + self.convert_trivia(trivia_token)?, + ); + } + + for trivia_token in first.trailing_trivia() { + end_token.push_trailing_trivia( + self.convert_trivia(trivia_token)?, + ); + } + (start_token, end_token) + } + InterpolatedStringKind::Middle | InterpolatedStringKind::End => { + return Err(ConvertError::InterpolatedString { + string: interpolated_string.to_string(), + }) + } + }, + _ => { + return Err(ConvertError::InterpolatedString { + string: interpolated_string.to_string(), + }) + } + }; + + let tokens = InterpolatedStringTokens { + opening_tick, + closing_tick, + }; + value.set_tokens(tokens); + } + + self.expressions.push(value.into()); + } ConvertWork::MakeFunctionExpression { body, token } => { let block = self.pop_block()?; let (parameters, is_variadic, tokens) = @@ -1049,6 +1159,14 @@ impl<'a> AstConverter<'a> { } } } + ast::Value::InterpolatedString(interpolated_string) => { + self.push_work(ConvertWork::MakeInterpolatedString { + interpolated_string, + }); + for segment in interpolated_string.segments() { + self.push_work(&segment.expression); + } + } _ => { return Err(ConvertError::Expression { expression: expression.to_string(), @@ -1366,6 +1484,34 @@ impl<'a> AstConverter<'a> { } Ok(()) } + + fn convert_string_interpolation_segment( + &self, + token: &tokenizer::TokenReference, + ) -> Result, ConvertError> { + match token.token_type() { + TokenType::InterpolatedString { literal, kind: _ } => { + if !literal.is_empty() { + let mut segment = StringSegment::new(literal.as_str()); + + if self.hold_token_data { + let segment_token = Token::new_with_line( + token.start_position().bytes() + 1, + token.end_position().bytes().saturating_sub(1), + token.start_position().line(), + ); + // no trivia since it is grabbing a substring of the token + segment.set_token(segment_token); + } + + Ok(Some(segment)) + } else { + Ok(None) + } + } + _ => unreachable!(), + } + } } #[derive(Debug)] @@ -1459,6 +1605,9 @@ enum ConvertWork<'a> { MakePrefixExpression { variable: &'a ast::VarExpression, }, + MakeInterpolatedString { + interpolated_string: &'a ast::types::InterpolatedString, + }, } impl<'a> From<&'a ast::Block> for ConvertWork<'a> { @@ -1548,6 +1697,9 @@ pub(crate) enum ConvertError { UnaryOperator { operator: String, }, + InterpolatedString { + string: String, + }, UnexpectedTrivia(tokenizer::TokenKind), ExpectedFunctionName, InternalStack { @@ -1576,6 +1728,7 @@ impl fmt::Display for ConvertError { number, parsing_error ) } + ConvertError::InterpolatedString { string } => ("interpolated string", string), ConvertError::Expression { expression } => ("expression", expression), ConvertError::FunctionParameter { parameter } => ("parameter", parameter), ConvertError::FunctionParameters { parameters } => ("parameters", parameters), diff --git a/src/generator/dense.rs b/src/generator/dense.rs index 5cda014..5f84ea6 100644 --- a/src/generator/dense.rs +++ b/src/generator/dense.rs @@ -474,6 +474,7 @@ impl LuaGenerator for DenseLuaGenerator { Number(number) => self.write_number(number), Parenthese(parenthese) => self.write_parenthese(parenthese), String(string) => self.write_string(string), + InterpolatedString(interpolated_string) => todo!(), Table(table) => self.write_table(table), True(_) => self.push_str("true"), Unary(unary) => self.write_unary_expression(unary), diff --git a/src/generator/readable.rs b/src/generator/readable.rs index 98dd641..cfab381 100644 --- a/src/generator/readable.rs +++ b/src/generator/readable.rs @@ -692,6 +692,9 @@ impl LuaGenerator for ReadableLuaGenerator { Number(number) => self.write_number(number), Parenthese(parenthese) => self.write_parenthese(parenthese), String(string) => self.write_string(string), + InterpolatedString(interpolated_string) => { + todo!() + } Table(table) => self.write_table(table), True(_) => self.push_str("true"), Unary(unary) => self.write_unary_expression(unary), diff --git a/src/generator/token_based.rs b/src/generator/token_based.rs index 9ee325e..5684eec 100644 --- a/src/generator/token_based.rs +++ b/src/generator/token_based.rs @@ -1147,6 +1147,9 @@ impl<'a> LuaGenerator for TokenBasedLuaGenerator<'a> { Number(number) => self.write_number(number), Parenthese(parenthese) => self.write_parenthese(parenthese), String(string) => self.write_string(string), + InterpolatedString(interpolated_string) => { + todo!() + } Table(table) => self.write_table(table), True(token) => { if let Some(token) = token { diff --git a/src/generator/utils.rs b/src/generator/utils.rs index d4229e0..d3b256e 100644 --- a/src/generator/utils.rs +++ b/src/generator/utils.rs @@ -113,6 +113,7 @@ fn expression_ends_with_prefix(expression: &Expression) -> bool { | Expression::Nil(_) | Expression::Number(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::True(_) | Expression::VariableArguments(_) => false, diff --git a/src/nodes/expressions/binary.rs b/src/nodes/expressions/binary.rs index 35eb227..23783d8 100644 --- a/src/nodes/expressions/binary.rs +++ b/src/nodes/expressions/binary.rs @@ -38,6 +38,7 @@ fn ends_with_if_expression(expression: &Expression) -> bool { | Expression::Number(_) | Expression::Parenthese(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::True(_) | Expression::VariableArguments(_) => break false, diff --git a/src/nodes/expressions/interpolated_string.rs b/src/nodes/expressions/interpolated_string.rs new file mode 100644 index 0000000..b05fdca --- /dev/null +++ b/src/nodes/expressions/interpolated_string.rs @@ -0,0 +1,204 @@ +use std::iter::FromIterator; + +use crate::nodes::Token; + +use super::Expression; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StringSegment { + value: String, + token: Option, +} + +impl StringSegment { + pub fn new(value: impl Into) -> Self { + Self { + value: value.into(), + token: None, + } + } + + pub fn with_token(mut self, token: Token) -> Self { + self.token = Some(token); + self + } + + pub fn set_token(&mut self, token: Token) { + self.token = Some(token); + } + + pub fn get_token(&self) -> Option<&Token> { + self.token.as_ref() + } + + pub fn clear_comments(&mut self) { + if let Some(token) = &mut self.token { + token.clear_comments(); + } + } + + pub fn clear_whitespaces(&mut self) { + if let Some(token) = &mut self.token { + token.clear_whitespaces(); + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValueSegment { + value: Expression, + tokens: Option, +} + +impl ValueSegment { + pub fn new(value: impl Into) -> Self { + Self { + value: value.into(), + tokens: None, + } + } + + pub fn with_tokens(mut self, tokens: ValueSegmentTokens) -> Self { + self.tokens = Some(tokens); + self + } + + pub fn set_tokens(&mut self, tokens: ValueSegmentTokens) { + self.tokens = Some(tokens); + } + + pub fn get_tokens(&self) -> Option<&ValueSegmentTokens> { + self.tokens.as_ref() + } + + pub fn clear_comments(&mut self) { + if let Some(tokens) = &mut self.tokens { + tokens.clear_comments(); + } + } + + pub fn clear_whitespaces(&mut self) { + if let Some(tokens) = &mut self.tokens { + tokens.clear_whitespaces(); + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValueSegmentTokens { + pub opening_brace: Token, + pub closing_brace: Token, +} + +impl ValueSegmentTokens { + pub fn clear_comments(&mut self) { + self.opening_brace.clear_comments(); + self.closing_brace.clear_comments(); + } + + pub fn clear_whitespaces(&mut self) { + self.opening_brace.clear_whitespaces(); + self.closing_brace.clear_whitespaces(); + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum InterpolationSegment { + String(StringSegment), + Value(ValueSegment), +} + +impl InterpolationSegment { + pub fn clear_comments(&mut self) { + match self { + InterpolationSegment::String(segment) => segment.clear_comments(), + InterpolationSegment::Value(segment) => segment.clear_comments(), + } + } + + pub fn clear_whitespaces(&mut self) { + match self { + InterpolationSegment::String(segment) => segment.clear_whitespaces(), + InterpolationSegment::Value(segment) => segment.clear_whitespaces(), + } + } +} + +impl From for InterpolationSegment { + fn from(segment: StringSegment) -> Self { + Self::String(segment) + } +} + +impl From for InterpolationSegment { + fn from(segment: ValueSegment) -> Self { + Self::Value(segment) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InterpolatedStringExpression { + segments: Vec, + tokens: Option, +} + +impl InterpolatedStringExpression { + pub fn new(segments: Vec) -> Self { + Self { + segments, + tokens: None, + } + } + + pub fn empty() -> Self { + Self::new(Vec::default()) + } + + pub fn with_tokens(mut self, tokens: InterpolatedStringTokens) -> Self { + self.tokens = Some(tokens); + self + } + + pub fn set_tokens(&mut self, tokens: InterpolatedStringTokens) { + self.tokens = Some(tokens); + } + + pub fn clear_comments(&mut self) { + if let Some(tokens) = &mut self.tokens { + tokens.clear_comments(); + } + } + + pub fn clear_whitespaces(&mut self) { + if let Some(tokens) = &mut self.tokens { + tokens.clear_whitespaces(); + } + } +} + +impl FromIterator for InterpolatedStringExpression { + fn from_iter>(iter: T) -> Self { + Self { + segments: iter.into_iter().collect(), + tokens: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InterpolatedStringTokens { + pub opening_tick: Token, + pub closing_tick: Token, +} + +impl InterpolatedStringTokens { + pub fn clear_comments(&mut self) { + self.opening_tick.clear_comments(); + self.closing_tick.clear_comments(); + } + + pub fn clear_whitespaces(&mut self) { + self.opening_tick.clear_whitespaces(); + self.closing_tick.clear_whitespaces(); + } +} diff --git a/src/nodes/expressions/mod.rs b/src/nodes/expressions/mod.rs index 5cff6b9..bc0cdbf 100644 --- a/src/nodes/expressions/mod.rs +++ b/src/nodes/expressions/mod.rs @@ -3,6 +3,7 @@ mod field; mod function; mod if_expression; mod index; +mod interpolated_string; mod number; mod parenthese; mod prefix; @@ -15,6 +16,7 @@ pub use field::*; pub use function::*; pub use if_expression::*; pub use index::*; +pub use interpolated_string::*; pub use number::*; pub use parenthese::*; pub use prefix::*; @@ -40,6 +42,7 @@ pub enum Expression { Number(NumberExpression), Parenthese(Box), String(StringExpression), + InterpolatedString(InterpolatedStringExpression), Table(TableExpression), True(Option), Unary(Box), @@ -272,6 +275,12 @@ impl From for Expression { } } +impl From for Expression { + fn from(interpolated_string: InterpolatedStringExpression) -> Self { + Self::InterpolatedString(interpolated_string) + } +} + impl From for Expression { fn from(table: TableExpression) -> Self { Self::Table(table) diff --git a/src/nodes/expressions/prefix.rs b/src/nodes/expressions/prefix.rs index 3974e78..87946ee 100644 --- a/src/nodes/expressions/prefix.rs +++ b/src/nodes/expressions/prefix.rs @@ -20,11 +20,11 @@ impl Prefix { impl From for Prefix { fn from(expression: Expression) -> Self { match expression { - Expression::Call(call) => return Prefix::Call(*call), - Expression::Field(field) => return Prefix::Field(field), - Expression::Identifier(identifier) => return Prefix::Identifier(identifier), - Expression::Index(index) => return Prefix::Index(index), - Expression::Parenthese(parenthese) => return Prefix::Parenthese(*parenthese), + Expression::Call(call) => Prefix::Call(*call), + Expression::Field(field) => Prefix::Field(field), + Expression::Identifier(identifier) => Prefix::Identifier(identifier), + Expression::Index(index) => Prefix::Index(index), + Expression::Parenthese(parenthese) => Prefix::Parenthese(*parenthese), Expression::Binary(_) | Expression::False(_) | Expression::Function(_) @@ -32,12 +32,14 @@ impl From for Prefix { | Expression::Nil(_) | Expression::Number(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::True(_) | Expression::Unary(_) - | Expression::VariableArguments(_) => {} + | Expression::VariableArguments(_) => { + Prefix::Parenthese(ParentheseExpression::new(expression)) + } } - Prefix::Parenthese(ParentheseExpression::new(expression)) } } diff --git a/src/parser.rs b/src/parser.rs index fc6c907..419a766 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -113,6 +113,34 @@ mod test { ), return_true_false("return true, false") => ReturnStatement::one(Expression::from(true)) .with_expression(false), + return_empty_single_quote_string("return ''") => ReturnStatement::one(StringExpression::new("''").unwrap()), + return_empty_double_quote_string("return \"\"") => ReturnStatement::one(StringExpression::new("\"\"").unwrap()), + return_empty_backtick_string("return ``") => ReturnStatement::one(InterpolatedStringExpression::empty()), + return_backtick_string_hello("return `hello`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![StringSegment::new("hello").into()] + )), + return_backtick_string_with_single_value("return `{true}`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![ValueSegment::new(true).into()] + )), + return_backtick_string_with_prefixed_single_value("return `value = {true}`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![ + StringSegment::new("value = ").into(), + ValueSegment::new(true).into(), + ] + )), + return_backtick_string_with_suffixed_single_value("return `{false} -> condition`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![ + ValueSegment::new(false).into(), + StringSegment::new(" -> condition").into(), + ] + )), + return_backtick_string_with_prefix_and_suffixed_single_value("return `-> {value} (value)`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![ + StringSegment::new("-> ").into(), + ValueSegment::new(Expression::identifier("value")).into(), + StringSegment::new(" (value)").into(), + ] + )), empty_while_true_do("while true do end") => WhileStatement::new(Block::default(), true), while_false_do_break("while false do break end") => WhileStatement::new( LastStatement::new_break(), @@ -587,6 +615,112 @@ mod test { r#return: spaced_token(0, 6), commas: Vec::new(), }), + return_empty_backtick_string("return ``") => ReturnStatement::one( + InterpolatedStringExpression::empty().with_tokens( + InterpolatedStringTokens { + opening_tick: token_at_first_line(7, 8), + closing_tick: token_at_first_line(8, 9), + } + ) + ).with_tokens(ReturnTokens { + r#return: spaced_token(0, 6), + commas: Vec::new(), + }), + return_backtick_string_hello("return `hello`") => ReturnStatement::one( + InterpolatedStringExpression::new(vec![ + StringSegment::new("hello") + .with_token(token_at_first_line(8, 13)) + .into() + ]) + .with_tokens(InterpolatedStringTokens { + opening_tick: token_at_first_line(7, 8), + closing_tick: token_at_first_line(13, 14), + }) + ).with_tokens(ReturnTokens { + r#return: spaced_token(0, 6), + commas: Vec::new(), + }), + return_backtick_string_with_single_value("return `{true}`") => ReturnStatement::one( + InterpolatedStringExpression::new(vec![ + ValueSegment::new(create_true(9, 0)).with_tokens(ValueSegmentTokens { + opening_brace: token_at_first_line(8, 9), + closing_brace: token_at_first_line(13, 14), + }).into() + ]) + .with_tokens(InterpolatedStringTokens { + opening_tick: token_at_first_line(7, 8), + closing_tick: token_at_first_line(14, 15), + }) + ).with_tokens(ReturnTokens { + r#return: spaced_token(0, 6), + commas: Vec::new(), + }), + return_backtick_string_with_prefixed_single_value("return `value = {true}`") => ReturnStatement::one( + InterpolatedStringExpression::new( + vec![ + StringSegment::new("value = ") + .with_token(token_at_first_line(8, 16)) + .into(), + ValueSegment::new(create_true(17, 0)) + .with_tokens(ValueSegmentTokens { + opening_brace: token_at_first_line(16, 17), + closing_brace: token_at_first_line(21, 22), + }).into(), + ] + ) + .with_tokens(InterpolatedStringTokens { + opening_tick: token_at_first_line(7, 8), + closing_tick: token_at_first_line(22, 23), + }) + ).with_tokens(ReturnTokens { + r#return: spaced_token(0, 6), + commas: Vec::new(), + }), + return_backtick_string_with_suffixed_single_value("return `{true} -> condition`") => ReturnStatement::one( + InterpolatedStringExpression::new( + vec![ + ValueSegment::new(create_true(9, 0)) + .with_tokens(ValueSegmentTokens { + opening_brace: token_at_first_line(8, 9), + closing_brace: token_at_first_line(13, 14), + }).into(), + StringSegment::new(" -> condition") + .with_token(token_at_first_line(14, 27)) + .into(), + ] + ) + .with_tokens(InterpolatedStringTokens { + opening_tick: token_at_first_line(7, 8), + closing_tick: token_at_first_line(27, 28), + }) + ).with_tokens(ReturnTokens { + r#return: spaced_token(0, 6), + commas: Vec::new(), + }), + return_backtick_string_with_prefix_and_suffixed_single_value("return `-> {value} (value)`") => ReturnStatement::one( + InterpolatedStringExpression::new( + vec![ + StringSegment::new("-> ") + .with_token(token_at_first_line(8, 11)) + .into(), + ValueSegment::new(create_identifier("value", 12, 0)) + .with_tokens(ValueSegmentTokens { + opening_brace: token_at_first_line(11, 12), + closing_brace: token_at_first_line(17, 18), + }).into(), + StringSegment::new(" (value)") + .with_token(token_at_first_line(18, 26)) + .into(), + ] + ) + .with_tokens(InterpolatedStringTokens { + opening_tick: token_at_first_line(7, 8), + closing_tick: token_at_first_line(26, 27), + }) + ).with_tokens(ReturnTokens { + r#return: spaced_token(0, 6), + commas: Vec::new(), + }), return_integer_number("return 123") => ReturnStatement::one( DecimalNumber::new(123.0).with_token(token_at_first_line(7, 10)) ).with_tokens(ReturnTokens { diff --git a/src/process/evaluator/mod.rs b/src/process/evaluator/mod.rs index d27c51c..89a0629 100644 --- a/src/process/evaluator/mod.rs +++ b/src/process/evaluator/mod.rs @@ -37,6 +37,9 @@ impl Evaluator { self.evaluate(parenthese.inner_expression()) } Expression::If(if_expression) => self.evaluate_if(if_expression), + Expression::InterpolatedString(interpolated_string) => { + todo!() + } Expression::Call(_) | Expression::Field(_) | Expression::Identifier(_) @@ -61,6 +64,7 @@ impl Evaluator { | Expression::Number(_) | Expression::Parenthese(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::True(_) => false, } @@ -131,6 +135,9 @@ impl Evaluator { .iter() .any(|entry| self.table_entry_has_side_effects(entry)), Expression::Call(call) => self.call_has_side_effects(call), + Expression::InterpolatedString(interpolated_string) => { + todo!() + } } } diff --git a/src/process/node_processor.rs b/src/process/node_processor.rs index 93cb4ee..9168f2a 100644 --- a/src/process/node_processor.rs +++ b/src/process/node_processor.rs @@ -35,6 +35,7 @@ pub trait NodeProcessor { fn process_prefix_expression(&mut self, _: &mut Prefix) {} fn process_parenthese_expression(&mut self, _: &mut ParentheseExpression) {} fn process_string_expression(&mut self, _: &mut StringExpression) {} + fn process_interpolated_string_expression(&mut self, _: &mut InterpolatedStringExpression) {} fn process_table_expression(&mut self, _: &mut TableExpression) {} fn process_unary_expression(&mut self, _: &mut UnaryExpression) {} } diff --git a/src/process/visitors.rs b/src/process/visitors.rs index 932e294..ec338e1 100644 --- a/src/process/visitors.rs +++ b/src/process/visitors.rs @@ -65,6 +65,10 @@ pub trait NodeVisitor { Self::visit_expression(expression.mutate_inner_expression(), processor) } Expression::String(string) => processor.process_string_expression(string), + Expression::InterpolatedString(interpolated_string) => { + processor.process_interpolated_string_expression(interpolated_string); + todo!() + } Expression::Table(table) => Self::visit_table(table, processor), Expression::Unary(unary) => { processor.process_unary_expression(unary); diff --git a/src/rules/remove_comments.rs b/src/rules/remove_comments.rs index ca2adc0..9fbe18a 100644 --- a/src/rules/remove_comments.rs +++ b/src/rules/remove_comments.rs @@ -94,6 +94,7 @@ impl NodeProcessor for RemoveCommentProcessor { | Expression::Number(_) | Expression::Parenthese(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::Unary(_) => {} } @@ -143,6 +144,10 @@ impl NodeProcessor for RemoveCommentProcessor { unary.clear_comments(); } + fn process_interpolated_string_expression(&mut self, string: &mut InterpolatedStringExpression) { + string.clear_comments(); + } + fn process_prefix_expression(&mut self, _: &mut Prefix) {} } diff --git a/src/rules/remove_compound_assign.rs b/src/rules/remove_compound_assign.rs index 2f5ceed..713262b 100644 --- a/src/rules/remove_compound_assign.rs +++ b/src/rules/remove_compound_assign.rs @@ -73,6 +73,7 @@ impl Processor { | Expression::Identifier(_) | Expression::Number(_) | Expression::Nil(_) + | Expression::InterpolatedString(_) | Expression::String(_) | Expression::True(_) | Expression::VariableArguments(_) => None, diff --git a/src/rules/remove_spaces.rs b/src/rules/remove_spaces.rs index 41ad7de..db7eab2 100644 --- a/src/rules/remove_spaces.rs +++ b/src/rules/remove_spaces.rs @@ -94,6 +94,7 @@ impl NodeProcessor for RemoveWhitespacesProcessor { | Expression::Number(_) | Expression::Parenthese(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::Unary(_) => {} } @@ -143,6 +144,10 @@ impl NodeProcessor for RemoveWhitespacesProcessor { unary.clear_whitespaces(); } + fn process_interpolated_string_expression(&mut self, string: &mut InterpolatedStringExpression) { + string.clear_whitespaces(); + } + fn process_prefix_expression(&mut self, _: &mut Prefix) {} } From 3618ee21b9b4c0abf6a33a3cb289fdab937cda21 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Mon, 8 May 2023 16:39:43 -0400 Subject: [PATCH 02/14] implement lua generator for interpolated strings --- src/ast_converter.rs | 11 +- src/generator/dense.rs | 31 ++- src/generator/mod.rs | 20 +- src/generator/readable.rs | 29 ++- ...ense_interpolated_string_only_letters.snap | 5 + ...se_interpolated_string_with_backticks.snap | 5 + ...nterpolated_string_with_double_quotes.snap | 5 + ..._interpolated_string_with_empty_table.snap | 5 + ..._string_with_single_and_double_quotes.snap | 5 + ...nterpolated_string_with_single_quotes.snap | 5 + ...e_interpolated_string_with_true_value.snap | 5 + ...ing__dense_string_with_double_quotes.snap} | 0 ...able_interpolated_string_only_letters.snap | 5 + ...le_interpolated_string_with_backticks.snap | 5 + ...nterpolated_string_with_double_quotes.snap | 5 + ..._interpolated_string_with_empty_table.snap | 5 + ..._string_with_single_and_double_quotes.snap | 5 + ...nterpolated_string_with_single_quotes.snap | 5 + ...e_interpolated_string_with_true_value.snap | 5 + ...__readable_string_with_double_quotes.snap} | 0 ...ased_interpolated_string_only_letters.snap | 5 + ...ed_interpolated_string_with_backticks.snap | 5 + ...nterpolated_string_with_double_quotes.snap | 5 + ..._interpolated_string_with_empty_table.snap | 5 + ..._string_with_single_and_double_quotes.snap | 5 + ...nterpolated_string_with_single_quotes.snap | 5 + ...d_interpolated_string_with_true_value.snap | 5 + ...oken_based_string_with_double_quotes.snap} | 0 src/generator/token_based.rs | 102 ++++++++- src/generator/utils.rs | 31 ++- src/nodes/expressions/interpolated_string.rs | 43 +++- src/nodes/expressions/mod.rs | 2 + src/nodes/expressions/string.rs | 193 ++++------------- src/nodes/expressions/string_utils.rs | 198 ++++++++++++++++++ src/parser.rs | 33 ++- 35 files changed, 620 insertions(+), 178 deletions(-) create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_only_letters.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_backticks.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_double_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_and_double_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_true_value.snap rename src/generator/snapshots/{darklua_core__generator__test__dense__snapshots__string__dense_string_with_dougle_quotes.snap => darklua_core__generator__test__dense__snapshots__string__dense_string_with_double_quotes.snap} (100%) create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_only_letters.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_backticks.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_double_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_and_double_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_true_value.snap rename src/generator/snapshots/{darklua_core__generator__test__readable__snapshots__string__readable_string_with_dougle_quotes.snap => darklua_core__generator__test__readable__snapshots__string__readable_string_with_double_quotes.snap} (100%) create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_only_letters.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_backticks.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_double_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_and_double_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_quotes.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_true_value.snap rename src/generator/snapshots/{darklua_core__generator__test__token_based__snapshots__string__token_based_string_with_dougle_quotes.snap => darklua_core__generator__test__token_based__snapshots__string__token_based_string_with_double_quotes.snap} (100%) create mode 100644 src/nodes/expressions/string_utils.rs diff --git a/src/ast_converter.rs b/src/ast_converter.rs index 6100347..9362f8f 100644 --- a/src/ast_converter.rs +++ b/src/ast_converter.rs @@ -338,12 +338,16 @@ impl<'a> AstConverter<'a> { let mut value_segment = ValueSegment::new(expression); if self.hold_token_data { - let opening_brace = Token::new_with_line( + let mut opening_brace = Token::new_with_line( literal.end_position().bytes().saturating_sub(1), literal.end_position().bytes(), literal.end_position().line(), ); + for trivia_token in literal.trailing_trivia() { + opening_brace.push_trailing_trivia(self.convert_trivia(trivia_token)?); + } + let next_literal = segments_iter .peek() .map(|next_segment| &next_segment.literal) @@ -403,7 +407,7 @@ impl<'a> AstConverter<'a> { ); } - for trivia_token in first.trailing_trivia() { + for trivia_token in last.trailing_trivia() { end_token.push_trailing_trivia( self.convert_trivia(trivia_token)?, ); @@ -1492,7 +1496,8 @@ impl<'a> AstConverter<'a> { match token.token_type() { TokenType::InterpolatedString { literal, kind: _ } => { if !literal.is_empty() { - let mut segment = StringSegment::new(literal.as_str()); + let mut segment = StringSegment::new(literal.as_str()) + .expect("unable to convert interpolated string segment"); if self.hold_token_data { let segment_token = Token::new_with_line( diff --git a/src/generator/dense.rs b/src/generator/dense.rs index 5f84ea6..08314f5 100644 --- a/src/generator/dense.rs +++ b/src/generator/dense.rs @@ -474,7 +474,9 @@ impl LuaGenerator for DenseLuaGenerator { Number(number) => self.write_number(number), Parenthese(parenthese) => self.write_parenthese(parenthese), String(string) => self.write_string(string), - InterpolatedString(interpolated_string) => todo!(), + InterpolatedString(interpolated_string) => { + self.write_interpolated_string(interpolated_string) + } Table(table) => self.write_table(table), True(_) => self.push_str("true"), Unary(unary) => self.write_unary_expression(unary), @@ -721,6 +723,33 @@ impl LuaGenerator for DenseLuaGenerator { } } + fn write_interpolated_string( + &mut self, + interpolated_string: &nodes::InterpolatedStringExpression, + ) { + self.push_char('`'); + + for segment in interpolated_string.iter_segments() { + match segment { + nodes::InterpolationSegment::String(string_segment) => { + self.raw_push_str(&utils::write_interpolated_string_segment(string_segment)); + } + nodes::InterpolationSegment::Value(value) => { + self.raw_push_char('{'); + // add space when value segment is a table + let expression = value.get_expression(); + if matches!(expression, nodes::Expression::Table(_)) { + self.raw_push_char(' '); + } + self.write_expression(expression); + self.push_char('}'); + } + } + } + + self.raw_push_char('`'); + } + fn write_identifier(&mut self, identifier: &nodes::Identifier) { self.push_str(identifier.get_name()); } diff --git a/src/generator/mod.rs b/src/generator/mod.rs index 53f70d4..f0dac32 100644 --- a/src/generator/mod.rs +++ b/src/generator/mod.rs @@ -100,6 +100,7 @@ pub trait LuaGenerator { fn write_tuple_arguments(&mut self, arguments: &nodes::TupleArguments); fn write_string(&mut self, string: &nodes::StringExpression); + fn write_interpolated_string(&mut self, string: &nodes::InterpolatedStringExpression); } #[cfg(test)] @@ -681,10 +682,27 @@ mod $mod_name { snapshot_node!($mod_name, $generator, string, write_expression => ( only_letters => StringExpression::from_value("hello"), with_single_quotes => StringExpression::from_value("I'm cool"), - with_dougle_quotes => StringExpression::from_value(r#"Say: "Hi""#), + with_double_quotes => StringExpression::from_value(r#"Say: "Hi""#), with_single_and_double_quotes => StringExpression::from_value(r#"Say: "Don't""#), )); + snapshot_node!($mod_name, $generator, interpolated_string, write_expression => ( + only_letters => InterpolatedStringExpression::empty() + .with_segment("hello"), + with_single_quotes => InterpolatedStringExpression::empty() + .with_segment("I'm cool"), + with_double_quotes => InterpolatedStringExpression::empty() + .with_segment(r#"Say: "Hi""#), + with_backticks => InterpolatedStringExpression::empty() + .with_segment("Say: `Hi`"), + with_single_and_double_quotes => InterpolatedStringExpression::empty() + .with_segment(r#"Say: "Don't""#), + with_true_value => InterpolatedStringExpression::empty() + .with_segment(Expression::from(true)), + with_empty_table => InterpolatedStringExpression::empty() + .with_segment(Expression::from(TableExpression::default())), + )); + snapshot_node!($mod_name, $generator, number, write_expression => ( number_1 => 1.0, number_0_5 => 0.5, diff --git a/src/generator/readable.rs b/src/generator/readable.rs index cfab381..c395407 100644 --- a/src/generator/readable.rs +++ b/src/generator/readable.rs @@ -693,7 +693,7 @@ impl LuaGenerator for ReadableLuaGenerator { Parenthese(parenthese) => self.write_parenthese(parenthese), String(string) => self.write_string(string), InterpolatedString(interpolated_string) => { - todo!() + self.write_interpolated_string(interpolated_string); } Table(table) => self.write_table(table), True(_) => self.push_str("true"), @@ -933,6 +933,33 @@ impl LuaGenerator for ReadableLuaGenerator { } } + fn write_interpolated_string( + &mut self, + interpolated_string: &nodes::InterpolatedStringExpression, + ) { + self.push_char('`'); + + for segment in interpolated_string.iter_segments() { + match segment { + nodes::InterpolationSegment::String(string_segment) => { + self.raw_push_str(&utils::write_interpolated_string_segment(string_segment)); + } + nodes::InterpolationSegment::Value(value) => { + self.raw_push_char('{'); + // add space when value segment is a table + let expression = value.get_expression(); + if matches!(expression, nodes::Expression::Table(_)) { + self.raw_push_char(' '); + } + self.write_expression(expression); + self.push_char('}'); + } + } + } + + self.raw_push_char('`'); + } + fn write_identifier(&mut self, identifier: &nodes::Identifier) { self.push_str(identifier.get_name()); } diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_only_letters.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_only_letters.snap new file mode 100644 index 0000000..912bd2e --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_only_letters.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`hello` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_backticks.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_backticks.snap new file mode 100644 index 0000000..fd774b2 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_backticks.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: \`Hi\`` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_double_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_double_quotes.snap new file mode 100644 index 0000000..75da430 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_double_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: "Hi"` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table.snap new file mode 100644 index 0000000..a396af7 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{ {}}` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_and_double_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_and_double_quotes.snap new file mode 100644 index 0000000..644d636 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_and_double_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: "Don't"` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_quotes.snap new file mode 100644 index 0000000..38215de --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_single_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`I'm cool` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_true_value.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_true_value.snap new file mode 100644 index 0000000..e659e5f --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_true_value.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{true}` diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__string__dense_string_with_dougle_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__string__dense_string_with_double_quotes.snap similarity index 100% rename from src/generator/snapshots/darklua_core__generator__test__dense__snapshots__string__dense_string_with_dougle_quotes.snap rename to src/generator/snapshots/darklua_core__generator__test__dense__snapshots__string__dense_string_with_double_quotes.snap diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_only_letters.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_only_letters.snap new file mode 100644 index 0000000..912bd2e --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_only_letters.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`hello` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_backticks.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_backticks.snap new file mode 100644 index 0000000..fd774b2 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_backticks.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: \`Hi\`` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_double_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_double_quotes.snap new file mode 100644 index 0000000..75da430 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_double_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: "Hi"` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table.snap new file mode 100644 index 0000000..a396af7 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{ {}}` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_and_double_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_and_double_quotes.snap new file mode 100644 index 0000000..644d636 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_and_double_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: "Don't"` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_quotes.snap new file mode 100644 index 0000000..38215de --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_single_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`I'm cool` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_true_value.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_true_value.snap new file mode 100644 index 0000000..e659e5f --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_true_value.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{true}` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__string__readable_string_with_dougle_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__string__readable_string_with_double_quotes.snap similarity index 100% rename from src/generator/snapshots/darklua_core__generator__test__readable__snapshots__string__readable_string_with_dougle_quotes.snap rename to src/generator/snapshots/darklua_core__generator__test__readable__snapshots__string__readable_string_with_double_quotes.snap diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_only_letters.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_only_letters.snap new file mode 100644 index 0000000..912bd2e --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_only_letters.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`hello` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_backticks.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_backticks.snap new file mode 100644 index 0000000..fd774b2 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_backticks.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: \`Hi\`` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_double_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_double_quotes.snap new file mode 100644 index 0000000..75da430 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_double_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: "Hi"` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table.snap new file mode 100644 index 0000000..a396af7 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{ {}}` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_and_double_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_and_double_quotes.snap new file mode 100644 index 0000000..644d636 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_and_double_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`Say: "Don't"` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_quotes.snap new file mode 100644 index 0000000..38215de --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_single_quotes.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`I'm cool` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_true_value.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_true_value.snap new file mode 100644 index 0000000..e659e5f --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_true_value.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{true}` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__string__token_based_string_with_dougle_quotes.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__string__token_based_string_with_double_quotes.snap similarity index 100% rename from src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__string__token_based_string_with_dougle_quotes.snap rename to src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__string__token_based_string_with_double_quotes.snap diff --git a/src/generator/token_based.rs b/src/generator/token_based.rs index 5684eec..22b3cb5 100644 --- a/src/generator/token_based.rs +++ b/src/generator/token_based.rs @@ -653,6 +653,68 @@ impl<'a> TokenBasedLuaGenerator<'a> { self.write_token(&tokens.end); } + fn write_interpolated_string_with_tokens( + &mut self, + interpolated_string: &InterpolatedStringExpression, + tokens: &InterpolatedStringTokens, + ) { + self.write_token(&tokens.opening_tick); + + for segment in interpolated_string.iter_segments() { + match segment { + InterpolationSegment::String(string_segment) => { + if let Some(token) = string_segment.get_token() { + self.write_token(token); + } else { + self.write_symbol(&utils::write_interpolated_string_segment(string_segment)) + } + } + InterpolationSegment::Value(value) => { + if let Some(tokens) = value.get_tokens() { + self.write_string_value_segment_with_tokens(value, tokens); + } else { + self.write_string_value_segment_with_tokens( + value, + &self.generate_string_value_segment_tokens(value), + ); + } + } + } + } + + self.write_token(&tokens.closing_tick); + } + + fn write_string_value_segment_with_tokens( + &mut self, + value: &ValueSegment, + tokens: &ValueSegmentTokens, + ) { + self.write_token(&tokens.opening_brace); + let expression = value.get_expression(); + if self.output.ends_with('{') { + match expression { + Expression::Table(table) => { + if let Some(tokens) = table.get_tokens() { + if let Some(first_trivia) = + tokens.opening_brace.iter_leading_trivia().next() + { + let trivia_str = first_trivia.read(self.original_code); + if trivia_str.is_empty() { + self.output.push(' '); + } + } + } else { + self.output.push(' '); + } + } + _ => {} + } + } + self.write_expression(expression); + self.write_token(&tokens.closing_brace); + } + fn generate_block_tokens(&self, _block: &Block) -> BlockTokens { BlockTokens { semicolons: Vec::new(), @@ -900,6 +962,26 @@ impl<'a> TokenBasedLuaGenerator<'a> { } } + fn generate_interpolated_string_tokens( + &self, + _interpolated_string: &InterpolatedStringExpression, + ) -> InterpolatedStringTokens { + InterpolatedStringTokens { + opening_tick: Token::from_content("`"), + closing_tick: Token::from_content("`"), + } + } + + fn generate_string_value_segment_tokens( + &self, + _value_segment: &ValueSegment, + ) -> ValueSegmentTokens { + ValueSegmentTokens { + opening_brace: Token::from_content("{"), + closing_brace: Token::from_content("}"), + } + } + fn write_symbol(&mut self, symbol: &str) { if self.currently_commenting { self.uncomment(); @@ -1148,7 +1230,7 @@ impl<'a> LuaGenerator for TokenBasedLuaGenerator<'a> { Parenthese(parenthese) => self.write_parenthese(parenthese), String(string) => self.write_string(string), InterpolatedString(interpolated_string) => { - todo!() + self.write_interpolated_string(interpolated_string); } Table(table) => self.write_table(table), True(token) => { @@ -1319,6 +1401,17 @@ impl<'a> LuaGenerator for TokenBasedLuaGenerator<'a> { } } + fn write_interpolated_string(&mut self, interpolated_string: &InterpolatedStringExpression) { + if let Some(tokens) = interpolated_string.get_tokens() { + self.write_interpolated_string_with_tokens(interpolated_string, tokens); + } else { + self.write_interpolated_string_with_tokens( + interpolated_string, + &self.generate_interpolated_string_tokens(interpolated_string), + ); + } + } + fn write_identifier(&mut self, identifier: &Identifier) { if let Some(token) = identifier.get_token() { let name_in_token = token.read(self.original_code); @@ -1466,6 +1559,13 @@ mod test { return_double_quote_string => "return \"ok\"", return_identifier => "return var", return_bracket_string => "return [[ [ok] ]]", + return_empty_interpolated_string => "return ``", + return_interpolated_string_escape_curly_brace => "return `Open: \\{`", + return_interpolated_string_followed_by_comment => "return `ok` -- comment", + return_interpolated_string_with_true_value => "return `{ true }`", + return_interpolated_string_with_true_value_and_prefix => "return `Result = { true }`", + return_interpolated_string_with_true_value_and_suffix => "return `{ variable } !`", + return_interpolated_string_with_various_segments => "return `Variable = { variable } ({ --[[len]] #variable })` -- display", return_empty_table => "return { }", return_table_with_field => "return { field = {} }", return_table_with_index => "return { [field] = {} }", diff --git a/src/generator/utils.rs b/src/generator/utils.rs index d3b256e..7eab3e4 100644 --- a/src/generator/utils.rs +++ b/src/generator/utils.rs @@ -3,7 +3,7 @@ use crate::nodes::{ Expression, FieldExpression, FunctionCall, IndexExpression, NumberExpression, Prefix, - Statement, StringExpression, Variable, + Statement, StringExpression, StringSegment, Variable, }; const QUOTED_STRING_MAX_LENGTH: usize = 60; @@ -266,6 +266,35 @@ pub fn write_string(string: &StringExpression) -> String { } } +pub fn write_interpolated_string_segment(segment: &StringSegment) -> String { + let value = segment.get_value(); + + if value.is_empty() { + return "".to_owned(); + } + + let mut result = String::new(); + + result.reserve(value.len()); + + for character in value.chars() { + match character { + '`' | '{' => { + result.push('\\'); + result.push(character); + } + _ if needs_escaping(character) => { + result.push_str(&escape(character)); + } + _ => { + result.push(character); + } + } + } + + result +} + fn write_long_bracket(value: &str) -> String { let mut i: usize = value.ends_with(']').into(); let mut equals = "=".repeat(i); diff --git a/src/nodes/expressions/interpolated_string.rs b/src/nodes/expressions/interpolated_string.rs index b05fdca..8ac9d2f 100644 --- a/src/nodes/expressions/interpolated_string.rs +++ b/src/nodes/expressions/interpolated_string.rs @@ -1,8 +1,8 @@ use std::iter::FromIterator; -use crate::nodes::Token; +use crate::nodes::{StringError, Token}; -use super::Expression; +use super::{string_utils, Expression}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct StringSegment { @@ -11,7 +11,11 @@ pub struct StringSegment { } impl StringSegment { - pub fn new(value: impl Into) -> Self { + pub fn new(value: impl AsRef) -> Result { + string_utils::read_escaped_string(value.as_ref().char_indices(), None).map(Self::from_value) + } + + pub fn from_value(value: impl Into) -> Self { Self { value: value.into(), token: None, @@ -42,6 +46,10 @@ impl StringSegment { token.clear_whitespaces(); } } + + pub fn get_value(&self) -> &str { + self.value.as_str() + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -82,6 +90,10 @@ impl ValueSegment { tokens.clear_whitespaces(); } } + + pub fn get_expression(&self) -> &Expression { + &self.value + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -136,6 +148,18 @@ impl From for InterpolationSegment { } } +impl From for InterpolationSegment { + fn from(value: Expression) -> Self { + Self::Value(ValueSegment::new(value)) + } +} + +impl> From for InterpolationSegment { + fn from(string: T) -> Self { + Self::String(StringSegment::from_value(string.as_ref())) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct InterpolatedStringExpression { segments: Vec, @@ -154,11 +178,20 @@ impl InterpolatedStringExpression { Self::new(Vec::default()) } + pub fn with_segment(mut self, segment: impl Into) -> Self { + self.segments.push(segment.into()); + self + } + pub fn with_tokens(mut self, tokens: InterpolatedStringTokens) -> Self { self.tokens = Some(tokens); self } + pub fn get_tokens(&self) -> Option<&InterpolatedStringTokens> { + self.tokens.as_ref() + } + pub fn set_tokens(&mut self, tokens: InterpolatedStringTokens) { self.tokens = Some(tokens); } @@ -174,6 +207,10 @@ impl InterpolatedStringExpression { tokens.clear_whitespaces(); } } + + pub fn iter_segments(&self) -> impl Iterator { + self.segments.iter() + } } impl FromIterator for InterpolatedStringExpression { diff --git a/src/nodes/expressions/mod.rs b/src/nodes/expressions/mod.rs index bc0cdbf..07c97fb 100644 --- a/src/nodes/expressions/mod.rs +++ b/src/nodes/expressions/mod.rs @@ -8,6 +8,7 @@ mod number; mod parenthese; mod prefix; mod string; +pub(crate) mod string_utils; mod table; mod unary; @@ -21,6 +22,7 @@ pub use number::*; pub use parenthese::*; pub use prefix::*; pub use string::*; +pub use string_utils::StringError; pub use table::*; pub use unary::*; diff --git a/src/nodes/expressions/string.rs b/src/nodes/expressions/string.rs index 6aa2115..a2fc2c3 100644 --- a/src/nodes/expressions/string.rs +++ b/src/nodes/expressions/string.rs @@ -1,9 +1,8 @@ -use std::{ - iter::Peekable, - str::{CharIndices, Chars}, -}; +use std::str::CharIndices; -use crate::nodes::Token; +use crate::nodes::{StringError, Token}; + +use super::string_utils; #[derive(Clone, Debug, PartialEq, Eq)] pub struct StringExpression { @@ -12,13 +11,14 @@ pub struct StringExpression { } impl StringExpression { - pub fn new(string: &str) -> Option { + pub fn new(string: &str) -> Result { if string.starts_with('[') { return string .chars() .skip(1) .enumerate() .find_map(|(indice, character)| if character == '[' { Some(indice) } else { None }) + .ok_or_else(|| StringError::invalid("unable to find `[` delimiter")) .and_then(|indice| { let length = 2 + indice; let start = if string @@ -30,120 +30,26 @@ impl StringExpression { } else { length }; - string.get(start..string.len() - length).map(str::to_owned) + string + .get(start..string.len() - length) + .map(str::to_owned) + .ok_or_else(|| StringError::invalid("")) }) - .map(|value| Self { value, token: None }); + .map(Self::from_value); } - let mut chars = string.chars().peekable(); - chars.next(); - chars.next_back(); - let mut value = String::new(); - value.reserve(string.as_bytes().len()); - - while let Some(char) = chars.next() { - if char == '\\' { - if let Some(next_char) = chars.next() { - let escaped = match next_char { - 'n' | '\n' => Some('\n'), - '"' => Some('"'), - '\'' => Some('\''), - '\\' => Some('\\'), - 't' => Some('\t'), - 'a' => Some('\u{7}'), - 'b' => Some('\u{8}'), - 'v' => Some('\u{B}'), - 'f' => Some('\u{C}'), - 'r' => Some('\r'), - first_digit if first_digit.is_ascii_digit() => { - let number = read_number(&mut chars, Some(first_digit), 10, 3); - - if number < 256 { - value.push(number as u8 as char); - } else { - // malformed string sequence: cannot escape ascii character - // with a number >= 256 - return None; - } - - None - } - 'x' => { - if let (Some(first_digit), Some(second_digit)) = ( - chars.next().filter(char::is_ascii_hexdigit), - chars.next().filter(char::is_ascii_hexdigit), - ) { - let number = 16 * first_digit.to_digit(16).unwrap() - + second_digit.to_digit(16).unwrap(); - - if number < 256 { - value.push(number as u8 as char); - } else { - unreachable!( - "malformed string sequence: cannot escape ascii character >= 256", - ); - } - } else { - // malformed string sequence: missing one or both hex digits - return None; - } - None - } - 'u' => { - if !contains(&chars.next(), &'{') { - // malformed string sequence: missing opening curly brace `{` - return None; - } - - let number = read_number(&mut chars, None, 16, 8); - - if !contains(&chars.next(), &'}') { - // malformed string sequence: missing closing curly brace `}` - return None; - } - - if number > 0x10FFFF { - // malformed string sequence: invalid unicode value (too large) - return None; - } - - value.push( - char::from_u32(number).expect("unable to convert u32 to char"), - ); + let mut chars = string.char_indices(); - None - } - 'z' => { - while chars - .peek() - .filter(|char| char.is_ascii_whitespace()) - .is_some() - { - chars.next(); - } - None - } - _ => { - // malformed string sequence: invalid character after `\` - return None; - } - }; - - if let Some(escaped) = escaped { - value.push(escaped); - } - } else { - // malformed string sequence: string ended after `\` - return None; - } - } else { - value.push(char); + match (chars.next(), chars.next_back()) { + (Some((_, first_char)), Some((_, last_char))) if first_char == last_char => { + string_utils::read_escaped_string(chars, Some(string.as_bytes().len())) + .map(Self::from_value) } + (None, None) | (None, Some(_)) | (Some(_), None) => { + Err(StringError::invalid("missing quotes")) + } + (Some(_), Some(_)) => Err(StringError::invalid("quotes do not match")), } - - value.shrink_to_fit(); - - Some(Self::from_value(value)) } pub fn empty() -> Self { @@ -232,46 +138,6 @@ impl StringExpression { } } -fn read_number( - chars: &mut Peekable, - first_digit: Option, - radix: u32, - max: usize, -) -> u32 { - let filter = match radix { - 10 => char::is_ascii_digit, - 16 => char::is_ascii_hexdigit, - _ => panic!("unsupported radix {}", radix), - }; - let mut amount = first_digit - .map(|char| char.to_digit(radix).unwrap()) - .unwrap_or(0); - let mut iteration_count: usize = first_digit.is_some().into(); - - while let Some(next_digit) = chars.peek().cloned().filter(filter) { - chars.next(); - - amount = amount * radix + next_digit.to_digit(radix).unwrap(); - iteration_count += 1; - - if iteration_count >= max { - break; - } - } - - amount -} - -fn contains(option: &Option, x: &U) -> bool -where - U: PartialEq, -{ - match option { - Some(y) => x == y, - None => false, - } -} - #[cfg(test)] mod test { use super::*; @@ -346,7 +212,7 @@ mod test { #[test] fn $name() { let quoted = format!("'{}'", $input); - assert!(StringExpression::new("ed).is_none()); + assert!(StringExpression::new("ed).is_err()); } )* } @@ -357,7 +223,7 @@ mod test { #[test] fn $name() { let quoted = format!("\"{}\"", $input); - assert!(StringExpression::new("ed).is_none()); + assert!(StringExpression::new("ed).is_err()); } )* } @@ -370,7 +236,6 @@ mod test { escaped_too_large_unicode => "\\u{110000}", escaped_missing_opening_brace_unicode => "\\uAB", escaped_missing_closing_brace_unicode => "\\u{0p", - invalid_escape => "\\o", ); #[test] @@ -415,6 +280,20 @@ mod test { assert_eq!(string.get_value(), "hello"); } + #[test] + fn new_skip_invalid_escape_in_double_quoted_string() { + let string = StringExpression::new("'\\oo'").unwrap(); + + assert_eq!(string.get_value(), "oo"); + } + + #[test] + fn new_skip_invalid_escape_in_single_quoted_string() { + let string = StringExpression::new("\"\\oo\"").unwrap(); + + assert_eq!(string.get_value(), "oo"); + } + #[test] fn has_single_quote_is_false_if_no_single_quotes() { let string = StringExpression::from_value("hello"); diff --git a/src/nodes/expressions/string_utils.rs b/src/nodes/expressions/string_utils.rs new file mode 100644 index 0000000..66d0b91 --- /dev/null +++ b/src/nodes/expressions/string_utils.rs @@ -0,0 +1,198 @@ +use std::{fmt, iter::Peekable, str::CharIndices}; + +#[derive(Debug, Clone)] +enum StringErrorKind { + Invalid { message: String }, + MalformedEscapeSequence { position: usize, message: String }, +} + +#[derive(Debug, Clone)] +pub struct StringError { + kind: StringErrorKind, +} + +impl fmt::Display for StringError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.kind { + StringErrorKind::Invalid { message } => { + write!(f, "invalid string: {}", message) + } + StringErrorKind::MalformedEscapeSequence { position, message } => { + write!(f, "malformed escape sequence at {}: {}", position, message) + } + } + } +} + +impl StringError { + pub(crate) fn invalid(message: impl Into) -> Self { + Self { + kind: StringErrorKind::Invalid { + message: message.into(), + }, + } + } + pub(crate) fn malformed_escape_sequence(position: usize, message: impl Into) -> Self { + Self { + kind: StringErrorKind::MalformedEscapeSequence { + position, + message: message.into(), + }, + } + } +} + +pub(crate) fn read_escaped_string( + chars: CharIndices, + reserve_size: Option, +) -> Result { + let mut chars = chars.peekable(); + + let mut value = String::new(); + if let Some(reserve_size) = reserve_size { + value.reserve(reserve_size); + } + + while let Some((position, char)) = chars.next() { + if char == '\\' { + if let Some((_, next_char)) = chars.next() { + match next_char { + '\n' | '"' | '\'' | '\\' => value.push(next_char), + 'n' => value.push('\n'), + 't' => value.push('\t'), + 'a' => value.push('\u{7}'), + 'b' => value.push('\u{8}'), + 'v' => value.push('\u{B}'), + 'f' => value.push('\u{C}'), + 'r' => value.push('\r'), + first_digit if first_digit.is_ascii_digit() => { + let number = read_number(&mut chars, Some(first_digit), 10, 3); + + if number < 256 { + value.push(number as u8 as char); + } else { + return Err(StringError::malformed_escape_sequence( + position, + "cannot escape ascii character greater than 256", + )); + } + } + 'x' => { + if let (Some(first_digit), Some(second_digit)) = ( + chars.next().map(|(_, c)| c).filter(char::is_ascii_hexdigit), + chars.next().map(|(_, c)| c).filter(char::is_ascii_hexdigit), + ) { + let number = 16 * first_digit.to_digit(16).unwrap() + + second_digit.to_digit(16).unwrap(); + + if number < 256 { + value.push(number as u8 as char); + } else { + return Err(StringError::malformed_escape_sequence( + position, + "cannot escape ascii character greater than 256", + )); + } + } else { + return Err(StringError::malformed_escape_sequence( + position, + "exactly two hexadecimal digit expected", + )); + } + } + 'u' => { + if !contains(&chars.next().map(|(_, c)| c), &'{') { + return Err(StringError::malformed_escape_sequence( + position, + "expected opening curly brace", + )); + } + + let number = read_number(&mut chars, None, 16, 8); + + if !contains(&chars.next().map(|(_, c)| c), &'}') { + return Err(StringError::malformed_escape_sequence( + position, + "expected closing curly brace", + )); + } + + if number > 0x10FFFF { + return Err(StringError::malformed_escape_sequence( + position, + "invalid unicode value", + )); + } + + value.push(char::from_u32(number).expect("unable to convert u32 to char")); + } + 'z' => { + while chars + .peek() + .filter(|(_, char)| char.is_ascii_whitespace()) + .is_some() + { + chars.next(); + } + } + _ => { + // an invalid escape does not error: it simply skips the backslash + value.push(next_char); + } + } + } else { + return Err(StringError::malformed_escape_sequence( + position, + "string ended after '\\'", + )); + } + } else { + value.push(char); + } + } + + value.shrink_to_fit(); + + Ok(value) +} + +fn read_number( + chars: &mut Peekable, + first_digit: Option, + radix: u32, + max: usize, +) -> u32 { + let filter = match radix { + 10 => char::is_ascii_digit, + 16 => char::is_ascii_hexdigit, + _ => panic!("unsupported radix {}", radix), + }; + let mut amount = first_digit + .map(|char| char.to_digit(radix).unwrap()) + .unwrap_or(0); + let mut iteration_count: usize = first_digit.is_some().into(); + + while let Some(next_digit) = chars.peek().map(|(_, c)| *c).filter(filter) { + chars.next(); + + amount = amount * radix + next_digit.to_digit(radix).unwrap(); + iteration_count += 1; + + if iteration_count >= max { + break; + } + } + + amount +} + +#[inline] +fn contains(option: &Option, x: &U) -> bool +where + U: PartialEq, +{ + match option { + Some(y) => x == y, + None => false, + } +} diff --git a/src/parser.rs b/src/parser.rs index 419a766..1920413 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -117,30 +117,43 @@ mod test { return_empty_double_quote_string("return \"\"") => ReturnStatement::one(StringExpression::new("\"\"").unwrap()), return_empty_backtick_string("return ``") => ReturnStatement::one(InterpolatedStringExpression::empty()), return_backtick_string_hello("return `hello`") => ReturnStatement::one(InterpolatedStringExpression::new( - vec![StringSegment::new("hello").into()] + vec![StringSegment::from_value("hello").into()] )), return_backtick_string_with_single_value("return `{true}`") => ReturnStatement::one(InterpolatedStringExpression::new( vec![ValueSegment::new(true).into()] )), return_backtick_string_with_prefixed_single_value("return `value = {true}`") => ReturnStatement::one(InterpolatedStringExpression::new( vec![ - StringSegment::new("value = ").into(), + StringSegment::from_value("value = ").into(), ValueSegment::new(true).into(), ] )), return_backtick_string_with_suffixed_single_value("return `{false} -> condition`") => ReturnStatement::one(InterpolatedStringExpression::new( vec![ ValueSegment::new(false).into(), - StringSegment::new(" -> condition").into(), + StringSegment::from_value(" -> condition").into(), ] )), return_backtick_string_with_prefix_and_suffixed_single_value("return `-> {value} (value)`") => ReturnStatement::one(InterpolatedStringExpression::new( vec![ - StringSegment::new("-> ").into(), + StringSegment::from_value("-> ").into(), ValueSegment::new(Expression::identifier("value")).into(), - StringSegment::new(" (value)").into(), + StringSegment::from_value(" (value)").into(), ] )), + return_backtick_string_escape_braces("return `Hello \\{}`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![StringSegment::from_value("Hello {}").into()] + )), + return_backtick_string_escape_backtick("return `Delimiter: \\``") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![StringSegment::from_value("Delimiter: `").into()] + )), + return_backtick_string_escape_backslash("return `\\\\`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![StringSegment::from_value("\\").into()] + )), + // todo: the test can be enabled once full-moon fixes the parse issue + // return_backtick_string_with_table_value("return `{ {} }`") => ReturnStatement::one(InterpolatedStringExpression::new( + // vec![ValueSegment::new(TableExpression::default()).into()] + // )), empty_while_true_do("while true do end") => WhileStatement::new(Block::default(), true), while_false_do_break("while false do break end") => WhileStatement::new( LastStatement::new_break(), @@ -628,7 +641,7 @@ mod test { }), return_backtick_string_hello("return `hello`") => ReturnStatement::one( InterpolatedStringExpression::new(vec![ - StringSegment::new("hello") + StringSegment::from_value("hello") .with_token(token_at_first_line(8, 13)) .into() ]) @@ -658,7 +671,7 @@ mod test { return_backtick_string_with_prefixed_single_value("return `value = {true}`") => ReturnStatement::one( InterpolatedStringExpression::new( vec![ - StringSegment::new("value = ") + StringSegment::from_value("value = ") .with_token(token_at_first_line(8, 16)) .into(), ValueSegment::new(create_true(17, 0)) @@ -684,7 +697,7 @@ mod test { opening_brace: token_at_first_line(8, 9), closing_brace: token_at_first_line(13, 14), }).into(), - StringSegment::new(" -> condition") + StringSegment::from_value(" -> condition") .with_token(token_at_first_line(14, 27)) .into(), ] @@ -700,7 +713,7 @@ mod test { return_backtick_string_with_prefix_and_suffixed_single_value("return `-> {value} (value)`") => ReturnStatement::one( InterpolatedStringExpression::new( vec![ - StringSegment::new("-> ") + StringSegment::from_value("-> ") .with_token(token_at_first_line(8, 11)) .into(), ValueSegment::new(create_identifier("value", 12, 0)) @@ -708,7 +721,7 @@ mod test { opening_brace: token_at_first_line(11, 12), closing_brace: token_at_first_line(17, 18), }).into(), - StringSegment::new(" (value)") + StringSegment::from_value(" (value)") .with_token(token_at_first_line(18, 26)) .into(), ] From 6a4e03f7697f22293bc20c5d3285475b10fe90c5 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Mon, 8 May 2023 22:27:54 -0400 Subject: [PATCH 03/14] support interpolated strings in evaluator --- src/nodes/expressions/interpolated_string.rs | 8 ++ src/process/evaluator/mod.rs | 77 +++++++++++++++++++- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/nodes/expressions/interpolated_string.rs b/src/nodes/expressions/interpolated_string.rs index 8ac9d2f..3c5bb22 100644 --- a/src/nodes/expressions/interpolated_string.rs +++ b/src/nodes/expressions/interpolated_string.rs @@ -94,6 +94,10 @@ impl ValueSegment { pub fn get_expression(&self) -> &Expression { &self.value } + + pub fn mutate_expression(&mut self) -> &mut Expression { + &mut self.value + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -211,6 +215,10 @@ impl InterpolatedStringExpression { pub fn iter_segments(&self) -> impl Iterator { self.segments.iter() } + + pub fn iter_mut_segments(&mut self) -> impl Iterator { + self.segments.iter_mut() + } } impl FromIterator for InterpolatedStringExpression { diff --git a/src/process/evaluator/mod.rs b/src/process/evaluator/mod.rs index 89a0629..5792171 100644 --- a/src/process/evaluator/mod.rs +++ b/src/process/evaluator/mod.rs @@ -38,7 +38,35 @@ impl Evaluator { } Expression::If(if_expression) => self.evaluate_if(if_expression), Expression::InterpolatedString(interpolated_string) => { - todo!() + let mut result = String::new(); + for segment in interpolated_string.iter_segments() { + match segment { + InterpolationSegment::String(string) => { + result.push_str(string.get_value()); + } + InterpolationSegment::Value(value) => { + match self.evaluate(value.get_expression()) { + LuaValue::False => { + result.push_str("false"); + } + LuaValue::True => { + result.push_str("true"); + } + LuaValue::Nil => { + result.push_str("nil"); + } + LuaValue::String(string) => { + result.push_str(&string); + } + LuaValue::Function + | LuaValue::Number(_) + | LuaValue::Table + | LuaValue::Unknown => return LuaValue::Unknown, + } + } + } + } + LuaValue::String(result) } Expression::Call(_) | Expression::Field(_) @@ -135,9 +163,14 @@ impl Evaluator { .iter() .any(|entry| self.table_entry_has_side_effects(entry)), Expression::Call(call) => self.call_has_side_effects(call), - Expression::InterpolatedString(interpolated_string) => { - todo!() - } + Expression::InterpolatedString(interpolated_string) => interpolated_string + .iter_segments() + .any(|segment| match segment { + InterpolationSegment::String(_) => false, + InterpolationSegment::Value(value) => { + self.has_side_effects(value.get_expression()) + } + }), } } @@ -436,6 +469,36 @@ mod test { nil_expression(Expression::nil()) => LuaValue::Nil, number_expression(DecimalNumber::new(0.0)) => LuaValue::Number(0.0), string_expression(StringExpression::from_value("foo")) => LuaValue::String("foo".to_owned()), + empty_interpolated_string_expression(InterpolatedStringExpression::empty()) => LuaValue::String("".to_owned()), + interpolated_string_expression_with_one_string(InterpolatedStringExpression::empty().with_segment("hello")) + => LuaValue::String("hello".to_owned()), + interpolated_string_expression_with_multiple_string_segments( + InterpolatedStringExpression::empty() + .with_segment("hello") + .with_segment("-") + .with_segment("bye") + ) => LuaValue::String("hello-bye".to_owned()), + interpolated_string_expression_with_true_segment( + InterpolatedStringExpression::empty().with_segment(Expression::from(true)) + ) => LuaValue::String("true".to_owned()), + interpolated_string_expression_with_false_segment( + InterpolatedStringExpression::empty().with_segment(Expression::from(false)) + ) => LuaValue::String("false".to_owned()), + interpolated_string_expression_with_nil_segment( + InterpolatedStringExpression::empty().with_segment(Expression::nil()) + ) => LuaValue::String("nil".to_owned()), + interpolated_string_expression_with_mixed_segments( + InterpolatedStringExpression::empty() + .with_segment("variable = ") + .with_segment(Expression::from(true)) + .with_segment("?") + ) => LuaValue::String("variable = true?".to_owned()), + interpolated_string_expression_with_mixed_segments_unknown( + InterpolatedStringExpression::empty() + .with_segment("variable = ") + .with_segment(Expression::identifier("test")) + .with_segment("!") + ) => LuaValue::Unknown, true_wrapped_in_parens(ParentheseExpression::new(true)) => LuaValue::True, false_wrapped_in_parens(ParentheseExpression::new(false)) => LuaValue::False, nil_wrapped_in_parens(ParentheseExpression::new(Expression::nil())) => LuaValue::Nil, @@ -995,6 +1058,9 @@ mod test { field_index => FieldExpression::new(Identifier::new("var"), "field"), table_value_with_call_in_entry => TableExpression::default() .append_array_value(FunctionCall::from_name("call")), + + interpolated_string_with_function_call => InterpolatedStringExpression::empty() + .with_segment(Expression::from(FunctionCall::from_name("foo"))), ); has_no_side_effects!( @@ -1004,6 +1070,9 @@ mod test { table_value => TableExpression::default(), number_value => Expression::Number(DecimalNumber::new(0.0).into()), string_value => StringExpression::from_value(""), + empty_interpolated_string_value => InterpolatedStringExpression::empty(), + interpolated_string_with_true_value => InterpolatedStringExpression::empty() + .with_segment(Expression::from(true)), identifier => Expression::identifier("foo"), identifier_in_parentheses => Expression::identifier("foo").in_parentheses(), binary_false_and_call => BinaryExpression::new( From 79fc60eb3fd73fa82607206f77420dd0b23df743 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Tue, 9 May 2023 09:51:06 -0400 Subject: [PATCH 04/14] update visitor --- src/process/node_counter.rs | 5 +++++ src/process/visitors.rs | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/process/node_counter.rs b/src/process/node_counter.rs index 5ad997f..e941703 100644 --- a/src/process/node_counter.rs +++ b/src/process/node_counter.rs @@ -21,6 +21,7 @@ pub struct NodeCounter { pub return_count: usize, pub expression_count: usize, pub variable_count: usize, + pub interpolated_string_count: usize, } impl NodeCounter { @@ -97,4 +98,8 @@ impl NodeProcessor for NodeCounter { fn process_variable_expression(&mut self, _: &mut Identifier) { self.variable_count += 1; } + + fn process_interpolated_string_expression(&mut self, _: &mut InterpolatedStringExpression) { + self.interpolated_string_count += 1; + } } diff --git a/src/process/visitors.rs b/src/process/visitors.rs index ec338e1..59cfeb4 100644 --- a/src/process/visitors.rs +++ b/src/process/visitors.rs @@ -67,7 +67,15 @@ pub trait NodeVisitor { Expression::String(string) => processor.process_string_expression(string), Expression::InterpolatedString(interpolated_string) => { processor.process_interpolated_string_expression(interpolated_string); - todo!() + + for segment in interpolated_string.iter_mut_segments() { + match segment { + InterpolationSegment::String(_) => {} + InterpolationSegment::Value(value) => { + Self::visit_expression(value.mutate_expression(), processor) + } + } + } } Expression::Table(table) => Self::visit_table(table, processor), Expression::Unary(unary) => { @@ -403,4 +411,18 @@ mod test { assert_eq!(counter.expression_count, 1); assert_eq!(counter.variable_count, 1); } + + #[test] + fn visit_interpolated_string() { + let mut counter = NodeCounter::new(); + let statement = LocalAssignStatement::from_variable("value") + .with_value(InterpolatedStringExpression::empty().with_segment(Expression::from(true))); + + let mut block = statement.into(); + + DefaultVisitor::visit_block(&mut block, &mut counter); + + assert_eq!(counter.interpolated_string_count, 1); + assert_eq!(counter.expression_count, 2); + } } From 33b188ea6578bc359a310db86a939fc5b64f262f Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Tue, 9 May 2023 10:35:11 -0400 Subject: [PATCH 05/14] implement fuzzer but comment the implementation --- src/nodes/expressions/interpolated_string.rs | 28 ++++++++- tests/fuzz.rs | 64 ++++++++++++++------ 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/nodes/expressions/interpolated_string.rs b/src/nodes/expressions/interpolated_string.rs index 3c5bb22..94a5fa4 100644 --- a/src/nodes/expressions/interpolated_string.rs +++ b/src/nodes/expressions/interpolated_string.rs @@ -183,7 +183,7 @@ impl InterpolatedStringExpression { } pub fn with_segment(mut self, segment: impl Into) -> Self { - self.segments.push(segment.into()); + self.push_segment(segment); self } @@ -219,6 +219,19 @@ impl InterpolatedStringExpression { pub fn iter_mut_segments(&mut self) -> impl Iterator { self.segments.iter_mut() } + + pub fn push_segment(&mut self, segment: impl Into) { + let new_segment = segment.into(); + match &new_segment { + InterpolationSegment::String(string_segment) => { + if string_segment.get_value().is_empty() { + return; + } + } + InterpolationSegment::Value(_) => {} + } + self.segments.push(new_segment); + } } impl FromIterator for InterpolatedStringExpression { @@ -247,3 +260,16 @@ impl InterpolatedStringTokens { self.closing_tick.clear_whitespaces(); } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn push_segment_with_empty_string_does_not_mutate() { + let mut string = InterpolatedStringExpression::empty(); + string.push_segment(""); + + pretty_assertions::assert_eq!(string, InterpolatedStringExpression::empty()); + } +} diff --git a/tests/fuzz.rs b/tests/fuzz.rs index bbe8331..37585ae 100644 --- a/tests/fuzz.rs +++ b/tests/fuzz.rs @@ -399,7 +399,7 @@ impl Fuzz for Expression { context.take_expression(); if context.can_have_expression(2) { - match thread_rng().gen_range(0, 16) { + match thread_rng().gen_range(0, 17) { 0 => true.into(), 1 => false.into(), 2 => Expression::nil(), @@ -414,11 +414,12 @@ impl Fuzz for Expression { 11 => NumberExpression::fuzz(context).into(), 12 => StringExpression::fuzz(context).into(), 13 => TableExpression::fuzz(&mut context.share_budget()).into(), - 14 => IfExpression::fuzz(context).into(), + 14 => IfExpression::fuzz(&mut context.share_budget()).into(), + // 15 => InterpolatedStringExpression::fuzz(&mut context.share_budget()).into(), _ => UnaryExpression::fuzz(context).into(), } } else { - match thread_rng().gen_range(0, 15) { + match thread_rng().gen_range(0, 11) { 0 => true.into(), 1 => false.into(), 2 => Expression::nil(), @@ -428,6 +429,7 @@ impl Fuzz for Expression { 6 => Identifier::fuzz(context).into(), 7 => NumberExpression::fuzz(context).into(), 8 => StringExpression::fuzz(context).into(), + 9 => InterpolatedStringExpression::fuzz(&mut context.share_budget()).into(), _ => TableExpression::fuzz(&mut context.share_budget()).into(), } } @@ -551,6 +553,45 @@ impl Fuzz for IfExpression { } } +fn fuzz_string_value() -> String { + let poisson = Poisson::new(3.0).unwrap(); + + let mut rng = thread_rng(); + let length: u64 = rng.sample(poisson); + + const GEN_CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\ + abcdefghijklmnopqrstuvwxyz\ + 0123456789\ + ()[]{}=<>.!?,:;+-*/%^|&#`"; + + iter::repeat(()) + .take(length as usize) + .map(|()| GEN_CHARSET[rng.gen_range(0, GEN_CHARSET.len())] as char) + .collect() +} + +impl Fuzz for InterpolatedStringExpression { + fn fuzz(context: &mut FuzzContext) -> Self { + let mut string = InterpolatedStringExpression::empty(); + + let segment_count = normal_sample(1.0, 2.0); + + let flip = if rand::random::() { 0 } else { 1 }; + + for i in 0..segment_count { + if i % 2 == flip { + string.push_segment(fuzz_string_value()); + } else { + string.push_segment(ValueSegment::new(Expression::fuzz( + &mut context.share_budget(), + ))); + } + } + + string + } +} + impl Fuzz for NumberExpression { fn fuzz(_context: &mut FuzzContext) -> Self { match thread_rng().gen_range(0, 4) { @@ -587,22 +628,7 @@ impl Fuzz for ParentheseExpression { impl Fuzz for StringExpression { fn fuzz(_context: &mut FuzzContext) -> Self { - let poisson = Poisson::new(3.0).unwrap(); - - let mut rng = thread_rng(); - let length: u64 = rng.sample(poisson); - - const GEN_CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\ - abcdefghijklmnopqrstuvwxyz\ - 0123456789\ - ()[]{}=<>.!?,:;+-*/%^|&#"; - - Self::from_value::( - iter::repeat(()) - .take(length as usize) - .map(|()| GEN_CHARSET[rng.gen_range(0, GEN_CHARSET.len())] as char) - .collect(), - ) + Self::from_value(fuzz_string_value()) } } From a1fba5494a309ac13686ac1afda6face6e6e1630 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Tue, 9 May 2023 10:38:01 -0400 Subject: [PATCH 06/14] fix style --- src/ast_converter.rs | 3 ++- src/rules/remove_comments.rs | 5 ++++- src/rules/remove_spaces.rs | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ast_converter.rs b/src/ast_converter.rs index 9362f8f..3d343e7 100644 --- a/src/ast_converter.rs +++ b/src/ast_converter.rs @@ -345,7 +345,8 @@ impl<'a> AstConverter<'a> { ); for trivia_token in literal.trailing_trivia() { - opening_brace.push_trailing_trivia(self.convert_trivia(trivia_token)?); + opening_brace + .push_trailing_trivia(self.convert_trivia(trivia_token)?); } let next_literal = segments_iter diff --git a/src/rules/remove_comments.rs b/src/rules/remove_comments.rs index 9fbe18a..69ccc55 100644 --- a/src/rules/remove_comments.rs +++ b/src/rules/remove_comments.rs @@ -144,7 +144,10 @@ impl NodeProcessor for RemoveCommentProcessor { unary.clear_comments(); } - fn process_interpolated_string_expression(&mut self, string: &mut InterpolatedStringExpression) { + fn process_interpolated_string_expression( + &mut self, + string: &mut InterpolatedStringExpression, + ) { string.clear_comments(); } diff --git a/src/rules/remove_spaces.rs b/src/rules/remove_spaces.rs index db7eab2..6f14290 100644 --- a/src/rules/remove_spaces.rs +++ b/src/rules/remove_spaces.rs @@ -144,7 +144,10 @@ impl NodeProcessor for RemoveWhitespacesProcessor { unary.clear_whitespaces(); } - fn process_interpolated_string_expression(&mut self, string: &mut InterpolatedStringExpression) { + fn process_interpolated_string_expression( + &mut self, + string: &mut InterpolatedStringExpression, + ) { string.clear_whitespaces(); } From 08dd661a39653a8d68d94e2ee155d6c327cc2e09 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Tue, 9 May 2023 11:16:44 -0400 Subject: [PATCH 07/14] comment fuzz test --- tests/fuzz.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzz.rs b/tests/fuzz.rs index 37585ae..820816b 100644 --- a/tests/fuzz.rs +++ b/tests/fuzz.rs @@ -429,7 +429,7 @@ impl Fuzz for Expression { 6 => Identifier::fuzz(context).into(), 7 => NumberExpression::fuzz(context).into(), 8 => StringExpression::fuzz(context).into(), - 9 => InterpolatedStringExpression::fuzz(&mut context.share_budget()).into(), + // 9 => InterpolatedStringExpression::fuzz(&mut context.share_budget()).into(), _ => TableExpression::fuzz(&mut context.share_budget()).into(), } } From 82b263a25feaf024f48ccdc310e4d986b7f992ab Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Tue, 9 May 2023 11:16:55 -0400 Subject: [PATCH 08/14] fix clippy --- src/generator/token_based.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/generator/token_based.rs b/src/generator/token_based.rs index 22b3cb5..2ddb9c9 100644 --- a/src/generator/token_based.rs +++ b/src/generator/token_based.rs @@ -693,22 +693,17 @@ impl<'a> TokenBasedLuaGenerator<'a> { self.write_token(&tokens.opening_brace); let expression = value.get_expression(); if self.output.ends_with('{') { - match expression { - Expression::Table(table) => { - if let Some(tokens) = table.get_tokens() { - if let Some(first_trivia) = - tokens.opening_brace.iter_leading_trivia().next() - { - let trivia_str = first_trivia.read(self.original_code); - if trivia_str.is_empty() { - self.output.push(' '); - } + if let Expression::Table(table) = expression { + if let Some(tokens) = table.get_tokens() { + if let Some(first_trivia) = tokens.opening_brace.iter_leading_trivia().next() { + let trivia_str = first_trivia.read(self.original_code); + if trivia_str.is_empty() { + self.output.push(' '); } - } else { - self.output.push(' '); } + } else { + self.output.push(' '); } - _ => {} } } self.write_expression(expression); From 4338ff9408d56a666aa732dbed1b651d4eeb37a5 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Thu, 23 Nov 2023 10:30:09 -0500 Subject: [PATCH 09/14] adjust string interpolation to recent darklua changes --- src/ast_converter.rs | 4 +- src/nodes/expressions/interpolated_string.rs | 111 ++++++++++++++++-- .../path_require_mode/module_definitions.rs | 3 + src/rules/replace_referenced_tokens.rs | 8 ++ src/rules/shift_token_line.rs | 8 ++ tests/ast_fuzzer/fuzzer_work.rs | 3 + tests/ast_fuzzer/mod.rs | 32 +++++ tests/ast_fuzzer/random.rs | 17 +++ tests/bundle.rs | 4 +- 9 files changed, 177 insertions(+), 13 deletions(-) diff --git a/src/ast_converter.rs b/src/ast_converter.rs index ce95de9..fa51da7 100644 --- a/src/ast_converter.rs +++ b/src/ast_converter.rs @@ -2490,7 +2490,7 @@ impl<'a> AstConverter<'a> { string: &tokenizer::TokenReference, ) -> Result { let mut expression = - StringExpression::new(&string.token().to_string()).ok_or_else(|| { + StringExpression::new(&string.token().to_string()).map_err(|_err| { ConvertError::String { string: string.to_string(), } @@ -2508,7 +2508,7 @@ impl<'a> AstConverter<'a> { string: &tokenizer::TokenReference, ) -> Result { let mut expression = - StringType::new(&string.token().to_string()).ok_or_else(|| ConvertError::String { + StringType::new(&string.token().to_string()).map_err(|_err| ConvertError::String { string: string.to_string(), })?; if self.hold_token_data { diff --git a/src/nodes/expressions/interpolated_string.rs b/src/nodes/expressions/interpolated_string.rs index 94a5fa4..05b8240 100644 --- a/src/nodes/expressions/interpolated_string.rs +++ b/src/nodes/expressions/interpolated_string.rs @@ -35,6 +35,15 @@ impl StringSegment { self.token.as_ref() } + pub fn get_value(&self) -> &str { + self.value.as_str() + } + + fn append(&mut self, mut other: Self) { + self.value.extend(other.value.drain(..)); + self.token = None; + } + pub fn clear_comments(&mut self) { if let Some(token) = &mut self.token { token.clear_comments(); @@ -47,8 +56,16 @@ impl StringSegment { } } - pub fn get_value(&self) -> &str { - self.value.as_str() + pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { + if let Some(token) = &mut self.token { + token.replace_referenced_tokens(code); + } + } + + pub(crate) fn shift_token_line(&mut self, amount: usize) { + if let Some(token) = &mut self.token { + token.shift_token_line(amount); + } } } @@ -79,6 +96,14 @@ impl ValueSegment { self.tokens.as_ref() } + pub fn get_expression(&self) -> &Expression { + &self.value + } + + pub fn mutate_expression(&mut self) -> &mut Expression { + &mut self.value + } + pub fn clear_comments(&mut self) { if let Some(tokens) = &mut self.tokens { tokens.clear_comments(); @@ -91,12 +116,16 @@ impl ValueSegment { } } - pub fn get_expression(&self) -> &Expression { - &self.value + pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { + if let Some(tokens) = &mut self.tokens { + tokens.replace_referenced_tokens(code); + } } - pub fn mutate_expression(&mut self) -> &mut Expression { - &mut self.value + pub(crate) fn shift_token_line(&mut self, amount: usize) { + if let Some(tokens) = &mut self.tokens { + tokens.shift_token_line(amount); + } } } @@ -116,6 +145,16 @@ impl ValueSegmentTokens { self.opening_brace.clear_whitespaces(); self.closing_brace.clear_whitespaces(); } + + pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { + self.opening_brace.replace_referenced_tokens(code); + self.closing_brace.replace_referenced_tokens(code); + } + + pub(crate) fn shift_token_line(&mut self, amount: usize) { + self.opening_brace.shift_token_line(amount); + self.closing_brace.shift_token_line(amount); + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -138,6 +177,20 @@ impl InterpolationSegment { InterpolationSegment::Value(segment) => segment.clear_whitespaces(), } } + + pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { + match self { + InterpolationSegment::String(segment) => segment.replace_referenced_tokens(code), + InterpolationSegment::Value(segment) => segment.replace_referenced_tokens(code), + } + } + + pub(crate) fn shift_token_line(&mut self, amount: usize) { + match self { + InterpolationSegment::String(segment) => segment.shift_token_line(amount), + InterpolationSegment::Value(segment) => segment.shift_token_line(amount), + } + } } impl From for InterpolationSegment { @@ -204,12 +257,36 @@ impl InterpolatedStringExpression { if let Some(tokens) = &mut self.tokens { tokens.clear_comments(); } + for segment in &mut self.segments { + segment.clear_comments(); + } } pub fn clear_whitespaces(&mut self) { if let Some(tokens) = &mut self.tokens { tokens.clear_whitespaces(); } + for segment in &mut self.segments { + segment.clear_whitespaces(); + } + } + + pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { + if let Some(tokens) = &mut self.tokens { + tokens.replace_referenced_tokens(code); + } + for segment in &mut self.segments { + segment.replace_referenced_tokens(code); + } + } + + pub(crate) fn shift_token_line(&mut self, amount: usize) { + if let Some(tokens) = &mut self.tokens { + tokens.shift_token_line(amount); + } + for segment in &mut self.segments { + segment.shift_token_line(amount); + } } pub fn iter_segments(&self) -> impl Iterator { @@ -222,15 +299,21 @@ impl InterpolatedStringExpression { pub fn push_segment(&mut self, segment: impl Into) { let new_segment = segment.into(); - match &new_segment { + match new_segment { InterpolationSegment::String(string_segment) => { if string_segment.get_value().is_empty() { return; } + if let Some(InterpolationSegment::String(last)) = self.segments.last_mut() { + last.append(string_segment); + } else { + self.segments.push(string_segment.into()); + } + } + InterpolationSegment::Value(_) => { + self.segments.push(new_segment); } - InterpolationSegment::Value(_) => {} } - self.segments.push(new_segment); } } @@ -259,6 +342,16 @@ impl InterpolatedStringTokens { self.opening_tick.clear_whitespaces(); self.closing_tick.clear_whitespaces(); } + + pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { + self.opening_tick.replace_referenced_tokens(code); + self.closing_tick.replace_referenced_tokens(code); + } + + pub(crate) fn shift_token_line(&mut self, amount: usize) { + self.opening_tick.shift_token_line(amount); + self.closing_tick.shift_token_line(amount); + } } #[cfg(test)] diff --git a/src/rules/bundle/path_require_mode/module_definitions.rs b/src/rules/bundle/path_require_mode/module_definitions.rs index 34913fd..8c529e4 100644 --- a/src/rules/bundle/path_require_mode/module_definitions.rs +++ b/src/rules/bundle/path_require_mode/module_definitions.rs @@ -324,6 +324,9 @@ fn last_expression_token(expression: &Expression) -> Option<&Token> { .get_tokens() .map(|tokens| &tokens.right_parenthese), Expression::String(string) => string.get_token(), + Expression::InterpolatedString(string) => { + string.get_tokens().map(|tokens| &tokens.closing_tick) + } Expression::Table(table) => table.get_tokens().map(|tokens| &tokens.closing_brace), Expression::Nil(token) | Expression::False(token) diff --git a/src/rules/replace_referenced_tokens.rs b/src/rules/replace_referenced_tokens.rs index 366c8ef..30493c9 100644 --- a/src/rules/replace_referenced_tokens.rs +++ b/src/rules/replace_referenced_tokens.rs @@ -106,6 +106,7 @@ impl<'a> NodeProcessor for Processor<'a> { | Expression::Number(_) | Expression::Parenthese(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::Unary(_) | Expression::TypeCast(_) => {} @@ -148,6 +149,13 @@ impl<'a> NodeProcessor for Processor<'a> { string.replace_referenced_tokens(self.code); } + fn process_interpolated_string_expression( + &mut self, + string: &mut InterpolatedStringExpression, + ) { + string.replace_referenced_tokens(self.code); + } + fn process_table_expression(&mut self, table: &mut TableExpression) { table.replace_referenced_tokens(self.code); } diff --git a/src/rules/shift_token_line.rs b/src/rules/shift_token_line.rs index 891a1bf..6818c52 100644 --- a/src/rules/shift_token_line.rs +++ b/src/rules/shift_token_line.rs @@ -106,6 +106,7 @@ impl NodeProcessor for Processor { | Expression::Number(_) | Expression::Parenthese(_) | Expression::String(_) + | Expression::InterpolatedString(_) | Expression::Table(_) | Expression::Unary(_) | Expression::TypeCast(_) => {} @@ -148,6 +149,13 @@ impl NodeProcessor for Processor { string.shift_token_line(self.shift_amount); } + fn process_interpolated_string_expression( + &mut self, + string: &mut InterpolatedStringExpression, + ) { + string.shift_token_line(self.shift_amount); + } + fn process_table_expression(&mut self, table: &mut TableExpression) { table.shift_token_line(self.shift_amount); } diff --git a/tests/ast_fuzzer/fuzzer_work.rs b/tests/ast_fuzzer/fuzzer_work.rs index 4bf4951..67c161a 100644 --- a/tests/ast_fuzzer/fuzzer_work.rs +++ b/tests/ast_fuzzer/fuzzer_work.rs @@ -62,6 +62,9 @@ pub enum AstFuzzerWork { MakeParentheseExpression, MakeUnaryExpression, MakeTypeCastExpression, + MakeInterpolatedString { + segment_is_expression: Vec, + }, MakeFunctionExpression { parameters: usize, has_return_type: bool, diff --git a/tests/ast_fuzzer/mod.rs b/tests/ast_fuzzer/mod.rs index 7f3d145..8014f42 100644 --- a/tests/ast_fuzzer/mod.rs +++ b/tests/ast_fuzzer/mod.rs @@ -530,6 +530,23 @@ impl AstFuzzer { self.push_work(AstFuzzerWork::MakeTableExpression); self.push_work(AstFuzzerWork::FuzzTable); } + 16 => { + let length = self.random.interpolated_string_segments(); + let segment_is_expression: Vec<_> = iter::repeat_with(|| { + self.random.interpolated_segment_is_expression() + && self.budget.take_expression() + }) + .take(length) + .collect(); + + let expression_count = + segment_is_expression.iter().filter(|v| **v).count(); + + self.push_work(AstFuzzerWork::MakeInterpolatedString { + segment_is_expression, + }); + self.fuzz_multiple_nested_expression(depth, expression_count); + } _ => { self.budget.try_take_expressions(1); @@ -1254,6 +1271,21 @@ impl AstFuzzer { UnaryExpression::new(self.random.unary_operator(), expression).into(), ); } + AstFuzzerWork::MakeInterpolatedString { + segment_is_expression, + } => { + let mut string = InterpolatedStringExpression::empty(); + + for is_expression in segment_is_expression { + if is_expression { + string.push_segment(self.pop_expression()); + } else { + string.push_segment(self.random.string_content()); + } + } + + self.expressions.push(string.into()); + } AstFuzzerWork::MakeTypeCastExpression => { let mut expression = self.pop_expression(); diff --git a/tests/ast_fuzzer/random.rs b/tests/ast_fuzzer/random.rs index f7c4773..edabe75 100644 --- a/tests/ast_fuzzer/random.rs +++ b/tests/ast_fuzzer/random.rs @@ -21,6 +21,9 @@ pub struct RandomAst { return_length_std_dev: f64, numeric_for_step_prob: f64, function_type_argument_name_prob: f64, + interpolated_string_segments_mean: f64, + interpolated_string_segments_std_def: f64, + interpolated_segment_is_expression_prob: f64, } impl Default for RandomAst { @@ -43,6 +46,9 @@ impl Default for RandomAst { return_length_std_dev: 2.5, numeric_for_step_prob: 0.3, function_type_argument_name_prob: 0.4, + interpolated_string_segments_mean: 1.5, + interpolated_string_segments_std_def: 2.5, + interpolated_segment_is_expression_prob: 0.5, } } } @@ -94,6 +100,17 @@ impl RandomAst { generate_string_content(3.0) } + pub fn interpolated_string_segments(&self) -> usize { + 1 + normal_sample( + self.interpolated_string_segments_mean, + self.interpolated_string_segments_std_def, + ) + } + + pub fn interpolated_segment_is_expression(&self) -> bool { + thread_rng().gen_bool(self.interpolated_segment_is_expression_prob) + } + pub fn table_length(&self) -> usize { normal_sample(self.table_mean, self.table_std_dev) } diff --git a/tests/bundle.rs b/tests/bundle.rs index 255a0e5..cf25c6d 100644 --- a/tests/bundle.rs +++ b/tests/bundle.rs @@ -529,10 +529,10 @@ data: result .map_err(|err| { - std::fs::write("fuzz_bundle_failure.repro.txt", block_file).unwrap(); + std::fs::write("fuzz_bundle_failure.repro.lua", block_file).unwrap(); let out = resources.get("out.lua").unwrap(); - std::fs::write("fuzz_bundle_failure.txt", out).unwrap(); + std::fs::write("fuzz_bundle_failure.lua", out).unwrap(); err }) From b6c1faee26976702291aad4808912cf57e997190 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Thu, 23 Nov 2023 10:48:38 -0500 Subject: [PATCH 10/14] upgrade full-moon --- Cargo.lock | 4 +- Cargo.toml | 2 +- src/ast_converter.rs | 212 ++++++++++++++++++++----------------------- src/parser.rs | 7 +- 4 files changed, 105 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3983d68..4399d6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -577,9 +577,9 @@ dependencies = [ [[package]] name = "full_moon" -version = "0.18.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b9a9bf5e42aec08f4b59be1438d66b01ab0a0f51dca309626e219697b60871c" +checksum = "24ef4f8ad0689d3a86bb483650422d72e6f79a37fdc83ed5426cafe96b776ce1" dependencies = [ "bytecount", "cfg-if 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index 9cc2ea5..57650cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ clap = { version = "4.4.6", features = ["derive"] } durationfmt = "0.1.1" elsa = "1.9.0" env_logger = "0.10.0" -full_moon = { version = "0.18.1", features = ["roblox"] } +full_moon = { version = "0.19.0", features = ["roblox"] } json5 = "0.4.1" log = "0.4.20" pathdiff = "0.2.1" diff --git a/src/ast_converter.rs b/src/ast_converter.rs index fa51da7..f57cc7a 100644 --- a/src/ast_converter.rs +++ b/src/ast_converter.rs @@ -219,7 +219,7 @@ impl<'a> AstConverter<'a> { ast::Prefix::Expression(expression) => { self.work_stack .push(ConvertWork::MakePrefixFromExpression { prefix }); - self.push_work(expression); + self.push_work(expression.as_ref()); } ast::Prefix::Name(name) => { self.prefixes @@ -1789,129 +1789,115 @@ impl<'a> AstConverter<'a> { .push(ConvertWork::MakeUnaryExpression { operator: unop }); self.work_stack.push(ConvertWork::Expression(expression)); } - ast::Expression::Value { - value, + ast::Expression::TypeAssertion { + expression, type_assertion, } => { - if let Some(type_assertion) = type_assertion { - self.work_stack - .push(ConvertWork::MakeTypeCast { type_assertion }); - self.push_work(type_assertion.cast_to()); - } - match value.as_ref() { - ast::Value::Function((token, body)) => { - self.work_stack - .push(ConvertWork::MakeFunctionExpression { body, token }); + self.work_stack + .push(ConvertWork::MakeTypeCast { type_assertion }); + self.push_work(type_assertion.cast_to()); + self.push_work(expression.as_ref()); + } + ast::Expression::Function((token, body)) => { + self.work_stack + .push(ConvertWork::MakeFunctionExpression { body, token }); - self.push_function_body_work(body); - } - ast::Value::FunctionCall(call) => { - self.work_stack - .push(ConvertWork::MakeFunctionCallExpression { call }); - self.convert_function_call(call)?; - } - ast::Value::TableConstructor(table) => { - self.work_stack - .push(ConvertWork::MakeTableExpression { table }); - self.convert_table(table)?; - } - ast::Value::Number(number) => { - let mut expression = NumberExpression::from_str( - &number.token().to_string(), - ) - .map_err(|err| ConvertError::Number { - number: number.to_string(), - parsing_error: err.to_string(), - })?; - if self.hold_token_data { - expression.set_token(self.convert_token(number)?); - } - self.work_stack - .push(ConvertWork::PushExpression(expression.into())); - } - ast::Value::ParenthesesExpression(expression) => { - self.push_work(expression); - } - ast::Value::String(token_ref) => { - self.work_stack.push(ConvertWork::PushExpression( - self.convert_string_expression(token_ref)?.into(), - )); - } - ast::Value::Symbol(symbol_token) => match symbol_token.token().token_type() { - TokenType::Symbol { symbol } => { - let token = if self.hold_token_data { - Some(self.convert_token(symbol_token)?) - } else { - None - }; - let expression = match symbol { - Symbol::True => Expression::True(token), - Symbol::False => Expression::False(token), - Symbol::Nil => Expression::Nil(token), - Symbol::Ellipse => Expression::VariableArguments(token), - _ => { - return Err(ConvertError::Expression { - expression: expression.to_string(), - }) - } - }; - self.work_stack - .push(ConvertWork::PushExpression(expression)); - } - _ => { - return Err(ConvertError::Expression { - expression: expression.to_string(), - }) - } - }, - ast::Value::Var(var) => match var { - ast::Var::Expression(var_expression) => { - self.work_stack.push(ConvertWork::MakePrefixExpression { - variable: var_expression, - }); - self.push_work(var_expression.prefix()); - self.convert_suffixes(var_expression.suffixes())?; - } - ast::Var::Name(token_ref) => { - self.work_stack.push(ConvertWork::PushExpression( - Expression::Identifier( - self.convert_token_to_identifier(token_ref)?, - ), - )); - } + self.push_function_body_work(body); + } + ast::Expression::FunctionCall(call) => { + self.work_stack + .push(ConvertWork::MakeFunctionCallExpression { call }); + self.convert_function_call(call)?; + } + ast::Expression::TableConstructor(table) => { + self.work_stack + .push(ConvertWork::MakeTableExpression { table }); + self.convert_table(table)?; + } + ast::Expression::Number(number) => { + let mut expression = NumberExpression::from_str(&number.token().to_string()) + .map_err(|err| ConvertError::Number { + number: number.to_string(), + parsing_error: err.to_string(), + })?; + if self.hold_token_data { + expression.set_token(self.convert_token(number)?); + } + self.work_stack + .push(ConvertWork::PushExpression(expression.into())); + } + ast::Expression::String(token_ref) => { + self.work_stack.push(ConvertWork::PushExpression( + self.convert_string_expression(token_ref)?.into(), + )); + } + ast::Expression::Symbol(symbol_token) => match symbol_token.token().token_type() { + TokenType::Symbol { symbol } => { + let token = if self.hold_token_data { + Some(self.convert_token(symbol_token)?) + } else { + None + }; + let expression = match symbol { + Symbol::True => Expression::True(token), + Symbol::False => Expression::False(token), + Symbol::Nil => Expression::Nil(token), + Symbol::Ellipse => Expression::VariableArguments(token), _ => { return Err(ConvertError::Expression { expression: expression.to_string(), }) } - }, - ast::Value::IfExpression(if_expression) => { - self.push_work(ConvertWork::MakeIfExpression { if_expression }); - self.push_work(if_expression.condition()); - self.push_work(if_expression.if_expression()); - self.push_work(if_expression.else_expression()); - if let Some(elseif_expressions) = if_expression.else_if_expressions() { - for elseif in elseif_expressions { - self.push_work(elseif.condition()); - self.push_work(elseif.expression()); - } - } - } - ast::Value::InterpolatedString(interpolated_string) => { - self.push_work(ConvertWork::MakeInterpolatedString { - interpolated_string, - }); - for segment in interpolated_string.segments() { - self.push_work(&segment.expression); - } - } - _ => { - return Err(ConvertError::Expression { - expression: expression.to_string(), - }) + }; + self.work_stack + .push(ConvertWork::PushExpression(expression)); + } + _ => { + return Err(ConvertError::Expression { + expression: expression.to_string(), + }) + } + }, + ast::Expression::Var(var) => match var { + ast::Var::Expression(var_expression) => { + self.work_stack.push(ConvertWork::MakePrefixExpression { + variable: var_expression, + }); + self.push_work(var_expression.prefix()); + self.convert_suffixes(var_expression.suffixes())?; + } + ast::Var::Name(token_ref) => { + self.work_stack + .push(ConvertWork::PushExpression(Expression::Identifier( + self.convert_token_to_identifier(token_ref)?, + ))); + } + _ => { + return Err(ConvertError::Expression { + expression: expression.to_string(), + }) + } + }, + ast::Expression::IfExpression(if_expression) => { + self.push_work(ConvertWork::MakeIfExpression { if_expression }); + self.push_work(if_expression.condition()); + self.push_work(if_expression.if_expression()); + self.push_work(if_expression.else_expression()); + if let Some(elseif_expressions) = if_expression.else_if_expressions() { + for elseif in elseif_expressions { + self.push_work(elseif.condition()); + self.push_work(elseif.expression()); } } } + ast::Expression::InterpolatedString(interpolated_string) => { + self.push_work(ConvertWork::MakeInterpolatedString { + interpolated_string, + }); + for segment in interpolated_string.segments() { + self.push_work(&segment.expression); + } + } _ => { return Err(ConvertError::Expression { expression: expression.to_string(), diff --git a/src/parser.rs b/src/parser.rs index b55220a..773010f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -155,10 +155,9 @@ mod test { return_backtick_string_escape_backslash("return `\\\\`") => ReturnStatement::one(InterpolatedStringExpression::new( vec![StringSegment::from_value("\\").into()] )), - // todo: the test can be enabled once full-moon fixes the parse issue - // return_backtick_string_with_table_value("return `{ {} }`") => ReturnStatement::one(InterpolatedStringExpression::new( - // vec![ValueSegment::new(TableExpression::default()).into()] - // )), + return_backtick_string_with_table_value("return `{ {} }`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![ValueSegment::new(TableExpression::default()).into()] + )), empty_while_true_do("while true do end") => WhileStatement::new(Block::default(), true), while_false_do_break("while false do break end") => WhileStatement::new( LastStatement::new_break(), From 1955d29a4c6f6f239959a9352f69607b2ca88539 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Thu, 23 Nov 2023 14:55:05 -0500 Subject: [PATCH 11/14] Fix generators to add space before writing tables inside interpolated strings --- src/generator/dense.rs | 2 +- src/generator/mod.rs | 6 ++-- src/generator/readable.rs | 2 +- ..._string_with_empty_table_in_type_cast.snap | 5 +++ ..._string_with_empty_table_in_type_cast.snap | 5 +++ ..._string_with_empty_table_in_type_cast.snap | 5 +++ src/generator/token_based.rs | 21 +++++++------ src/generator/utils.rs | 31 ++++++++++++++++++- src/nodes/expressions/interpolated_string.rs | 24 ++++++++++---- src/parser.rs | 5 +++ 10 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table_in_type_cast.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table_in_type_cast.snap create mode 100644 src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table_in_type_cast.snap diff --git a/src/generator/dense.rs b/src/generator/dense.rs index 44bf927..079ef77 100644 --- a/src/generator/dense.rs +++ b/src/generator/dense.rs @@ -900,7 +900,7 @@ impl LuaGenerator for DenseLuaGenerator { self.raw_push_char('{'); // add space when value segment is a table let expression = value.get_expression(); - if matches!(expression, nodes::Expression::Table(_)) { + if utils::starts_with_table(expression).is_some() { self.raw_push_char(' '); } self.write_expression(expression); diff --git a/src/generator/mod.rs b/src/generator/mod.rs index 00ab165..cf09dc5 100644 --- a/src/generator/mod.rs +++ b/src/generator/mod.rs @@ -1008,9 +1008,11 @@ mod $mod_name { with_single_and_double_quotes => InterpolatedStringExpression::empty() .with_segment(r#"Say: "Don't""#), with_true_value => InterpolatedStringExpression::empty() - .with_segment(Expression::from(true)), + .with_segment(true), with_empty_table => InterpolatedStringExpression::empty() - .with_segment(Expression::from(TableExpression::default())), + .with_segment(TableExpression::default()), + with_empty_table_in_type_cast => InterpolatedStringExpression::empty() + .with_segment(TypeCastExpression::new(TableExpression::default(), TypeName::new("any"))), )); snapshot_node!($mod_name, $generator, number, write_expression => ( diff --git a/src/generator/readable.rs b/src/generator/readable.rs index f372f2d..7d69826 100644 --- a/src/generator/readable.rs +++ b/src/generator/readable.rs @@ -1128,7 +1128,7 @@ impl LuaGenerator for ReadableLuaGenerator { self.raw_push_char('{'); // add space when value segment is a table let expression = value.get_expression(); - if matches!(expression, nodes::Expression::Table(_)) { + if utils::starts_with_table(expression).is_some() { self.raw_push_char(' '); } self.write_expression(expression); diff --git a/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table_in_type_cast.snap b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table_in_type_cast.snap new file mode 100644 index 0000000..a572516 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__dense__snapshots__interpolated_string__dense_interpolated_string_with_empty_table_in_type_cast.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{ {}::any}` diff --git a/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table_in_type_cast.snap b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table_in_type_cast.snap new file mode 100644 index 0000000..a572516 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__readable__snapshots__interpolated_string__readable_interpolated_string_with_empty_table_in_type_cast.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{ {}::any}` diff --git a/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table_in_type_cast.snap b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table_in_type_cast.snap new file mode 100644 index 0000000..a572516 --- /dev/null +++ b/src/generator/snapshots/darklua_core__generator__test__token_based__snapshots__interpolated_string__token_based_interpolated_string_with_empty_table_in_type_cast.snap @@ -0,0 +1,5 @@ +--- +source: src/generator/mod.rs +expression: generator.into_string() +--- +`{ {}::any}` diff --git a/src/generator/token_based.rs b/src/generator/token_based.rs index 196c64a..b5744ce 100644 --- a/src/generator/token_based.rs +++ b/src/generator/token_based.rs @@ -1110,15 +1110,18 @@ impl<'a> TokenBasedLuaGenerator<'a> { self.write_token(&tokens.opening_brace); let expression = value.get_expression(); if self.output.ends_with('{') { - if let Expression::Table(table) = expression { - if let Some(tokens) = table.get_tokens() { - if let Some(first_trivia) = tokens.opening_brace.iter_leading_trivia().next() { - let trivia_str = first_trivia.read(self.original_code); - if trivia_str.is_empty() { - self.output.push(' '); - } - } - } else { + if let Some(table) = utils::starts_with_table(expression) { + if table + .get_tokens() + .and_then(|tokens| { + tokens + .opening_brace + .iter_leading_trivia() + .next() + .filter(|trivia| !trivia.read(self.original_code).is_empty()) + }) + .is_none() + { self.output.push(' '); } } diff --git a/src/generator/utils.rs b/src/generator/utils.rs index 146ff4a..cb83c66 100644 --- a/src/generator/utils.rs +++ b/src/generator/utils.rs @@ -3,7 +3,7 @@ use crate::nodes::{ Expression, FieldExpression, FunctionCall, IndexExpression, NumberExpression, Prefix, - Statement, StringSegment, Variable, + Statement, StringSegment, TableExpression, Variable, }; const QUOTED_STRING_MAX_LENGTH: usize = 60; @@ -93,6 +93,35 @@ pub fn ends_with_prefix(statement: &Statement) -> bool { } } +pub fn starts_with_table(mut expression: &Expression) -> Option<&TableExpression> { + loop { + match expression { + Expression::Table(table) => break Some(table), + Expression::Binary(binary) => { + expression = binary.left(); + } + Expression::Call(_) + | Expression::False(_) + | Expression::Field(_) + | Expression::Function(_) + | Expression::Identifier(_) + | Expression::If(_) + | Expression::Index(_) + | Expression::Nil(_) + | Expression::Number(_) + | Expression::Parenthese(_) + | Expression::String(_) + | Expression::InterpolatedString(_) + | Expression::True(_) + | Expression::Unary(_) + | Expression::VariableArguments(_) => break None, + Expression::TypeCast(type_cast) => { + expression = type_cast.get_expression(); + } + } + } +} + pub fn starts_with_parenthese(statement: &Statement) -> bool { match statement { Statement::Assign(assign) => { diff --git a/src/nodes/expressions/interpolated_string.rs b/src/nodes/expressions/interpolated_string.rs index 05b8240..04de8b6 100644 --- a/src/nodes/expressions/interpolated_string.rs +++ b/src/nodes/expressions/interpolated_string.rs @@ -205,15 +205,27 @@ impl From for InterpolationSegment { } } -impl From for InterpolationSegment { - fn from(value: Expression) -> Self { - Self::Value(ValueSegment::new(value)) +impl> From for InterpolationSegment { + fn from(value: T) -> Self { + Self::Value(ValueSegment::new(value.into())) } } -impl> From for InterpolationSegment { - fn from(string: T) -> Self { - Self::String(StringSegment::from_value(string.as_ref())) +impl From<&str> for InterpolationSegment { + fn from(string: &str) -> Self { + Self::String(StringSegment::from_value(string)) + } +} + +impl From<&String> for InterpolationSegment { + fn from(string: &String) -> Self { + Self::String(StringSegment::from_value(string)) + } +} + +impl From for InterpolationSegment { + fn from(string: String) -> Self { + Self::String(StringSegment::from_value(string)) } } diff --git a/src/parser.rs b/src/parser.rs index 773010f..fc6abb7 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -158,6 +158,11 @@ mod test { return_backtick_string_with_table_value("return `{ {} }`") => ReturnStatement::one(InterpolatedStringExpression::new( vec![ValueSegment::new(TableExpression::default()).into()] )), + return_backtick_string_with_backtrick_string_value("return `{`a`}`") => ReturnStatement::one(InterpolatedStringExpression::new( + vec![ValueSegment::new( + InterpolatedStringExpression::new(vec![StringSegment::from_value("a").into()]) + ).into()] + )), empty_while_true_do("while true do end") => WhileStatement::new(Block::default(), true), while_false_do_break("while false do break end") => WhileStatement::new( LastStatement::new_break(), From 76466a21f77e844f2fb6336a1da2a0377cf1b30f Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Thu, 23 Nov 2023 14:56:56 -0500 Subject: [PATCH 12/14] add entry to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48636f5..7e32bbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* add support for Luau interpolated strings ([#94](https://github.com/seaofvoices/darklua/pull/94)) * add rule to append text comments ([#141](https://github.com/seaofvoices/darklua/pull/141)) ## 0.11.3 From 874dff7c08dab332f4fd06b02496ce4f70cc08b7 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Thu, 23 Nov 2023 15:59:39 -0500 Subject: [PATCH 13/14] remove unnecessary comments --- src/generator/readable.rs | 7 ----- src/nodes/types/table.rs | 20 -------------- src/nodes/types/type_name.rs | 52 ------------------------------------ 3 files changed, 79 deletions(-) diff --git a/src/generator/readable.rs b/src/generator/readable.rs index 7d69826..e8b6714 100644 --- a/src/generator/readable.rs +++ b/src/generator/readable.rs @@ -166,13 +166,6 @@ impl ReadableLuaGenerator { } else { false } - // utils::is_relevant_for_spacing(&next_character) - // && self - // .output - // .chars() - // .last() - // .filter(utils::is_relevant_for_spacing) - // .is_some() } #[inline] diff --git a/src/nodes/types/table.rs b/src/nodes/types/table.rs index d30edaa..1b26ab3 100644 --- a/src/nodes/types/table.rs +++ b/src/nodes/types/table.rs @@ -429,26 +429,6 @@ impl TableType { self.entries.iter_mut() } - // #[inline] - // pub fn iter_property_type(&self) -> impl Iterator { - // self.properties.iter() - // } - - // #[inline] - // pub fn iter_mut_property_type(&mut self) -> impl Iterator { - // self.properties.iter_mut() - // } - - // #[inline] - // pub fn get_indexer_type(&self) -> Option<&TableIndexerType> { - // self.indexer.as_ref() - // } - - // #[inline] - // pub fn mutate_indexer_type(&mut self) -> Option<&mut TableIndexerType> { - // self.indexer.as_mut() - // } - pub fn with_tokens(mut self, tokens: TableTypeTokens) -> Self { self.tokens = Some(tokens); self diff --git a/src/nodes/types/type_name.rs b/src/nodes/types/type_name.rs index 5f6bca7..e6cbda8 100644 --- a/src/nodes/types/type_name.rs +++ b/src/nodes/types/type_name.rs @@ -220,58 +220,6 @@ pub enum TypeParameter { GenericTypePack(GenericTypePack), } -impl TypeParameter { - // pub fn clear_comments(&mut self) { - // match self { - // Self::Type(r#type) => r#type.clear_comments(), - // Self::TypePack(type_pack) => type_pack.clear_comments(), - // Self::VariadicTypePack(variadic_type_pack) => { - // variadic_type_pack.clear_comments() - // } - // Self::GenericTypePack(generic_type_pack) => generic_type_pack.clear_comments(), - // } - // } - - // pub fn clear_whitespaces(&mut self) { - // match self { - // Self::Type(r#type) => r#type.clear_whitespaces(), - // Self::TypePack(type_pack) => type_pack.clear_whitespaces(), - // Self::VariadicTypePack(variadic_type_pack) => { - // variadic_type_pack.clear_whitespaces() - // } - // Self::GenericTypePack(generic_type_pack) => { - // generic_type_pack.clear_whitespaces() - // } - // } - // } - - // pub(crate) fn replace_referenced_tokens(&mut self, code: &str) { - // match self { - // Self::Type(r#type) => r#type.replace_referenced_tokens(code), - // Self::TypePack(type_pack) => type_pack.replace_referenced_tokens(code), - // Self::VariadicTypePack(variadic_type_pack) => { - // variadic_type_pack.replace_referenced_tokens(code) - // } - // Self::GenericTypePack(generic_type_pack) => { - // generic_type_pack.replace_referenced_tokens(code) - // } - // } - // } - - // pub(crate) fn shift_token_line(&mut self, amount: usize) { - // match self { - // Self::Type(r#type) => r#type.shift_token_line(amount), - // Self::TypePack(type_pack) => type_pack.shift_token_line(amount), - // Self::VariadicTypePack(variadic_type_pack) => { - // variadic_type_pack.shift_token_line(amount) - // } - // Self::GenericTypePack(generic_type_pack) => { - // generic_type_pack.shift_token_line(amount) - // } - // } - // } -} - impl> From for TypeParameter { fn from(value: T) -> Self { Self::Type(value.into()) From a7fe7c133f86b714146063dc4136a4a392bd420b Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Thu, 23 Nov 2023 23:20:28 -0500 Subject: [PATCH 14/14] add interpolated string case --- ...__rules__remove_comments__test__remove_comments_in_code.snap | 2 ++ ...core__rules__remove_spaces__test__remove_spaces_in_code.snap | 2 ++ tests/test_cases/spaces_and_comments.lua | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/rules/snapshots/darklua_core__rules__remove_comments__test__remove_comments_in_code.snap b/src/rules/snapshots/darklua_core__rules__remove_comments__test__remove_comments_in_code.snap index 7995018..065dffa 100644 --- a/src/rules/snapshots/darklua_core__rules__remove_comments__test__remove_comments_in_code.snap +++ b/src/rules/snapshots/darklua_core__rules__remove_comments__test__remove_comments_in_code.snap @@ -57,3 +57,5 @@ object : method ({ key = [[true]], [ true ] = ( nil ) object . field : method { if value then ok else err, { }, } +local string = `-{ true }-{ object }={ c + 8 }` + diff --git a/src/rules/snapshots/darklua_core__rules__remove_spaces__test__remove_spaces_in_code.snap b/src/rules/snapshots/darklua_core__rules__remove_spaces__test__remove_spaces_in_code.snap index 1b11993..0a1934c 100644 --- a/src/rules/snapshots/darklua_core__rules__remove_spaces__test__remove_spaces_in_code.snap +++ b/src/rules/snapshots/darklua_core__rules__remove_spaces__test__remove_spaces_in_code.snap @@ -56,3 +56,5 @@ object:method--[[call]]({key--[[]]=[[true]],[true--[[key]] ]=(nil)-- nil object.--[[get field]]field:method-- {if--[[condition]]value then--[[true]]ok else--[[false]]err,{--[[empty table]]},--[[trailing comma]]} + +local string=`-{true}-{object--[[ok]]}={c+8}`-- interpolated string diff --git a/tests/test_cases/spaces_and_comments.lua b/tests/test_cases/spaces_and_comments.lua index babf6fc..5f2c418 100644 --- a/tests/test_cases/spaces_and_comments.lua +++ b/tests/test_cases/spaces_and_comments.lua @@ -52,3 +52,5 @@ object : method --[[call]] ({ key --[[]] = [[true]], [ true --[[key]]] = ( nil ) object . --[[get field]] field : method -- { if --[[condition]] value then --[[true]] ok else --[[false]] err, { --[[empty table]] }, --[[trailing comma]] } + +local string = `-{ true }-{ object --[[ok]] }={ c + 8 }` -- interpolated string