feat: New list syntax with enforced ,-operator (#100)

* Char<...> syntax for character classes

Step 1

* Use of new Char-syntax in tokay.tok

Step 2

* Op::collect() with sequence mode

- sequence mode is the previous behavior
- void-values with an alias become null values
- non-sequence mode always results in a dict (probably empty)

* Drafting new dict/list syntax

This is unfinished and intermediate, as the syntax is still not clear between inline sequences and dicts.

Further development depends on iterators.

* Fixing tests for new Char<...> syntax

* Substitute `Any` by `Char`

* Adding `|`-syntax inside `[...]` as well

* Remove compiler warning

* Regenerating parser

* Restored original parser

* Rebuild parser from tokay.tok

* Some tests and experiments

Still unfinished. Trying several ways.

* Starting with alternative list syntax

This branch starts over with another syntax for lists, where the comma (,) becomes an operator with a meaning.

* repr lists in brackets

* Improving the new list (,) syntax

- Clean-up and fixing grammar
- Updating and investigating tests
- Updating prelude

* Implementing InlineExpressionList & InlineHoldAssignment

* Fixing some testcases

* Introducting InlineAssignment

- InlineAssignment separates the assigned value on the stack
- Enhanced dict.tok dests, fixes test suite in this branch for now

* Improving inline syntax to just one path

This commit makes InlineSequences as the only handler for content of brackets.
This simplifies things and cleans up the syntax.

Tests will be fixed soon.

* Further experiments with the sequence/inline_sequence syntax

Fixes more testcases

* Distinguish inlined lists and sequences

* Improving sequence, fixing tests/inline_parseable_sequence.tok

* Replacing `inline_sequence` by just `sequence`

Rebuiling prelude.rs using new tokay.tok resulted in smaller code.

* Still unclear on sequence/list severity on inline syntax

* Draft for a cleaner, less ambigous syntax for inlined list and sequence (dict)

* assign_drop

* Fixing tests/dict.tok for now

* Optimizing Assignment parselets

* Compacting tokay.tok by using generic Assignment parselet

src/compiler/ast.rs

@@ -779,7 +779,8 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
 
             let mut ops = Vec::new();
 
-            if parts.len() > 1 && parts[1] != "hold" {
+            /* assignment with operation */
+            if parts.len() > 1 && !["copy", "drop", "hold"].contains(&parts[1]) {
                 ops.push(traverse_node_lvalue(compiler, lvalue, false, false));
                 ops.push(traverse_node_rvalue(compiler, value, Rvalue::Load));
 
@@ -793,18 +794,26 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
                     _ => unreachable!(),
                 });
 
-                if *parts.last().unwrap() != "hold" {
-                    ops.push(Op::Inv.into());
+                match *parts.last().unwrap() {
+                    "hold" => {}
+                    "copy" => ops.push(Op::Sep.into()),
+                    _ => ops.push(Op::Inv.into()),
                 }
-            } else {
+            }
+            /* normal assignment without operation */
+            else {
                 ops.push(traverse_node_rvalue(compiler, value, Rvalue::Load));
                 ops.push(traverse_offset(node));
                 ops.push(traverse_node_lvalue(
                     compiler,
                     lvalue,
                     true,
-                    *parts.last().unwrap() == "hold",
+                    ["copy", "hold"].contains(parts.last().unwrap()),
                 ));
+
+                if *parts.last().unwrap() == "copy" {
+                    ops.push(Op::Sep.into())
+                }
             }
 
             ImlOp::from(ops)
@@ -1594,14 +1603,16 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
             ImlOp::from(ops)
         }
 
-        // sequence  ------------------------------------------------------
-        "sequence" | "inline_sequence" | "list" => {
+        // sequence, dict, list  -----------------------------------------
+        "sequence" | "dict" | "list" => {
             let children = if let Some(children) = node.get_str("children") {
                 List::from(children)
             } else {
                 List::new()
             };
 
+            //println!("{} => {:?}", emit, children);
+
             let mut ops = Vec::new();
 
             for node in children.iter() {
@@ -1612,14 +1623,14 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
                 ));
             }
 
-            // Lists are definitive lists with a given length and only non-aliased values
-            if emit == "list" {
-                ops.push(Op::MakeList(children.len()).into());
-                ImlOp::from(ops)
-            }
-            // In most cases, lists are parsed as sequences;
-            else {
-                ImlOp::seq(ops, true)
+            match emit {
+                "list" if ops.is_empty() => ImlOp::from(Op::MakeList(0)),
+                "list" => {
+                    ops.push(ImlOp::from(Op::MakeList(ops.len())));
+                    ImlOp::seq(ops, false)
+                }
+                "dict" if ops.is_empty() => ImlOp::from(Op::MakeDict(0)),
+                _ => ImlOp::seq(ops, true),
             }
         }
 
@@ -1701,7 +1712,7 @@ tokay_function!("ast : @emit, value=void, flatten=true, debug=false", {
     let value = if value.is_void() {
         Some(
             context
-                .collect(capture_start, false, debug.is_true())
+                .collect(capture_start, false, true, debug.is_true())
                 .extract(&context.thread.reader),
         )
     } else {
@@ -1733,7 +1744,7 @@ tokay_function!("ast : @emit, value=void, flatten=true, debug=false", {
             ret.insert_str("children", value.clone());
         }
         // Otherwise this is a value
-        else {
+        else if !value.is_void() {
             ret.insert_str("value", value.clone());
         }
     }

src/compiler/iml/imlop.rs

@@ -29,9 +29,7 @@ pub(in crate::compiler) enum ImlOp {
     // Sequence of ops, optionally a collection
     Seq {
         seq: Vec<ImlOp>,
-        collection: bool, /* According to these operation's semantics, or when an entire sequence is completely recognized,
-                          the sequence is getting accepted. Incomplete sequences are rejected, but might partly be
-                          processed, including data changes, which is a wanted behavior. */
+        collect: bool, // Run a Context::collect() on successfull sequence match
     },
 
     // Conditional block
@@ -53,14 +51,14 @@ pub(in crate::compiler) enum ImlOp {
 
 impl ImlOp {
     /// Creates a sequence from items, and optimizes stacked, unframed sequences
-    pub fn seq(items: Vec<ImlOp>, collection: bool) -> ImlOp {
+    pub fn seq(items: Vec<ImlOp>, collect: bool) -> ImlOp {
         let mut seq = Vec::new();
 
         for item in items {
             match item {
                 ImlOp::Nop => {}
                 ImlOp::Seq {
-                    collection: false,
+                    collect: false,
                     seq: items,
                 } => seq.extend(items),
                 item => seq.push(item),
@@ -69,8 +67,8 @@ impl ImlOp {
 
         match seq.len() {
             0 => ImlOp::Nop,
-            1 if !collection => seq.pop().unwrap(),
-            _ => ImlOp::Seq { seq, collection },
+            1 if !collect => seq.pop().unwrap(),
+            _ => ImlOp::Seq { seq, collect },
         }
     }
 
@@ -213,19 +211,13 @@ impl ImlOp {
 
                 ops.extend(ret);
             }
-            ImlOp::Seq { seq, collection } => {
+            ImlOp::Seq { seq, collect } => {
                 for item in seq.iter() {
                     item.compile(program, current, ops);
                 }
 
                 // Check if the sequence exists of more than one operational instruction
-                if *collection
-                    && ops[start..]
-                        .iter()
-                        .map(|op| if matches!(op, Op::Offset(_)) { 0 } else { 1 })
-                        .sum::<usize>()
-                        > 1
-                {
+                if *collect {
                     ops.insert(start, Op::Frame(0));
                     ops.push(Op::Collect);
                     ops.push(Op::Close);