Skip to content

Commit

Permalink
fix: Generic Self/self (#142)
Browse files Browse the repository at this point in the history 
* Add more trace and some comments

* Step 1

* Step 2

* Possible solution for #140

* Push through offset on generic

* Handle SelfToken in left-recursion finalization correctly

* Add some tests for generic Self
  • Loading branch information
@phorward
phorward committed Sep 17, 2024
1 parent 0e17b59 commit a245f24
Show file tree
Hide file tree
Showing 12 changed files with 146 additions and 154 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ $ RUST_LOG=tokay=debug tokay
Alternatively, tracing can be activated for the `__main__`-program by setting `TOKAY_LOG`. This is used to start tracing when the internal parser has been compiled and executed already, and parsed the actual program. `TOKAY_LOG` can be set to any `RUST_LOG`-compliant format, as it becomes `RUST_LOG` right after.

```
$ TOKAY_LOG=tokay=debug tokay
$ TOKAY_LOG=debug tokay
```

### Built-in AST and VM debugger using `TOKAY_DEBUG` and `TOKAY_PARSER_DEBUG`
Expand Down
15 changes: 12 additions & 3 deletions src/compiler/ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ fn traverse_node_value(scope: &Scope, node: &Dict, name: Option<String>) -> ImlV
"value_null" => ImlValue::Value(scope.compiler.statics.borrow()[1].clone()),
"value_true" => ImlValue::Value(scope.compiler.statics.borrow()[2].clone()),
"value_false" => ImlValue::Value(scope.compiler.statics.borrow()[3].clone()),
"value_self" => ImlValue::SelfValue,
"value_integer" => match node["value"].to_i64() {
Ok(0) => ImlValue::Value(scope.compiler.statics.borrow()[4].clone()),
Ok(1) => ImlValue::Value(scope.compiler.statics.borrow()[5].clone()),
Expand All @@ -90,7 +89,6 @@ fn traverse_node_value(scope: &Scope, node: &Dict, name: Option<String>) -> ImlV
"value_string" => scope.compiler.register_static(node["value"].clone()),

// Tokens
"value_token_self" => ImlValue::SelfToken,
"value_token_void" => ImlValue::VoidToken,
"value_token_match" | "value_token_touch" => {
let mut value = node["value"].to_string();
Expand Down Expand Up @@ -214,7 +212,18 @@ fn traverse_node_value(scope: &Scope, node: &Dict, name: Option<String>) -> ImlV
);
}

Some(default)
Some(match default {
// Any Self/self-generic becomes its definitive ImlValue pendant
// because there must be a distinguishment between
// default Self/self and instance Self/self.
ImlValue::Generic { name, .. } if name == "self" => {
ImlValue::SelfValue
}
ImlValue::Generic { name, .. } if name == "Self" => {
ImlValue::SelfToken
}
default => default,
})
} else {
None
};
Expand Down
48 changes: 34 additions & 14 deletions src/compiler/iml/imlparselet.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ impl Eq for ImlParselet {}

impl std::hash::Hash for ImlParselet {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
// Hash is generated from the model and the generics configuration
let model = &*self.model.borrow();
(model as *const ImlParseletModel as usize).hash(state);
self.generics.iter().collect::<Vec<_>>().hash(state);
Expand Down Expand Up @@ -203,7 +204,8 @@ impl ImlRefParselet {
}
}

/** Derives an intermediate parselet by another intermediate parselet (`from`).
/** Derives an intermediate parselet instance from the view of
another intermediate parselet instance (`from`).
The namespace defines the constant configuration of a surrounding parselet (`from`),
and extends the intermediate parselet's constant configuration, making it a derivation.
Expand All @@ -220,27 +222,38 @@ impl ImlRefParselet {
*/
pub fn derive(&self, from: &ImlRefParselet) -> Result<Self, String> {
let parselet = self.parselet.borrow();

// Fast track
if parselet.generics.is_empty() {
return Ok(self.clone());
}

let mut generics = parselet.generics.clone();
let mut changes = false;
let mut required = Vec::new();

log::info!("Deriving {} from {}", self, from);
log::debug!(" deriving {} from {}", self, from);

for (name, value) in generics.iter_mut() {
// Replace any generics until no more are open
// Replace any generics until no more are open;
// need to do it in a loop, as generics can reference other generics.
while let Some(ImlValue::Generic { name, .. }) = value {
*value = from.borrow().generics.get(name).unwrap().clone();
if name == "Self" || name == "self" {
*value = Some(ImlValue::Parselet(from.clone()));
} else {
*value = from.borrow().generics.get(name).unwrap().clone();
}

changes = true;
}

match value {
Some(ImlValue::SelfValue | ImlValue::SelfToken) => {
// Replace any references on `Self` or `self` by from
*value = Some(ImlValue::Parselet(from.clone()));
changes = true;
}
Some(_) => {}
None => required.push(name.to_string()),
// Generics pointing to ImlValue::SelfToken/SelfValue must be replaced, too
if changes && matches!(value, Some(ImlValue::SelfToken | ImlValue::SelfValue)) {
*value = Some(ImlValue::Parselet(from.clone()));
}

if value.is_none() {
required.push(name.to_string());
}
}

Expand All @@ -255,17 +268,24 @@ impl ImlRefParselet {

// When there is no change, there is no derivation
if !changes {
log::debug!(" no derivation");
// log::warn!(" {} => {}", self, self);
return Ok(self.clone());
}

Ok(Self::new(ImlParselet {
// Create new derivative parselet
let derived = Self::new(ImlParselet {
model: parselet.model.clone(),
generics,
offset: parselet.offset.clone(),
name: parselet.name.clone(),
severity: parselet.severity,
is_generated: parselet.is_generated,
}))
});

log::debug!(" derived = {}", derived);
// log::warn!("* {} => {}", self, derived);
Ok(derived)
}

/** Compiles an intermediate parselet into a compiled VM parselet,
Expand Down
49 changes: 26 additions & 23 deletions src/compiler/iml/imlprogram.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ impl ImlProgram {
}

// Finalize parselets
log::debug!("{} has {} parselets to finalize", self.main, finalize.len());
log::info!("{} has {} parselets to finalize", self.main, finalize.len());

for (i, parselet) in finalize.iter().enumerate() {
log::trace!(" {: >3} => {:#?}", i, parselet);
Expand Down Expand Up @@ -140,7 +140,7 @@ impl ImlProgram {
})
.collect();

log::debug!("{} has {} statics compiled", self.main, statics.len());
log::info!("{} has {} statics compiled", self.main, statics.len());

for (i, value) in statics.iter().enumerate() {
log::trace!(" {: >3} : {:#?}", i, value);
Expand Down Expand Up @@ -178,10 +178,14 @@ impl ImlProgram {
ImlValue::Shared(value) => {
finalize_value(&*value.borrow(), current, visited, configs)
}
ImlValue::SelfToken => Some(Consumable {
leftrec: true,
nullable: false,
}),
ImlValue::SelfToken => {
configs.get_mut(current).unwrap().leftrec = true;

Some(Consumable {
leftrec: true,
nullable: configs[current].nullable,
})
}
ImlValue::Parselet(parselet) => {
// Try to derive the parselet with current constants
let derived = parselet.derive(current).unwrap();
Expand All @@ -202,15 +206,12 @@ impl ImlProgram {
None
}
}
ImlValue::Generic { name, .. } => {
// fixme: Is this still relevant???
finalize_value(
current.borrow().generics[name].as_ref().unwrap(),
current,
visited,
configs,
)
}
ImlValue::Generic { name, .. } => finalize_value(
current.borrow().generics[name].as_ref().unwrap(),
current,
visited,
configs,
),
_ => None,
}
}
Expand Down Expand Up @@ -358,6 +359,12 @@ impl ImlProgram {
}
}

log::trace!(
"{} has {} parselets to finalize",
self.statics.keys()[0],
parselets.len()
);

// Now, start the closure algorithm with left-recursive and nullable configurations for all parselets
// put into the finalize list.
let mut changes = true;
Expand All @@ -383,20 +390,16 @@ impl ImlProgram {
}
}

for parselet in &parselets {
log::trace!(" {:?} consuming={:?}", parselet, configs[&parselet]);
}

log::debug!(
"{} has {} parselets finalized",
self.statics.keys()[0],
parselets.len()
);

for parselet in &parselets {
log::trace!(
" {} consuming={:?}",
parselet.borrow().name.as_deref().unwrap_or("(unnamed)"),
configs[&parselet]
);
}

configs.into_iter().map(|(k, v)| (k, v.leftrec)).collect()
}
}
22 changes: 15 additions & 7 deletions src/compiler/iml/imlvalue.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ impl ImlValue {
generics.insert(name.clone(), arg.1);
}

// Report any errors for unconsumed generic arguments.
// Report any errors for remaining generic arguments.
if !instance.args.is_empty() {
scope.push_error(
instance.args[0].0, // report first parameter
Expand Down Expand Up @@ -317,7 +317,6 @@ impl ImlValue {
pub fn is_consuming(&self) -> bool {
match self {
Self::Shared(value) => value.borrow().is_consuming(),
Self::SelfValue => false,
Self::SelfToken | Self::VoidToken => true,
Self::Value(value) => value.is_consuming(),
Self::Parselet(parselet) => parselet.borrow().model.borrow().is_consuming,
Expand Down Expand Up @@ -350,11 +349,17 @@ impl ImlValue {
Self::Shared(value) => {
return value.borrow().compile(program, current, offset, call, ops)
}
Self::Generic { name, .. } => {
return current.0.borrow().generics[name]
.as_ref()
.unwrap()
.compile(program, current, offset, call, ops)
Self::Generic {
name,
offset: generic_offset,
} => {
return current.0.borrow().generics[name].as_ref().unwrap().compile(
program,
current,
&generic_offset.or(*offset),
call,
ops,
)
}
Self::VoidToken => Some(Op::Next),
Self::Value(value) => match &*value.borrow() {
Expand Down Expand Up @@ -406,6 +411,7 @@ impl ImlValue {
}
},
ImlValue::Value(_) => program.register(self),
ImlValue::SelfToken | ImlValue::SelfValue => current.1,
_ => unreachable!("Can't compile {:?}", self),
};

Expand Down Expand Up @@ -493,6 +499,8 @@ impl std::hash::Hash for ImlValue {
state.write_u8('p' as u8);
parselet.borrow().id().hash(state);
}
Self::SelfToken => state.write_u8('S' as u8),
Self::SelfValue => state.write_u8('s' as u8),
other => unreachable!("{:?} is unhashable", other),
}
}
Expand Down
Loading

0 comments on commit a245f24

Please sign in to comment.