feat!: complete support memory64 proposal (#270)

.github/workflows/main.yml

@@ -14,6 +14,9 @@ jobs:
         submodules: true
     - name: Install Rust
       run: rustup update ${{ matrix.rust }} && rustup default ${{ matrix.rust }}
+    - uses: cargo-bins/cargo-binstall@main
+    - name: Install wasm-tools (need json-from-wast subcommand)
+      run: cargo binstall wasm-tools -y
     - name: Install wabt
       run: |
         set -e

crates/tests/tests/spec-tests.rs

@@ -30,7 +30,7 @@ fn run(wast: &Path) -> Result<(), anyhow::Error> {
         Some("extended-const") => return Ok(()),
         Some("function-references") => return Ok(()),
         Some("gc") => return Ok(()),
-        Some("memory64") => return Ok(()),
+        Some("memory64") => &["--enable-memory64"],
         Some("multi-memory") => &["--enable-multi-memory"],
         Some("relaxed-simd") => return Ok(()),
         Some("tail-call") => return Ok(()),
@@ -40,14 +40,21 @@ fn run(wast: &Path) -> Result<(), anyhow::Error> {
 
     let tempdir = TempDir::new()?;
     let json = tempdir.path().join("foo.json");
-    let status = Command::new("wast2json")
+    // Using `wasm-tools json-from-wast` instead of wabt's `wast2json`
+    // because the latter is slow to support new proposals.
+    let output = Command::new("wasm-tools")
+        .arg("json-from-wast")
+        .arg("--pretty")
         .arg(wast)
-        .arg("-o")
+        .arg("--output")
         .arg(&json)
-        .args(extra_args)
-        .status()
-        .context("executing `wast2json`")?;
-    assert!(status.success());
+        .arg("--wasm-dir")
+        .arg(tempdir.path())
+        .output()?;
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        bail!("failed to run `wasm-tools json-from-wast`\nstderr: {stderr}");
+    }
 
     let contents = fs::read_to_string(&json).context("failed to read file")?;
     let test: Test = serde_json::from_str(&contents).context("failed to parse file")?;
@@ -69,10 +76,23 @@ fn run(wast: &Path) -> Result<(), anyhow::Error> {
             Some(name) => name.as_str().unwrap().to_string(),
             None => continue,
         };
+        // walrus only process .wasm binary files
+        if filename.ends_with(".wat") {
+            continue;
+        }
         let line = command["line"].as_u64().unwrap();
         let path = tempdir.path().join(filename);
         match command["type"].as_str().unwrap() {
             "assert_invalid" | "assert_malformed" => {
+                if proposal.is_some()
+                    && ["zero byte expected", "multiple memories"]
+                        .contains(&command["text"].as_str().unwrap())
+                {
+                    // The multi-memory proposal is enabled for all proprosals tests
+                    // but some proposals tests still expect them to fail.
+                    continue;
+                }
+
                 let wasm = fs::read(&path)?;
                 if config.parse(&wasm).is_ok() {
                     should_not_parse.push(line);

src/dot.rs

@@ -514,8 +514,8 @@ impl DotNode for Data {
     }
 
     fn edges(&self, edges: &mut impl EdgeAggregator) {
-        if let DataKind::Active(ref a) = self.kind {
-            edges.add_edge_from_port("kind", &a.memory);
+        if let DataKind::Active { memory, offset: _ } = self.kind {
+            edges.add_edge_from_port("kind", &memory);
         }
     }
 }

src/module/config.rs

@@ -179,6 +179,7 @@ impl ModuleConfig {
         if !self.only_stable_features {
             // # Fully supported proposals.
             features.insert(WasmFeatures::MULTI_MEMORY);
+            features.insert(WasmFeatures::MEMORY64);
             // # Partially supported proposals.
             // ## threads
             // spec-tests/proposals/threads still fail

src/module/data.rs

@@ -4,7 +4,7 @@ use crate::emit::{Emit, EmitContext};
 use crate::ir::Value;
 use crate::parse::IndicesToIds;
 use crate::tombstone_arena::{Id, Tombstone, TombstoneArena};
-use crate::{ConstExpr, GlobalId, MemoryId, Module, Result, ValType};
+use crate::{ConstExpr, MemoryId, Module, Result, ValType};
 use anyhow::{bail, Context};
 
 /// A passive element segment identifier
@@ -35,35 +35,21 @@ pub struct Data {
 pub enum DataKind {
     /// An active data segment that is automatically initialized at some address
     /// in a static memory.
-    Active(ActiveData),
+    Active {
+        /// The memory that this active data segment will be automatically
+        /// initialized in.
+        memory: MemoryId,
+        /// The memory offset where this active data segment will be automatically
+        /// initialized.
+        offset: ConstExpr,
+    },
     /// A passive data segment that must be manually initialized at a dynamic
     /// address via the `memory.init` instruction (perhaps multiple times in
     /// multiple different memories) and then manually freed when it's no longer
     /// needed via the `data.drop` instruction.
     Passive,
 }
 
-/// The parts of a data segment that are only present in active data segments.
-#[derive(Clone, Debug)]
-pub struct ActiveData {
-    /// The memory that this active data segment will be automatically
-    /// initialized in.
-    pub memory: MemoryId,
-    /// The memory location where this active data segment will be automatically
-    /// initialized.
-    pub location: ActiveDataLocation,
-}
-
-/// The memory location where an active data segment will be automatically
-/// initialized.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum ActiveDataLocation {
-    /// A static, absolute address within the memory.
-    Absolute(u32),
-    /// A relative address (expressed as a global's value) within the memory.
-    Relative(GlobalId),
-}
-
 impl Tombstone for Data {
     fn on_delete(&mut self) {
         self.value = Vec::new();
@@ -231,21 +217,33 @@ impl Module {
                     memory.data_segments.insert(data.id);
 
                     let offset = ConstExpr::eval(&offset_expr, ids)
-                        .with_context(|| format!("in segment {}", i))?;
-                    data.kind = DataKind::Active(ActiveData {
-                        memory: memory_id,
-                        location: match offset {
-                            ConstExpr::Value(Value::I32(n)) => {
-                                ActiveDataLocation::Absolute(n as u32)
-                            }
+                        .with_context(|| format!("failed to evaluate the offset of data {}", i))?;
+
+                    if memory.memory64 {
+                        match offset {
+                            ConstExpr::Value(Value::I64(_)) => {}
                             ConstExpr::Global(global)
-                                if self.globals.get(global).ty == ValType::I32 =>
-                            {
-                                ActiveDataLocation::Relative(global)
-                            }
-                            _ => bail!("non-i32 constant in segment {}", i),
-                        },
-                    });
+                                if self.globals.get(global).ty == ValType::I64 => {}
+                            _ => bail!(
+                                "data {} is active for 64-bit memory but has non-i64 offset",
+                                i
+                            ),
+                        }
+                    } else {
+                        match offset {
+                            ConstExpr::Value(Value::I32(_)) => {}
+                            ConstExpr::Global(global)
+                                if self.globals.get(global).ty == ValType::I32 => {}
+                            _ => bail!(
+                                "data {} is active for 32-bit memory but has non-i32 offset",
+                                i
+                            ),
+                        }
+                    }
+                    data.kind = DataKind::Active {
+                        memory: memory_id,
+                        offset,
+                    }
                 }
             }
         }
@@ -270,17 +268,10 @@ impl Emit for ModuleData {
                 DataKind::Passive => {
                     wasm_data_section.passive(data.value.clone());
                 }
-                DataKind::Active(ref a) => {
+                DataKind::Active { memory, offset } => {
                     wasm_data_section.active(
-                        cx.indices.get_memory_index(a.memory),
-                        &match a.location {
-                            ActiveDataLocation::Absolute(a) => {
-                                wasm_encoder::ConstExpr::i32_const(a as i32)
-                            }
-                            ActiveDataLocation::Relative(g) => {
-                                wasm_encoder::ConstExpr::global_get(cx.indices.get_global_index(g))
-                            }
-                        },
+                        cx.indices.get_memory_index(memory),
+                        &offset.to_wasmencoder_type(cx),
                         data.value.clone(),
                     );
                 }

src/module/elements.rs

@@ -163,18 +163,39 @@ impl Module {
                     offset_expr,
                 } => {
                     // TODO: Why table_index is Option?
-                    let table = ids.get_table(table_index.unwrap_or_default())?;
-                    self.tables.get_mut(table).elem_segments.insert(id);
+                    let table_id = ids.get_table(table_index.unwrap_or_default())?;
+                    let table = self.tables.get_mut(table_id);
+                    table.elem_segments.insert(id);
 
-                    let offset = ConstExpr::eval(&offset_expr, ids)
-                        .with_context(|| format!("in segment {}", i))?;
-                    match offset {
-                        ConstExpr::Value(Value::I32(_)) => {}
-                        ConstExpr::Global(global)
-                            if self.globals.get(global).ty == ValType::I32 => {}
-                        _ => bail!("non-i32 constant in segment {}", i),
+                    let offset = ConstExpr::eval(&offset_expr, ids).with_context(|| {
+                        format!("failed to evaluate the offset of element {}", i)
+                    })?;
+                    if table.table64 {
+                        match offset {
+                            ConstExpr::Value(Value::I64(_)) => {}
+                            ConstExpr::Global(global)
+                                if self.globals.get(global).ty == ValType::I64 => {}
+                            _ => bail!(
+                                "element {} is active for 64-bit table but has non-i64 offset",
+                                i
+                            ),
+                        }
+                    } else {
+                        match offset {
+                            ConstExpr::Value(Value::I32(_)) => {}
+                            ConstExpr::Global(global)
+                                if self.globals.get(global).ty == ValType::I32 => {}
+                            _ => bail!(
+                                "element {} is active for 32-bit table but has non-i32 offset",
+                                i
+                            ),
+                        }
+                    }
+
+                    ElementKind::Active {
+                        table: table_id,
+                        offset,
                     }
-                    ElementKind::Active { table, offset }
                 }
             };
             self.elements.arena.alloc(Element {

src/module/imports.rs

@@ -2,7 +2,7 @@
 
 use std::convert::TryInto;
 
-use anyhow::{bail, Context};
+use anyhow::Context;
 
 use crate::emit::{Emit, EmitContext};
 use crate::parse::IndicesToIds;
@@ -172,9 +172,6 @@ impl Module {
                     ids.push_table(id.0);
                 }
                 wasmparser::TypeRef::Memory(m) => {
-                    if m.memory64 {
-                        bail!("64-bit memories not supported")
-                    };
                     let id = self.add_import_memory(
                         entry.module,
                         entry.name,

src/module/memories.rs

@@ -5,7 +5,6 @@ use crate::map::IdHashSet;
 use crate::parse::IndicesToIds;
 use crate::tombstone_arena::{Id, Tombstone, TombstoneArena};
 use crate::{Data, ImportId, Module, Result};
-use anyhow::bail;
 
 /// The id of a memory.
 pub type MemoryId = Id<Memory>;
@@ -157,9 +156,6 @@ impl Module {
         log::debug!("parse memory section");
         for m in section {
             let m = m?;
-            if m.memory64 {
-                bail!("64-bit memories not supported")
-            };
             let id = self.memories.add_local(
                 m.shared,
                 m.memory64,

src/module/mod.rs

@@ -22,7 +22,7 @@ pub use crate::module::custom::{
     CustomSection, CustomSectionId, ModuleCustomSections, RawCustomSection, TypedCustomSectionId,
     UntypedCustomSectionId,
 };
-pub use crate::module::data::{ActiveData, ActiveDataLocation, Data, DataId, DataKind, ModuleData};
+pub use crate::module::data::{Data, DataId, DataKind, ModuleData};
 pub use crate::module::debug::ModuleDebugData;
 pub use crate::module::elements::{Element, ElementId, ModuleElements};
 pub use crate::module::elements::{ElementItems, ElementKind};

src/passes/used.rs

@@ -1,6 +1,6 @@
 use crate::ir::*;
 use crate::map::IdHashSet;
-use crate::{ActiveDataLocation, ConstExpr, Data, DataId, DataKind, Element, ExportItem, Function};
+use crate::{ConstExpr, Data, DataId, DataKind, Element, ExportItem, Function};
 use crate::{ElementId, ElementItems, ElementKind, Module, RefType, Type, TypeId};
 use crate::{FunctionId, FunctionKind, Global, GlobalId};
 use crate::{GlobalKind, Memory, MemoryId, Table, TableId};
@@ -200,10 +200,10 @@ impl Used {
 
             while let Some(d) = stack.datas.pop() {
                 let d = module.data.get(d);
-                if let DataKind::Active(a) = &d.kind {
-                    stack.push_memory(a.memory);
-                    if let ActiveDataLocation::Relative(g) = a.location {
-                        stack.push_global(g);
+                if let DataKind::Active { memory, offset } = &d.kind {
+                    stack.push_memory(*memory);
+                    if let ConstExpr::Global(g) = offset {
+                        stack.push_global(*g);
                     }
                 }
             }