From 055599380b78154201074cb89d71fc329442b69e Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 6 Jun 2024 15:30:19 +0200 Subject: [PATCH] Made Network interface generic with trait and added implementation for existing TAP devices --- Cargo.toml | 3 +- src/net/macvtap.rs | 111 ++++++++++++++++++++++++++++ src/net/mod.rs | 25 ++++++- src/net/tap.rs | 175 +++++++++++++++++++-------------------------- src/virtio/net.rs | 21 +++--- 5 files changed, 222 insertions(+), 113 deletions(-) create mode 100644 src/net/macvtap.rs diff --git a/Cargo.toml b/Cargo.toml index dc022cf86..166006e2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ instrument = ["rftrace", "rftrace-frontend"] [dependencies] byte-unit = { version = "5", features = ["byte"] } clap = { version = "4.5", features = ["derive", "env"] } -nix = { version = "0.28", features = ["mman", "pthread", "signal", "net"] } +nix = { version = "0.28", features = ["mman", "pthread", "signal", "net", "ioctl"] } core_affinity = "0.8" either = "1.12" env_logger = "0.11" @@ -59,7 +59,6 @@ tun-tap = { version = "0.1.3", default-features = false } uhyve-interface = { version = "0.1.1", path = "uhyve-interface", features = ["std"] } virtio-bindings = { version = "0.2", features = ["virtio-v4_14_0"] } bitflags = "2.4" - rftrace = { version = "0.1", optional = true } rftrace-frontend = { version = "0.1", optional = true } vm-memory = { version = "0.14.1", features = ["backend-mmap"] } diff --git a/src/net/macvtap.rs b/src/net/macvtap.rs new file mode 100644 index 000000000..b0738c4e8 --- /dev/null +++ b/src/net/macvtap.rs @@ -0,0 +1,111 @@ +//! Tap device wrapper for uhyve. + +use std::{ + io::{self, Read, Write}, + os::fd::AsRawFd, + str::FromStr, + sync::Mutex, +}; + +use macvtap::{Iface, Mode}; +use nix::{ifaddrs::InterfaceAddress, net::if_::InterfaceFlags, sys::socket::LinkAddr}; + +use crate::net::{NetworkInterface, UHYVE_NET_MTU}; + +/// Wrapper for a tap device, containing the descriptor and mac address. +pub struct MacVTap { + tap: Iface, + interface_address: InterfaceAddress, +} + +impl MacVTap { + /// Create a Layer 2 Linux/*BSD tap device, named "tap[0-9]+". + pub fn new() -> io::Result { + let iface_name = std::env::var("TAP").unwrap_or("tap10".to_owned()); + + Self::from_str(&iface_name) + } + + /// Return the tap device name + pub fn name(&self) -> &str { + &self.interface_address.interface_name + } + + fn mac_addr(&self) -> LinkAddr { + *self + .interface_address + .address + .unwrap() + .as_link_addr() + .unwrap() + } +} + +impl NetworkInterface for Mutex { + fn mac_address_as_bytes(&self) -> [u8; 6] { + self.lock().unwrap().mac_addr().addr().unwrap() + } + + fn send(&self, buf: &[u8]) -> io::Result { + let mut guard = self.lock().unwrap(); + trace!("sending {} bytes on MacVTap {}", buf.len(), guard.name()); + guard.tap.write(buf) + } + + fn recv(&self, buf: &mut [u8]) -> io::Result { + let mut guard = self.lock().unwrap(); + let res = guard.tap.read(buf); + trace!("receiving {res:?} bytes on MacVTap {}", guard.name()); + res + } +} + +impl Drop for MacVTap { + fn drop(&mut self) { + self.tap.close(); + } +} + +impl AsRawFd for MacVTap { + fn as_raw_fd(&self) -> i32 { + self.tap.as_raw_fd() + } +} + +impl Default for MacVTap { + fn default() -> Self { + Self::new().unwrap() + } +} + +impl FromStr for MacVTap { + type Err = std::io::Error; + + fn from_str(name: &str) -> std::result::Result { + // TODO: MacVTap mode + let tap = Iface::new(name, Mode::Tap, UHYVE_NET_MTU.try_into().unwrap()) + .expect("Failed to create tap device (Device busy, or you may need to enable CAP_NET_ADMIN capability)."); + + let interface_address = nix::ifaddrs::getifaddrs() + .unwrap() + .find(|dev| dev.interface_name == name) + .expect("Could not find TAP interface."); + + // TODO: ensure the tap device is header-less + + assert!( + interface_address.flags.contains(InterfaceFlags::IFF_TAP), + "Interface is not a valid TAP device." + ); + + assert!( + interface_address.flags.contains(InterfaceFlags::IFF_UP), + "Interface is not up and running." + ); + + Ok(MacVTap { + tap, + interface_address, + }) + } +} diff --git a/src/net/mod.rs b/src/net/mod.rs index 73d6d91c9..e17e7c615 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -1,3 +1,5 @@ +use std::io; + pub use crate::consts::{UHYVE_NET_MTU, UHYVE_QUEUE_SIZE}; pub const BROADCAST_MAC_ADDR: [u8; 6] = [0xff; 6]; @@ -12,4 +14,25 @@ pub const UHYVE_PCI_CLASS_INFO: [u8; 3] = [ PCI_ETHERNET_SUBCLASS, ]; -pub mod tap; +pub(crate) mod macvtap; +pub(crate) mod tap; + +// TODO: Remove Sync and split in two +pub(crate) trait NetworkInterface: Sync + Send { + /// Return the MAC address as a byte array + fn mac_address_as_bytes(&self) -> [u8; 6]; + + /// Sends a packet to the interface. + /// + /// **NOTE**: ensure the packet has the appropriate format and header. + /// Incorrect packets will be dropped without warning. + fn send(&self, buf: &[u8]) -> io::Result; + + /// Receives a packet from the interface. + /// + /// Blocks until a packet is sent into the virtual interface. At that point, the content of the + /// packet is copied into the provided buffer. + /// + /// Returns the size of the received packet + fn recv(&self, buf: &mut [u8]) -> io::Result; +} diff --git a/src/net/tap.rs b/src/net/tap.rs index 6061d374d..a2fb85cf5 100644 --- a/src/net/tap.rs +++ b/src/net/tap.rs @@ -1,119 +1,94 @@ -//! Tap device wrapper for uhyve. - use std::{ - io::{self, Read, Write}, - os::fd::AsRawFd, - str::FromStr, + fs::{File, OpenOptions}, + io::{self, Error, Read, Write}, + os::unix::io::AsRawFd, + sync::Mutex, }; -use macvtap::{Iface, Mode}; -use nix::{ifaddrs::InterfaceAddress, net::if_::InterfaceFlags, sys::socket::LinkAddr}; +use libc::{ifreq, IFF_NO_PI, IFF_TAP}; +use nix::{ifaddrs::getifaddrs, ioctl_write_int}; +use vm_memory::address; -use crate::net::UHYVE_NET_MTU; +use crate::net::NetworkInterface; -/// Wrapper for a tap device, containing the descriptor and mac address. +/// An existing (externally created) TAP device pub struct Tap { - tap: Iface, - interface_address: InterfaceAddress, + fd: File, + mac: [u8; 6], + name: String, } impl Tap { - /// Create a Layer 2 Linux/*BSD tap device, named "tap[0-9]+". pub fn new() -> io::Result { let iface_name = std::env::var("TAP").unwrap_or("tap10".to_owned()); - - Self::from_str(&iface_name) - } - - /// Return the tap device name - pub fn name(&self) -> &str { - &self.interface_address.interface_name - } - - fn mac_addr(&self) -> LinkAddr { - *self - .interface_address - .address - .unwrap() - .as_link_addr() - .unwrap() - } - - /// Return the MAC address as a byte array - pub fn mac_address_as_bytes(&self) -> [u8; 6] { - self.mac_addr().addr().unwrap() - } - - /// Get the tap interface struct - pub fn get_iface(&self) -> &Iface { - &self.tap - } - - /// Sends a packet to the interface. - /// - /// **NOTE**: ensure the packet has the appropriate format and header. - /// Incorrect packets will be dropped without warning. - pub fn send(&mut self, buf: &[u8]) -> io::Result { - self.tap.write(buf) - } - - /// Receives a packet from the interface. - /// - /// Blocks until a packet is sent into the virtual interface. At that point, the content of the - /// packet is copied into the provided buffer. - /// - /// Returns the size of the received packet - pub fn recv(&mut self, buf: &mut [u8]) -> io::Result { - self.tap.read(buf) + if iface_name.as_bytes().len() > 16 { + return Err(Error::other("Interface name must not exceed 16 bytes")); + } + let mut ifr_name: [i8; 16] = [0; 16]; + iface_name + .as_bytes() + .iter() + .take(15) + .map(|b| *b as i8) + .enumerate() + .for_each(|(i, b)| ifr_name[i] = b); + + let config_str = ifreq { + ifr_name, + ifr_ifru: libc::__c_anonymous_ifr_ifru { + ifru_flags: IFF_TAP as i16 | IFF_NO_PI as i16, // TODO: Investigate if IFF_NO_PI is necessary as well + }, + }; + + let fd = OpenOptions::new() + .read(true) + .write(true) + .open("/dev/net/tun")?; + + ioctl_write_int!(tun_set_iff, b'T', 202); + + let res = + unsafe { tun_set_iff(fd.as_raw_fd(), &config_str as *const ifreq as u64).unwrap() }; + + if res == -1 { + error!("Can't open TAP device {iface_name}"); + return Err(Error::other("Can't open TAP device")); + } + + // Find MAC address of the TAP device + let mut mac_addr = None; + for ifaddr in getifaddrs().unwrap() { + if let Some(address) = ifaddr.address { + if ifaddr.interface_name == iface_name { + if let Some(link_addr) = address.as_link_addr() { + mac_addr = Some(link_addr.addr().unwrap()); + } + } + } + } + + Ok(Self { + fd, + name: iface_name, + mac: mac_addr.expect("TAP device without MAC address?"), + }) } } - -impl Drop for Tap { - fn drop(&mut self) { - self.tap.close(); +impl NetworkInterface for Mutex { + fn mac_address_as_bytes(&self) -> [u8; 6] { + self.lock().unwrap().mac } -} -impl AsRawFd for Tap { - fn as_raw_fd(&self) -> i32 { - self.tap.as_raw_fd() + fn send(&self, buf: &[u8]) -> io::Result { + let mut guard = self.lock().unwrap(); + trace!("sending {} bytes on {}", buf.len(), guard.name); + guard.fd.write(buf) } -} - -impl Default for Tap { - fn default() -> Self { - Self::new().unwrap() - } -} -impl FromStr for Tap { - type Err = std::io::Error; - - fn from_str(name: &str) -> std::result::Result { - // TODO: MacVTap mode - let tap = Iface::new(name, Mode::Tap, UHYVE_NET_MTU.try_into().unwrap()) - .expect("Failed to create tap device (Device busy, or you may need to enable CAP_NET_ADMIN capability)."); - - let interface_address = nix::ifaddrs::getifaddrs() - .unwrap() - .find(|dev| dev.interface_name == name) - .expect("Could not find TAP interface."); - - // TODO: ensure the tap device is header-less - - assert!( - interface_address.flags.contains(InterfaceFlags::IFF_TAP), - "Interface is not a valid TAP device." - ); - - assert!( - interface_address.flags.contains(InterfaceFlags::IFF_UP), - "Interface is not up and running." - ); - - Ok(Tap { - tap, - interface_address, - }) + fn recv(&self, buf: &mut [u8]) -> io::Result { + let mut guard = self.lock().unwrap(); + let res = guard.fd.read(buf); + trace!("receiving {res:?} bytes on {}", guard.name); + res } } diff --git a/src/virtio/net.rs b/src/virtio/net.rs index 9da701af6..de6524598 100644 --- a/src/virtio/net.rs +++ b/src/virtio/net.rs @@ -4,6 +4,7 @@ use std::{ io::{Read, Write}, mem, sync::{ + self, atomic::{AtomicBool, Ordering}, Arc, }, @@ -21,7 +22,7 @@ use vmm_sys_util::eventfd::EventFd; use crate::{ consts::{UHYVE_IRQ_NET, UHYVE_NET_MTU}, - net::tap::Tap, + net::{macvtap::MacVTap, tap::Tap, NetworkInterface}, pci::{MemoryBar64, PciDevice}, virtio::{ capabilities::IsrStatus, @@ -51,7 +52,7 @@ pub struct VirtioNetPciDevice { /// transmitted virtqueue tx_queue: Arc>, /// virtual network interface - iface: Option>, + iface: Option>, /// File Descriptor for IRQ event signalling to guest irq_evtfd: Option, /// File Descriptor for polling guest (MMIO) IOEventFD signals @@ -229,9 +230,10 @@ impl VirtioNetPciDevice { fn start_network_interface(&mut self) { // Create a TAP device without packet info headers. - let iface = self - .iface - .insert(Arc::new(Tap::new().expect("Could not create TAP device"))); + let iface = self.iface.insert(Arc::new(sync::Mutex::new( + // Tap::new().expect("Could not create TAP device"), + MacVTap::new().expect("Could not create TAP device"), + ))); let sink = iface.clone(); let notify_evtfd = self.notify_evtfd.take().unwrap(); @@ -245,7 +247,7 @@ impl VirtioNetPciDevice { debug!("Starting notification watcher."); loop { if notify_evtfd.read().is_ok() { - match send_available_packets(&sink, &poll_tx_queue, &mmap) { + match send_available_packets(&(*sink), &poll_tx_queue, &mmap) { Ok(_) => {} Err(VirtIOError::QueueNotReady) => { error!("Sending before queue is ready!") @@ -270,10 +272,9 @@ impl VirtioNetPciDevice { let mut _delay = time::Instant::now(); let mut buf = [0u8; UHYVE_NET_MTU]; - let len = stream.get_iface().read(&mut buf).unwrap(); + let len = stream.recv(&mut buf).unwrap(); let mmap = mmap.as_ref().clone(); frame_queue.push_back((buf, len)); - // let l = frame_queue.len(); // Not ideal to wait random values or queue lengths. if _delay @@ -561,7 +562,7 @@ fn write_packet( } fn send_available_packets( - sink: &Arc, + sink: &dyn NetworkInterface, tx_queue_locked: &Arc>, mem: &GuestMemoryMmap, ) -> std::result::Result { @@ -592,7 +593,7 @@ fn send_available_packets( let mut buf = vec![0; len as usize]; mem.read_slice(&mut buf, desc.addr()).unwrap(); - match sink.get_iface().write(&buf[VIRTIO_NET_HEADER_SZ..]) { + match (*sink).send(&buf[VIRTIO_NET_HEADER_SZ..]) { Ok(sent_len) => { if sent_len != len as usize - VIRTIO_NET_HEADER_SZ { error!("Could not send all data provided! sent {sent_len}, vs {len}");