diff --git a/Cargo.lock b/Cargo.lock index e800a87d82..2fb9e556de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3917,6 +3917,7 @@ dependencies = [ "arboard", "async-recursion", "axum-server", + "binary-stream", "clap", "copy_dir", "csv-async", diff --git a/Cargo.toml b/Cargo.toml index 73f8f34ac9..4cc453b3f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,8 +31,14 @@ test-utils = ["sos-net/test-utils"] [workspace.dependencies] csv-async = { version = "1", features = ["tokio", "with_serde"] } +[workspace.dependencies.binary-stream] +version = "8.0.0" +features = ["async"] +#path = "../../../../binary-stream" + [dependencies] csv-async.workspace = true +binary-stream.workspace = true thiserror = "1" tracing = "0.1" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } diff --git a/src/cli/sos.rs b/src/cli/sos.rs index 9b1831eb7d..2d83d0f70e 100644 --- a/src/cli/sos.rs +++ b/src/cli/sos.rs @@ -9,10 +9,10 @@ use std::path::PathBuf; use crate::{ commands::{ - account, audit, changes, check, device, folder, secret, + account, audit, changes, check, device, events, folder, secret, security_report::{self, SecurityReportFormat}, shell, AccountCommand, AuditCommand, CheckCommand, DeviceCommand, - FolderCommand, SecretCommand, + EventsCommand, FolderCommand, SecretCommand, }, Result, }; @@ -124,6 +124,11 @@ pub enum Command { #[clap(subcommand)] cmd: CheckCommand, }, + /// Inspect event records. + Events { + #[clap(subcommand)] + cmd: EventsCommand, + }, /// Interactive login shell. Shell { /// Folder name or identifier. @@ -179,6 +184,7 @@ pub async fn run() -> Result<()> { changes::run(server, server_public_key, account).await? } Command::Check { cmd } => check::run(cmd).await?, + Command::Events { cmd } => events::run(cmd).await?, Command::Shell { account, folder } => { shell::run(account, folder).await? } diff --git a/src/commands/check.rs b/src/commands/check.rs index 82f8c72d63..3a5a9e56db 100644 --- a/src/commands/check.rs +++ b/src/commands/check.rs @@ -2,10 +2,7 @@ use clap::Subcommand; use std::path::PathBuf; use sos_net::sdk::{ - commit::{ - event_log_commit_tree_file, vault_commit_tree_file, CommitHash, - }, - events::{EventRecord, FolderEventLog, WriteEvent}, + commit::{event_log_commit_tree_file, vault_commit_tree_file}, formats::vault_stream, hex, uuid::Uuid, @@ -42,15 +39,6 @@ pub enum Command { #[clap(short, long)] verbose: bool, - /// Log file path. - file: PathBuf, - }, - /// Print event log records. - Events { - /// Reverse the iteration direction. - #[clap(short, long)] - reverse: bool, - /// Log file path. file: PathBuf, }, @@ -66,9 +54,6 @@ pub async fn run(cmd: Command) -> Result<()> { Command::Log { verbose, file } => { verify_log(file, verbose).await?; } - Command::Events { file, reverse } => { - print_events(file, reverse).await?; - } } Ok(()) @@ -109,35 +94,6 @@ async fn verify_log(file: PathBuf, verbose: bool) -> Result<()> { Ok(()) } -/// Print the events of a log file. -async fn print_events(file: PathBuf, reverse: bool) -> Result<()> { - if !vfs::metadata(&file).await?.is_file() { - return Err(Error::NotFile(file)); - } - - let event_log = FolderEventLog::new(&file).await?; - let mut it = if reverse { - event_log.iter().await?.rev() - } else { - event_log.iter().await? - }; - let divider = "-".repeat(80); - - while let Some(record) = it.next_entry().await? { - println!("{}", divider); - println!(" time: {}", record.time()); - println!("before: {}", CommitHash(record.last_commit())); - println!("commit: {}", CommitHash(record.commit())); - let event_buffer = event_log.read_event_buffer(&record).await?; - let event_record: EventRecord = (record, event_buffer).into(); - let event = event_record.decode_event::().await?; - println!(" event: {}", event.event_kind()); - } - println!("{}", divider); - - Ok(()) -} - /// Print a vault header. pub async fn header(vault: PathBuf) -> Result<()> { if !vfs::metadata(&vault).await?.is_file() { diff --git a/src/commands/events.rs b/src/commands/events.rs new file mode 100644 index 0000000000..3754b87c7d --- /dev/null +++ b/src/commands/events.rs @@ -0,0 +1,100 @@ +use clap::Subcommand; +use std::path::PathBuf; + +use binary_stream::futures::{Decodable, Encodable}; +use sos_net::sdk::{ + commit::CommitHash, + events::{ + AccountEvent, AccountEventLog, EventLogFile, EventRecord, FileEvent, + FileEventLog, FolderEventLog, LogEvent, WriteEvent, + }, + vfs, +}; + +use crate::{Error, Result}; + +#[derive(Subcommand, Debug)] +pub enum Command { + /// Print account event log records. + Account { + /// Reverse the iteration direction. + #[clap(short, long)] + reverse: bool, + + /// Log file path. + file: PathBuf, + }, + /// Print folder event log records. + Folder { + /// Reverse the iteration direction. + #[clap(short, long)] + reverse: bool, + + /// Log file path. + file: PathBuf, + }, + /// Print file event log records. + File { + /// Reverse the iteration direction. + #[clap(short, long)] + reverse: bool, + + /// Log file path. + file: PathBuf, + }, +} + +pub async fn run(cmd: Command) -> Result<()> { + match cmd { + Command::Account { file, reverse } => { + if !vfs::metadata(&file).await?.is_file() { + return Err(Error::NotFile(file)); + } + let event_log = AccountEventLog::new_account(&file).await?; + print_events::(event_log, reverse).await?; + } + Command::Folder { file, reverse } => { + if !vfs::metadata(&file).await?.is_file() { + return Err(Error::NotFile(file)); + } + let event_log = FolderEventLog::new_folder(&file).await?; + print_events::(event_log, reverse).await?; + } + Command::File { file, reverse } => { + if !vfs::metadata(&file).await?.is_file() { + return Err(Error::NotFile(file)); + } + let event_log = FileEventLog::new_file(&file).await?; + print_events::(event_log, reverse).await?; + } + } + + Ok(()) +} + +/// Print the events of a log file. +async fn print_events( + event_log: EventLogFile, + reverse: bool, +) -> Result<()> { + let mut it = if reverse { + event_log.iter().await?.rev() + } else { + event_log.iter().await? + }; + let divider = "-".repeat(80); + + while let Some(record) = it.next_entry().await? { + println!("{}", divider); + println!(" time: {}", record.time()); + println!("before: {}", CommitHash(record.last_commit())); + println!("commit: {}", CommitHash(record.commit())); + let event_buffer = event_log.read_event_buffer(&record).await?; + let event_record: EventRecord = (record, event_buffer).into(); + let event = event_record.decode_event::().await?; + println!(" event: {}", event.event_kind()); + } + println!("{}", divider); + + Ok(()) +} diff --git a/src/commands/folder.rs b/src/commands/folder.rs index f4411497ae..edd3408fd3 100644 --- a/src/commands/folder.rs +++ b/src/commands/folder.rs @@ -1,7 +1,9 @@ use clap::Subcommand; use human_bytes::human_bytes; -use sos_net::sdk::{account::AccountRef, hex, vault::FolderRef}; +use sos_net::sdk::{ + account::AccountRef, events::LogEvent, hex, vault::FolderRef, +}; use crate::{ helpers::{ diff --git a/src/commands/mod.rs b/src/commands/mod.rs index b1623a0c63..b5917bb606 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -3,6 +3,7 @@ pub mod audit; pub mod changes; pub mod check; pub mod device; +pub mod events; pub mod folder; pub mod generate_keypair; pub mod secret; @@ -14,5 +15,6 @@ pub use account::Command as AccountCommand; pub use audit::Command as AuditCommand; pub use check::Command as CheckCommand; pub use device::Command as DeviceCommand; +pub use events::Command as EventsCommand; pub use folder::Command as FolderCommand; pub use secret::Command as SecretCommand; diff --git a/src/helpers/account.rs b/src/helpers/account.rs index 598e2c7dfa..9cb3ac4b8c 100644 --- a/src/helpers/account.rs +++ b/src/helpers/account.rs @@ -174,13 +174,16 @@ pub async fn verify(user: Owner) -> Result { /// List local accounts. pub async fn list_accounts(verbose: bool) -> Result<()> { let accounts = AccountsList::list_accounts(None).await?; - for account in accounts { + for account in &accounts { if verbose { println!("{} {}", account.address(), account.label()); } else { println!("{}", account.label()); } } + if accounts.is_empty() { + println!("no accounts yet"); + } Ok(()) } diff --git a/workspace/net/src/client/account/network_account.rs b/workspace/net/src/client/account/network_account.rs index 09a9ac3676..7071703e49 100644 --- a/workspace/net/src/client/account/network_account.rs +++ b/workspace/net/src/client/account/network_account.rs @@ -715,6 +715,9 @@ impl NetworkAccount { label: &str, id: Option<&SecretId>, ) -> Result { - Ok(self.account.document_exists_in_folder(vault_id, label, id).await?) + Ok(self + .account + .document_exists_in_folder(vault_id, label, id) + .await?) } } diff --git a/workspace/net/src/server/backend.rs b/workspace/net/src/server/backend.rs index 4c5ba050aa..318408c6e8 100644 --- a/workspace/net/src/server/backend.rs +++ b/workspace/net/src/server/backend.rs @@ -217,8 +217,10 @@ impl FileSystemBackend { let id = *summary.id(); let mut event_log_file = - FolderEventLog::new(&event_log_path) - .await?; + FolderEventLog::new_folder( + &event_log_path, + ) + .await?; event_log_file.load_tree().await?; // Store these file paths so locks @@ -265,7 +267,8 @@ impl FileSystemBackend { vfs::write(&vault_path, vault).await?; // Create the event log file - let mut event_log = FolderEventLog::new(&event_log_path).await?; + let mut event_log = + FolderEventLog::new_folder(&event_log_path).await?; let event = WriteEvent::CreateVault(vault.to_vec()); event_log.append_event(&event).await?; @@ -477,7 +480,8 @@ impl BackendHandler for FileSystemBackend { // Prepare a temp file with the new event log records let temp = NamedTempFile::new()?; - let mut temp_event_log = FolderEventLog::new(temp.path()).await?; + let mut temp_event_log = + FolderEventLog::new_folder(temp.path()).await?; temp_event_log.apply(events.iter().collect()).await?; let expected_root = temp_event_log @@ -624,7 +628,8 @@ impl BackendHandler for FileSystemBackend { .get_mut(vault_id) .ok_or_else(|| Error::VaultNotExist(*vault_id))?; - *event_log = FolderEventLog::new(&original_event_log).await?; + *event_log = + FolderEventLog::new_folder(&original_event_log).await?; event_log.load_tree().await?; let root = event_log .tree() diff --git a/workspace/sdk/Cargo.toml b/workspace/sdk/Cargo.toml index 2ef270ffd8..ad12b0d340 100644 --- a/workspace/sdk/Cargo.toml +++ b/workspace/sdk/Cargo.toml @@ -22,6 +22,7 @@ device = [] security-report = [] [dependencies] +binary-stream.workspace = true sos-vfs = "0.2" async-trait = "0.1" async-recursion = "1" @@ -79,11 +80,6 @@ mpc-protocol = "0.4.1" vsss-rs = {version = "3", optional = true } -[dependencies.binary-stream] -version = "8.0.0" -features = ["async"] -#path = "../../../../binary-stream" - [dependencies.probly-search] version = "2.0.0-alpha-2" #path = "../../../../forks/probly-search" diff --git a/workspace/sdk/src/account/account.rs b/workspace/sdk/src/account/account.rs index 6d43d7e1e4..ac6cffb5d7 100644 --- a/workspace/sdk/src/account/account.rs +++ b/workspace/sdk/src/account/account.rs @@ -362,10 +362,10 @@ impl Account { user, storage: Arc::new(RwLock::new(storage)), account_log: Arc::new(RwLock::new( - AccountEventLog::new(account_events).await?, + AccountEventLog::new_account(account_events).await?, )), file_log: Arc::new(RwLock::new( - FileEventLog::new(file_events).await?, + FileEventLog::new_file(file_events).await?, )), }); diff --git a/workspace/sdk/src/account/archive/backup.rs b/workspace/sdk/src/account/archive/backup.rs index df9d3111d4..28018c0e12 100644 --- a/workspace/sdk/src/account/archive/backup.rs +++ b/workspace/sdk/src/account/archive/backup.rs @@ -541,7 +541,7 @@ impl AccountBackup { let create_vault = WriteEvent::CreateVault(buffer.clone()); event_log_events.push(create_vault); let mut event_log = - FolderEventLog::new(event_log_path).await?; + FolderEventLog::new_folder(event_log_path).await?; event_log.apply(event_log_events.iter().collect()).await?; } diff --git a/workspace/sdk/src/account/local_storage.rs b/workspace/sdk/src/account/local_storage.rs index a7a650fb2b..49ac8738e5 100644 --- a/workspace/sdk/src/account/local_storage.rs +++ b/workspace/sdk/src/account/local_storage.rs @@ -332,7 +332,8 @@ impl FolderStorage { vault: Option, ) -> Result<()> { let event_log_path = self.event_log_path(summary); - let mut event_log = FolderEventLog::new(&event_log_path).await?; + let mut event_log = + FolderEventLog::new_folder(&event_log_path).await?; if let Some(vault) = &vault { // Must truncate the event log so that importing vaults diff --git a/workspace/sdk/src/commit/integrity.rs b/workspace/sdk/src/commit/integrity.rs index 496a550d9e..6101dda1a7 100644 --- a/workspace/sdk/src/commit/integrity.rs +++ b/workspace/sdk/src/commit/integrity.rs @@ -84,7 +84,8 @@ where let mut file = vfs::File::open(event_log_file.as_ref()).await?.compat(); let mut reader = BinaryReader::new(&mut file, encoding_options()); - let event_log = FolderEventLog::new(event_log_file.as_ref()).await?; + let event_log = + FolderEventLog::new_folder(event_log_file.as_ref()).await?; let mut it = event_log.iter().await?; let mut last_checksum: Option<[u8; 32]> = None; diff --git a/workspace/sdk/src/constants.rs b/workspace/sdk/src/constants.rs index dd36f364e3..335762ec21 100644 --- a/workspace/sdk/src/constants.rs +++ b/workspace/sdk/src/constants.rs @@ -14,8 +14,14 @@ mod identity { /// Audit log identity magic bytes (SOSA). pub const AUDIT_IDENTITY: [u8; 4] = [0x53, 0x4F, 0x53, 0x41]; - /// Write-ahead log identity magic bytes (SOSW). - pub const EVENT_LOG_IDENTITY: [u8; 4] = [0x53, 0x4F, 0x53, 0x57]; + /// Account event log identity magic bytes (SOSU). + pub const ACCOUNT_EVENT_LOG_IDENTITY: [u8; 4] = [0x53, 0x4F, 0x53, 0x55]; + + /// Folder event log identity magic bytes (SOSW). + pub const FOLDER_EVENT_LOG_IDENTITY: [u8; 4] = [0x53, 0x4F, 0x53, 0x57]; + + /// File event log identity magic bytes (SOSF). + pub const FILE_EVENT_LOG_IDENTITY: [u8; 4] = [0x53, 0x4F, 0x53, 0x46]; /// Patch file identity magic bytes (SOSP). pub const PATCH_IDENTITY: [u8; 4] = [0x53, 0x4F, 0x53, 0x50]; diff --git a/workspace/sdk/src/encoding/v1/events.rs b/workspace/sdk/src/encoding/v1/events.rs index 1556dc67cd..0d55ff5c02 100644 --- a/workspace/sdk/src/encoding/v1/events.rs +++ b/workspace/sdk/src/encoding/v1/events.rs @@ -5,7 +5,7 @@ use crate::{ encoding::encoding_error, events::{ AccountEvent, AuditData, AuditEvent, AuditLogFile, EventKind, - EventRecord, FileEvent, LogFlags, Patch, WriteEvent, + EventRecord, FileEvent, LogEvent, LogFlags, Patch, WriteEvent, }, formats::{EventLogFileRecord, FileIdentity, FileRecord, VaultRecord}, vault::{secret::SecretId, VaultCommit}, diff --git a/workspace/sdk/src/events/event.rs b/workspace/sdk/src/events/event.rs index bae70e9887..958443968b 100644 --- a/workspace/sdk/src/events/event.rs +++ b/workspace/sdk/src/events/event.rs @@ -1,6 +1,6 @@ //! Encoding of all operations. -use super::{AuditEvent, EventKind, ReadEvent, WriteEvent}; +use super::{AuditEvent, EventKind, LogEvent, ReadEvent, WriteEvent}; use crate::{vault::VaultId, Error, Result}; /// Events generated in the context of an account. @@ -20,9 +20,8 @@ pub enum AccountEvent { DeleteFolder(VaultId), } -impl AccountEvent { - /// Get the event kind for this event. - pub fn event_kind(&self) -> EventKind { +impl LogEvent for AccountEvent { + fn event_kind(&self) -> EventKind { match self { Self::Noop => EventKind::Noop, Self::CreateFolder(_) => EventKind::CreateVault, diff --git a/workspace/sdk/src/events/file.rs b/workspace/sdk/src/events/file.rs index 02f761e281..2661516ed9 100644 --- a/workspace/sdk/src/events/file.rs +++ b/workspace/sdk/src/events/file.rs @@ -1,4 +1,5 @@ //! Event for modifications to external files. +use super::{EventKind, LogEvent}; use crate::vault::{secret::SecretId, VaultId}; /// File event records changes to external files @@ -17,3 +18,18 @@ pub enum FileEvent { /// File was deleted. DeleteFile(VaultId, SecretId, String), } + +impl LogEvent for FileEvent { + fn event_kind(&self) -> EventKind { + todo!(); + + /* + match self { + Self::Noop => EventKind::Noop, + Self::CreateFolder(_) => EventKind::CreateVault, + Self::UpdateFolder(_) => EventKind::UpdateVault, + Self::DeleteFolder(_) => EventKind::DeleteVault, + } + */ + } +} diff --git a/workspace/sdk/src/events/log/file.rs b/workspace/sdk/src/events/log/file.rs index 2e5d828242..c972ed9bf8 100644 --- a/workspace/sdk/src/events/log/file.rs +++ b/workspace/sdk/src/events/log/file.rs @@ -18,9 +18,12 @@ use crate::{ commit::{ event_log_commit_tree_file, CommitHash, CommitState, CommitTree, }, - constants::EVENT_LOG_IDENTITY, + constants::{ + ACCOUNT_EVENT_LOG_IDENTITY, FILE_EVENT_LOG_IDENTITY, + FOLDER_EVENT_LOG_IDENTITY, + }, encode, - encoding::encoding_options, + encoding::{encoding_options, VERSION}, events::{AccountEvent, FileEvent, Patch, WriteEvent}, formats::{ event_log_stream, patch_stream, EventLogFileRecord, @@ -61,28 +64,17 @@ where file_path: PathBuf, file: File, tree: CommitTree, - identity: [u8; 4], + identity: &'static [u8], + version: Option, phantom: std::marker::PhantomData, } impl EventLogFile { - /// Create a new event log file. - pub async fn new>(file_path: P) -> Result { - let file = - Self::create(file_path.as_ref(), &EVENT_LOG_IDENTITY).await?; - Ok(Self { - file, - file_path: file_path.as_ref().to_path_buf(), - tree: Default::default(), - identity: EVENT_LOG_IDENTITY, - phantom: std::marker::PhantomData, - }) - } - /// Create the event log file. async fn create>( path: P, - identity: &[u8], + identity: &'static [u8], + encoding_version: Option, ) -> Result { let mut file = OpenOptions::new() .create(true) @@ -92,7 +84,11 @@ impl EventLogFile { let size = file.metadata().await?.len(); if size == 0 { - file.write_all(identity).await?; + let mut header = identity.to_vec(); + if let Some(version) = encoding_version { + header.extend_from_slice(&version.to_le_bytes()); + } + file.write_all(&header).await?; file.flush().await?; } Ok(file) @@ -119,6 +115,32 @@ impl EventLogFile { Ok((commit, record)) } + /// Length of the file magic bytes and optional + /// encoding version. + fn header_len(&self) -> usize { + let mut len = self.identity.len(); + if let Some(version) = self.version { + len += (u16::BITS / 8) as usize; + } + len + } + + /// Header bytes. + fn header(&self) -> Vec { + let mut header = self.identity.to_vec(); + if let Some(version) = self.version { + header.extend_from_slice(&version.to_le_bytes()); + } + header + } + + /// Get an iterator of the log records. + pub async fn iter(&self) -> Result { + let content_offset = self.header_len() as u64; + event_log_stream( + &self.file_path, self.identity, content_offset).await + } + /// Replace this event log with the contents of the buffer. /// /// The buffer should start with the event log identity bytes. @@ -133,7 +155,7 @@ impl EventLogFile { /// The buffer should start with the event log identity bytes. pub async fn append_buffer(&mut self, buffer: Vec) -> Result<()> { // Get buffer of log records after the identity bytes - let buffer = &buffer[self.identity.len()..]; + let buffer = &buffer[self.header_len()..]; let mut file = OpenOptions::new() .write(true) @@ -155,7 +177,7 @@ impl EventLogFile { /// Get the tail after the given item until the end of the log. pub async fn tail(&self, item: EventLogFileRecord) -> Result> { - let mut partial = self.identity.to_vec(); + let mut partial = self.header(); let start = item.offset().end as usize; let mut file = File::open(&self.file_path).await?; let end = file.metadata().await?.len() as usize; @@ -299,23 +321,17 @@ impl EventLogFile { Ok(()) } - /// Get an iterator of the log records. - pub async fn iter(&self) -> Result { - if self.identity == EVENT_LOG_IDENTITY { - event_log_stream(&self.file_path).await - } else { - patch_stream(&self.file_path).await - } - } - - /// Get the last commit hash. + /// Read the last commit hash from the file. pub async fn last_commit(&self) -> Result> { - let mut it = self.iter().await?.rev(); - if let Some(record) = it.next_entry().await? { - Ok(Some(CommitHash(record.commit()))) - } else { - Ok(None) - } + let file_len = self.file.metadata().await?.len() as usize; + if file_len > self.header_len() { + let mut it = self.iter().await?.rev(); + if let Some(record) = it.next_entry().await? { + Ok(Some(CommitHash(record.commit()))) + } else { + Ok(None) + } + } else { Ok(None) } } /// Get a diff of the records after the record with the @@ -395,7 +411,48 @@ impl EventLogFile { } } +impl EventLogFile { + /// Create a new account event log file. + pub async fn new_account>(file_path: P) -> Result { + let file = Self::create( + file_path.as_ref(), + &ACCOUNT_EVENT_LOG_IDENTITY, + Some(VERSION), + ) + .await?; + Ok(Self { + file, + file_path: file_path.as_ref().to_path_buf(), + tree: Default::default(), + identity: &ACCOUNT_EVENT_LOG_IDENTITY, + version: Some(VERSION), + phantom: std::marker::PhantomData, + }) + } +} + impl EventLogFile { + /// Create a new folder event log file. + pub async fn new_folder>(file_path: P) -> Result { + // Note that for backwards compatibility we don't + // encode a version, later we will need to upgrade + // the encoding to include a version + let file = Self::create( + file_path.as_ref(), + &FOLDER_EVENT_LOG_IDENTITY, + None, + ) + .await?; + Ok(Self { + file, + file_path: file_path.as_ref().to_path_buf(), + tree: Default::default(), + identity: &FOLDER_EVENT_LOG_IDENTITY, + version: None, + phantom: std::marker::PhantomData, + }) + } + /// Get a copy of this event log compacted. pub async fn compact(&self) -> Result<(Self, u64, u64)> { let old_size = self.path().metadata()?.len(); @@ -406,7 +463,7 @@ impl EventLogFile { let temp = NamedTempFile::new()?; // Apply them to a temporary event log file - let mut temp_event_log = Self::new(temp.path()).await?; + let mut temp_event_log = Self::new_folder(temp.path()).await?; temp_event_log.apply(events.iter().collect()).await?; let new_size = temp_event_log.file().metadata().await?.len(); @@ -422,7 +479,7 @@ impl EventLogFile { // determine whether to rename or copy. vfs::copy(temp.path(), self.path()).await?; - let mut new_event_log = Self::new(self.path()).await?; + let mut new_event_log = Self::new_folder(self.path()).await?; new_event_log.load_tree().await?; // Verify the new event log tree @@ -435,17 +492,45 @@ impl EventLogFile { } } +impl EventLogFile { + /// Create a new file event log file. + pub async fn new_file>(file_path: P) -> Result { + let file = Self::create( + file_path.as_ref(), + &FILE_EVENT_LOG_IDENTITY, + Some(VERSION), + ) + .await?; + Ok(Self { + file, + file_path: file_path.as_ref().to_path_buf(), + tree: Default::default(), + identity: &FILE_EVENT_LOG_IDENTITY, + version: Some(VERSION), + phantom: std::marker::PhantomData, + }) + } +} + #[cfg(test)] mod test { use anyhow::Result; use tempfile::NamedTempFile; use super::*; - use crate::{events::WriteEvent, test_utils::*}; + use crate::{events::WriteEvent, test_utils::*, vault::VaultId}; + + async fn mock_account_event_log() -> Result<(NamedTempFile, AccountEventLog)> + { + let temp = NamedTempFile::new()?; + let event_log = AccountEventLog::new_account(temp.path()).await?; + Ok((temp, event_log)) + } - async fn mock_event_log() -> Result<(NamedTempFile, FolderEventLog)> { + async fn mock_folder_event_log() -> Result<(NamedTempFile, FolderEventLog)> + { let temp = NamedTempFile::new()?; - let event_log = EventLogFile::new(temp.path()).await?; + let event_log = FolderEventLog::new_folder(temp.path()).await?; Ok((temp, event_log)) } @@ -454,7 +539,7 @@ mod test { let (encryption_key, _, _) = mock_encryption_key()?; let (_, mut vault, buffer) = mock_vault_file().await?; - let (temp, mut event_log) = mock_event_log().await?; + let (temp, mut event_log) = mock_folder_event_log().await?; let mut commits = Vec::new(); @@ -489,7 +574,7 @@ mod test { } #[tokio::test] - async fn event_log_iter_forward() -> Result<()> { + async fn folder_event_log_iter_forward() -> Result<()> { let (temp, event_log, commits) = mock_event_log_file().await?; let mut it = event_log.iter().await?; let first_row = it.next_entry().await?.unwrap(); @@ -506,7 +591,7 @@ mod test { } #[tokio::test] - async fn event_log_iter_backward() -> Result<()> { + async fn folder_event_log_iter_backward() -> Result<()> { let (temp, event_log, _) = mock_event_log_file().await?; let mut it = event_log.iter().await?.rev(); let _third_row = it.next_entry().await?.unwrap(); @@ -519,7 +604,7 @@ mod test { #[tokio::test] async fn event_log_last_commit() -> Result<()> { - let (temp, mut event_log) = mock_event_log().await?; + let (temp, mut event_log) = mock_folder_event_log().await?; let (_, _vault, buffer) = mock_vault_file().await?; assert!(event_log.last_commit().await?.is_none()); @@ -541,4 +626,22 @@ mod test { temp.close()?; Ok(()) } + + #[tokio::test] + async fn account_event_log() -> Result<()> { + let (temp, mut event_log) = mock_account_event_log().await?; + + let folder = VaultId::new_v4(); + event_log.apply(vec![ + &AccountEvent::CreateFolder(folder), + &AccountEvent::DeleteFolder(folder), + ]).await?; + + assert!(event_log.tree().len() > 0); + assert!(event_log.tree().root().is_some()); + assert!(event_log.last_commit().await.is_ok()); + + temp.close()?; + Ok(()) + } } diff --git a/workspace/sdk/src/events/log/mod.rs b/workspace/sdk/src/events/log/mod.rs index 454b4dfb05..5c90e392ea 100644 --- a/workspace/sdk/src/events/log/mod.rs +++ b/workspace/sdk/src/events/log/mod.rs @@ -101,7 +101,7 @@ mod test { let (id, data) = mock_secret().await?; // Create a simple event log - let mut server = EventLogFile::new(path).await?; + let mut server = EventLogFile::new_folder(path).await?; server .apply(vec![ &WriteEvent::CreateVault(vault_buffer), @@ -138,7 +138,7 @@ mod test { let (id, data) = mock_secret().await?; // Create a simple event log - let mut server = EventLogFile::new(&server_file).await?; + let mut server = EventLogFile::new_folder(&server_file).await?; server .apply(vec![ &WriteEvent::CreateVault(vault_buffer), @@ -147,7 +147,7 @@ mod test { .await?; // Duplicate the server events on the client - let mut client = EventLogFile::new(&client_file).await?; + let mut client = EventLogFile::new_folder(&client_file).await?; let mut it = server.iter().await?; while let Some(record) = it.next_entry().await? { let event = server.event_data(&record).await?; @@ -224,7 +224,8 @@ mod test { assert_eq!(vec![1], indices); let leaf = leaves.first().unwrap(); if let Some(buffer) = server.diff(*leaf).await? { - let mut partial_log = FolderEventLog::new(&partial).await?; + let mut partial_log = + FolderEventLog::new_folder(&partial).await?; partial_log.write_buffer(&buffer).await?; let mut records = Vec::new(); let mut it = partial_log.iter().await?; @@ -253,7 +254,7 @@ mod test { async fn event_log_file_load() -> Result<()> { mock_event_log_standalone().await?; let path = PathBuf::from(MOCK_LOG); - let event_log = FolderEventLog::new(path).await?; + let event_log = FolderEventLog::new_folder(path).await?; let mut it = event_log.iter().await?; while let Some(record) = it.next_entry().await? { let _event = event_log.event_data(&record).await?; diff --git a/workspace/sdk/src/events/log/reducer.rs b/workspace/sdk/src/events/log/reducer.rs index af304f20e0..c6d7602460 100644 --- a/workspace/sdk/src/events/log/reducer.rs +++ b/workspace/sdk/src/events/log/reducer.rs @@ -209,7 +209,7 @@ mod test { let (_, mut vault, buffer) = mock_vault_file().await?; let temp = NamedTempFile::new()?; - let mut event_log = FolderEventLog::new(temp.path()).await?; + let mut event_log = FolderEventLog::new_folder(temp.path()).await?; let mut commits = Vec::new(); @@ -313,7 +313,8 @@ mod test { assert_eq!(2, events.len()); let compact_temp = NamedTempFile::new()?; - let mut compact = FolderEventLog::new(compact_temp.path()).await?; + let mut compact = + FolderEventLog::new_folder(compact_temp.path()).await?; for event in events { compact.append_event(&event).await?; } diff --git a/workspace/sdk/src/events/mod.rs b/workspace/sdk/src/events/mod.rs index c0810dfcd6..136dc30bf0 100644 --- a/workspace/sdk/src/events/mod.rs +++ b/workspace/sdk/src/events/mod.rs @@ -26,6 +26,12 @@ pub use read::ReadEvent; pub use types::EventKind; pub use write::WriteEvent; +/// Trait for events that can be written to an event log.. +pub trait LogEvent { + /// Get the event kind for this event. + fn event_kind(&self) -> EventKind; +} + /// Patch wraps a changeset of events to be sent across the network. #[derive(Clone, Debug, Default)] pub struct Patch(pub Vec); diff --git a/workspace/sdk/src/events/write.rs b/workspace/sdk/src/events/write.rs index d0522a099d..4515e1009c 100644 --- a/workspace/sdk/src/events/write.rs +++ b/workspace/sdk/src/events/write.rs @@ -8,7 +8,7 @@ use crate::{ vault::{secret::SecretId, VaultCommit}, }; -use super::EventKind; +use super::{EventKind, LogEvent}; /// Write operations. #[derive(Default, Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] @@ -76,9 +76,8 @@ impl PartialOrd for WriteEvent { } } -impl WriteEvent { - /// Get the event kind for this event. - pub fn event_kind(&self) -> EventKind { +impl LogEvent for WriteEvent { + fn event_kind(&self) -> EventKind { match self { WriteEvent::Noop => EventKind::Noop, WriteEvent::CreateVault(_) => EventKind::CreateVault, diff --git a/workspace/sdk/src/formats/mod.rs b/workspace/sdk/src/formats/mod.rs index 8131abad05..36d69b72f3 100644 --- a/workspace/sdk/src/formats/mod.rs +++ b/workspace/sdk/src/formats/mod.rs @@ -8,9 +8,7 @@ pub use records::{EventLogFileRecord, FileItem, FileRecord, VaultRecord}; pub use stream::FormatStream; use crate::{ - constants::{ - AUDIT_IDENTITY, EVENT_LOG_IDENTITY, PATCH_IDENTITY, VAULT_IDENTITY, - }, + constants::{AUDIT_IDENTITY, PATCH_IDENTITY, VAULT_IDENTITY}, vault::Header, vfs::File, Result, @@ -65,14 +63,16 @@ pub async fn vault_stream_buffer<'a>( /// Get a stream for a event log file. pub async fn event_log_stream>( path: P, + identity: &'static [u8], + content_offset: u64, ) -> Result { - FileIdentity::read_file(path.as_ref(), &EVENT_LOG_IDENTITY).await?; + FileIdentity::read_file(path.as_ref(), &identity).await?; let read_stream = File::open(path.as_ref()).await?.compat(); FormatStream::>::new_file( read_stream, - &EVENT_LOG_IDENTITY, + &identity, true, - None, + Some(content_offset), ) .await } @@ -80,14 +80,16 @@ pub async fn event_log_stream>( /// Get a stream for a vault file buffer. pub async fn event_log_stream_buffer<'a>( buffer: &'a [u8], + identity: &'static [u8], + content_offset: u64, ) -> Result> { - FileIdentity::read_slice(&buffer, &EVENT_LOG_IDENTITY)?; + FileIdentity::read_slice(&buffer, identity)?; let read_stream = BufReader::new(Cursor::new(buffer)); FormatStream::>::new_buffer( read_stream, - &EVENT_LOG_IDENTITY, + identity, true, - None, + Some(content_offset), ) .await } diff --git a/workspace/sdk/src/formats/stream.rs b/workspace/sdk/src/formats/stream.rs index 3aa409b3e2..d46bfd86dc 100644 --- a/workspace/sdk/src/formats/stream.rs +++ b/workspace/sdk/src/formats/stream.rs @@ -64,6 +64,8 @@ impl FormatStream> { let header_offset = header_offset.unwrap_or(identity.len() as u64); read_stream.seek(SeekFrom::Start(header_offset)).await?; + println!("HEADER OFFSET {}", header_offset); + Ok(Self { header_offset, data_length_prefix, diff --git a/workspace/sdk/src/test_utils.rs b/workspace/sdk/src/test_utils.rs index e0d049e35f..333f362f9a 100644 --- a/workspace/sdk/src/test_utils.rs +++ b/workspace/sdk/src/test_utils.rs @@ -136,7 +136,7 @@ pub async fn mock_event_log_file( let (_, mut vault, buffer) = mock_vault_file().await?; let temp = NamedTempFile::new()?; - let mut event_log = FolderEventLog::new(temp.path()).await?; + let mut event_log = FolderEventLog::new_folder(temp.path()).await?; let mut commits = Vec::new();