From cdbe0aa86ca4dcc86faecd34c9c05b670180e16c Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Sun, 4 Aug 2024 02:49:15 +0200 Subject: [PATCH 01/13] implement mount creation --- src/fs.rs | 58 ++++++++++++++++++++++++++++++ src/runtime/fd.rs | 5 ++- src/runtime/structure_helpers.rs | 62 +++++++++++++++++++++++++++++++- src/storage/dummy.rs | 1 + src/storage/stable.rs | 4 ++- src/storage/transient.rs | 2 ++ src/storage/types.rs | 1 + src/test_utils.rs | 9 +++-- 8 files changed, 137 insertions(+), 5 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index ee4b69a..7767cac 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1,3 +1,8 @@ +use std::collections::HashMap; + +use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES; +use ic_stable_structures::Memory; + use crate::{ error::Error, runtime::{ @@ -23,6 +28,7 @@ pub struct FileSystem { root_fd: Fd, fd_table: FdTable, pub storage: Box, + mounted_nodes: HashMap>, } impl FileSystem { @@ -35,6 +41,7 @@ impl FileSystem { root_fd: 0, fd_table, storage, + mounted_nodes: HashMap::new(), }); } @@ -46,6 +53,7 @@ impl FileSystem { root_fd, fd_table, storage, + mounted_nodes: HashMap::new(), }) } @@ -97,6 +105,56 @@ impl FileSystem { } } + pub fn mount_memory_file( + &mut self, + filename: &str, + memory: Box, + ) -> Result<(), Error> { + // create a file for the mount + let fd = self.open_or_create( + self.root_fd, + filename, + FdStat::default(), + OpenFlags::CREATE, + 0, + )?; + + let mut meta = self.metadata(fd)?; + + let node = self.get_node(fd)?; + + self.mounted_nodes.insert(node, memory); + + if meta.mount_size.is_none() { + // init new memory with 0 if it was not known before + meta.mount_size = Some(0); + } + + self.storage.put_metadata(node, meta); + + self.close(fd)?; + + Ok(()) + } + + pub fn set_memory_file_size( + &mut self, + filename: &str, + new_size: FileSize, + ) -> Result<(), Error> { + let root_node = self.get_node(self.root_fd())?; + + let node = find_node(root_node, filename, self.storage.as_ref())?; + + let mut metadata = self.metadata_from_node(node)?; + + metadata.mount_size = Some(new_size); + + self.storage.put_metadata(node, metadata); + + Ok(()) + } + // Get dir entry for a given directory and the directory index. pub fn get_direntry(&self, fd: Fd, index: DirEntryIndex) -> Result { self.get_dir(fd)?.get_entry(index, self.storage.as_ref()) diff --git a/src/runtime/fd.rs b/src/runtime/fd.rs index 4bcabe6..256c2ab 100644 --- a/src/runtime/fd.rs +++ b/src/runtime/fd.rs @@ -17,7 +17,7 @@ pub enum FdEntry { // pub struct FdTable { - // currentlty open file descriptors. + // currently open file descriptors. table: BTreeMap, // backward links to see how many file descriptors are currently pointing to any particular node. node_refcount: BTreeMap, @@ -51,10 +51,12 @@ impl FdTable { // Update a file descriptor entry, it returns the old entry if existed. pub fn insert(&mut self, fd: Fd, entry: FdEntry) -> Option { self.inc_node_refcount(&entry); + let prev_entry = self.table.insert(fd, entry); if let Some(prev_entry) = prev_entry.as_ref() { self.dec_node_refcount(prev_entry); } + prev_entry } @@ -73,6 +75,7 @@ impl FdTable { fd } }; + let prev = self.insert(fd, entry); assert!(prev.is_none()); fd diff --git a/src/runtime/structure_helpers.rs b/src/runtime/structure_helpers.rs index 1fba20a..63621dc 100644 --- a/src/runtime/structure_helpers.rs +++ b/src/runtime/structure_helpers.rs @@ -15,6 +15,27 @@ struct EntryFindResult { next_entry: Option, } +pub fn unify_name(filename: &str) -> String { + let parts = filename.split('/'); + + let mut name_parts = Vec::new(); + + for part in parts { + if part.is_empty() || part == "." { + continue; + } + + if part == ".." { + name_parts.pop(); + continue; + } + + name_parts.push(part); + } + + name_parts.join("/") +} + fn find_node_with_index( parent_dir_node: Node, path: &str, @@ -126,6 +147,7 @@ pub fn create_dir_entry( }, first_dir_entry: None, last_dir_entry: None, + mount_size: None, }, ); @@ -390,7 +412,7 @@ mod tests { use crate::{ error::Error, - runtime::structure_helpers::{create_path, find_node}, + runtime::structure_helpers::{create_path, find_node, unify_name}, storage::{stable::StableStorage, types::FileType, Storage}, }; @@ -661,4 +683,42 @@ mod tests { find_node(root_node, "test1/test2/test4/test7.txt", storage).unwrap() ); } + + #[test] + fn unify_name_correct() { + let name = "test/text.txt".to_string(); + + let name1 = unify_name("./test/.//text.txt"); + let name2 = unify_name("/test//./text.txt"); + let name3 = unify_name("test/text.txt"); + let name4 = unify_name("test/a/s/../../text.txt"); + let name5 = unify_name("../test/text.txt"); + let name6 = unify_name("a/./s/../../test/text.txt"); + + assert_eq!(name1, name); + assert_eq!(name2, name); + assert_eq!(name3, name); + assert_eq!(name4, name); + assert_eq!(name5, name); + assert_eq!(name6, name); + } + + #[test] + fn empty_path() { + let name = "".to_string(); + + let name1 = unify_name("././"); + let name2 = unify_name(""); + let name3 = unify_name("."); + let name4 = unify_name("../a/s/../.."); + let name5 = unify_name("../test/text.txt/../.."); + let name6 = unify_name("a/./s/../..///"); + + assert_eq!(name1, name); + assert_eq!(name2, name); + assert_eq!(name3, name); + assert_eq!(name4, name); + assert_eq!(name5, name); + assert_eq!(name6, name); + } } diff --git a/src/storage/dummy.rs b/src/storage/dummy.rs index a998986..3485174 100644 --- a/src/storage/dummy.rs +++ b/src/storage/dummy.rs @@ -112,6 +112,7 @@ mod tests { times: Times::default(), first_dir_entry: Some(42), last_dir_entry: Some(24), + mount_size: None, }, ) } diff --git a/src/storage/stable.rs b/src/storage/stable.rs index e652236..73f3513 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -82,7 +82,7 @@ impl StableStorage { ) } - pub fn new_with_custom_memories( + fn new_with_custom_memories( header: VirtualMemory, metadata: VirtualMemory, direntry: VirtualMemory, @@ -118,6 +118,7 @@ impl StableStorage { times: Times::default(), first_dir_entry: None, last_dir_entry: None, + mount_size: None, }; result.put_metadata(ROOT_NODE, metadata); } @@ -285,6 +286,7 @@ mod tests { times: Times::default(), first_dir_entry: Some(42), last_dir_entry: Some(24), + mount_size: None, }, ); let metadata = storage.get_metadata(node).unwrap(); diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 43a5d17..2ca0533 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -40,6 +40,7 @@ impl TransientStorage { times: Times::default(), first_dir_entry: None, last_dir_entry: None, + mount_size: None, }; let mut result = Self { metadata: Default::default(), @@ -196,6 +197,7 @@ mod tests { times: Times::default(), first_dir_entry: None, last_dir_entry: None, + mount_size: None, }, ); storage.write_filechunk(node, 0, 0, &[42; 10]); diff --git a/src/storage/types.rs b/src/storage/types.rs index 2e96aea..a54c807 100644 --- a/src/storage/types.rs +++ b/src/storage/types.rs @@ -76,6 +76,7 @@ pub struct Metadata { pub times: Times, pub first_dir_entry: Option, pub last_dir_entry: Option, + pub mount_size: Option, } impl ic_stable_structures::Storable for Metadata { diff --git a/src/test_utils.rs b/src/test_utils.rs index 77f71c1..d7e6a01 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -4,8 +4,13 @@ use crate::{error::Error, fs::FileSystem, storage::stable::StableStorage}; #[cfg(test)] pub fn test_fs() -> FileSystem { - let storage = StableStorage::new(DefaultMemoryImpl::default()); - FileSystem::new(Box::new(storage)).unwrap() + let memory = DefaultMemoryImpl::default(); + + let storage = StableStorage::new(memory); + + let fs = FileSystem::new(Box::new(storage)).unwrap(); + + fs } #[cfg(test)] From 476d1982d170e2ce5185fc946fca045029ced726 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Sun, 4 Aug 2024 02:50:31 +0200 Subject: [PATCH 02/13] clean-up --- src/runtime/structure_helpers.rs | 60 +------------------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/src/runtime/structure_helpers.rs b/src/runtime/structure_helpers.rs index 63621dc..1cbaf52 100644 --- a/src/runtime/structure_helpers.rs +++ b/src/runtime/structure_helpers.rs @@ -15,27 +15,6 @@ struct EntryFindResult { next_entry: Option, } -pub fn unify_name(filename: &str) -> String { - let parts = filename.split('/'); - - let mut name_parts = Vec::new(); - - for part in parts { - if part.is_empty() || part == "." { - continue; - } - - if part == ".." { - name_parts.pop(); - continue; - } - - name_parts.push(part); - } - - name_parts.join("/") -} - fn find_node_with_index( parent_dir_node: Node, path: &str, @@ -412,7 +391,7 @@ mod tests { use crate::{ error::Error, - runtime::structure_helpers::{create_path, find_node, unify_name}, + runtime::structure_helpers::{create_path, find_node}, storage::{stable::StableStorage, types::FileType, Storage}, }; @@ -684,41 +663,4 @@ mod tests { ); } - #[test] - fn unify_name_correct() { - let name = "test/text.txt".to_string(); - - let name1 = unify_name("./test/.//text.txt"); - let name2 = unify_name("/test//./text.txt"); - let name3 = unify_name("test/text.txt"); - let name4 = unify_name("test/a/s/../../text.txt"); - let name5 = unify_name("../test/text.txt"); - let name6 = unify_name("a/./s/../../test/text.txt"); - - assert_eq!(name1, name); - assert_eq!(name2, name); - assert_eq!(name3, name); - assert_eq!(name4, name); - assert_eq!(name5, name); - assert_eq!(name6, name); - } - - #[test] - fn empty_path() { - let name = "".to_string(); - - let name1 = unify_name("././"); - let name2 = unify_name(""); - let name3 = unify_name("."); - let name4 = unify_name("../a/s/../.."); - let name5 = unify_name("../test/text.txt/../.."); - let name6 = unify_name("a/./s/../..///"); - - assert_eq!(name1, name); - assert_eq!(name2, name); - assert_eq!(name3, name); - assert_eq!(name4, name); - assert_eq!(name5, name); - assert_eq!(name6, name); - } } From 65b9cad7d76886f4a61759062f69cae1f61276b9 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Sun, 4 Aug 2024 08:04:35 +0200 Subject: [PATCH 03/13] clean-up --- src/fs.rs | 1 - src/runtime/structure_helpers.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index 7767cac..8e90437 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; -use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES; use ic_stable_structures::Memory; use crate::{ diff --git a/src/runtime/structure_helpers.rs b/src/runtime/structure_helpers.rs index 1cbaf52..695f333 100644 --- a/src/runtime/structure_helpers.rs +++ b/src/runtime/structure_helpers.rs @@ -662,5 +662,4 @@ mod tests { find_node(root_node, "test1/test2/test4/test7.txt", storage).unwrap() ); } - } From d811f74b1fd8f03bcaf32fe9d2c9fd32bbd577e6 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Sun, 4 Aug 2024 08:08:21 +0200 Subject: [PATCH 04/13] clippy --- src/test_utils.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index d7e6a01..de96eef 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -7,10 +7,7 @@ pub fn test_fs() -> FileSystem { let memory = DefaultMemoryImpl::default(); let storage = StableStorage::new(memory); - - let fs = FileSystem::new(Box::new(storage)).unwrap(); - - fs + FileSystem::new(Box::new(storage)).unwrap() } #[cfg(test)] From 6e16be25f84625d393dd5956048babfcf472dde0 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Sun, 4 Aug 2024 23:59:21 +0200 Subject: [PATCH 05/13] mounte memory with some basic examples --- src/error.rs | 2 + src/fs.rs | 113 +++++++++++-------- src/runtime/structure_helpers.rs | 1 - src/storage.rs | 13 +++ src/storage/dummy.rs | 25 ++++- src/storage/stable.rs | 168 +++++++++++++++++++++------- src/storage/transient.rs | 182 +++++++++++++++++++++++-------- src/storage/types.rs | 1 - src/test_utils.rs | 36 +++++- 9 files changed, 409 insertions(+), 132 deletions(-) diff --git a/src/error.rs b/src/error.rs index 694bdfd..256e3bd 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,6 +9,8 @@ pub enum Error { InvalidOpenFlags, InvalidFdFlags, FileAlreadyExists, + FileIsNotMounted, + IsMountedAlready, NameTooLong, DirectoryNotEmpty, ExpectedToRemoveFile, diff --git a/src/fs.rs b/src/fs.rs index 8e90437..3cf6d23 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use ic_stable_structures::Memory; use crate::{ @@ -27,7 +25,6 @@ pub struct FileSystem { root_fd: Fd, fd_table: FdTable, pub storage: Box, - mounted_nodes: HashMap>, } impl FileSystem { @@ -40,7 +37,6 @@ impl FileSystem { root_fd: 0, fd_table, storage, - mounted_nodes: HashMap::new(), }); } @@ -52,10 +48,10 @@ impl FileSystem { root_fd, fd_table, storage, - mounted_nodes: HashMap::new(), }) } + // pub fn get_storage_version(&self) -> u32 { self.storage.get_version() } @@ -118,21 +114,11 @@ impl FileSystem { 0, )?; - let mut meta = self.metadata(fd)?; - let node = self.get_node(fd)?; - - self.mounted_nodes.insert(node, memory); - - if meta.mount_size.is_none() { - // init new memory with 0 if it was not known before - meta.mount_size = Some(0); - } - - self.storage.put_metadata(node, meta); - self.close(fd)?; + self.storage.mount_node(node, memory)?; + Ok(()) } @@ -145,9 +131,13 @@ impl FileSystem { let node = find_node(root_node, filename, self.storage.as_ref())?; + if !self.storage.is_mounted(node) { + return Err(Error::FileIsNotMounted); + } + let mut metadata = self.metadata_from_node(node)?; - metadata.mount_size = Some(new_size); + metadata.size = new_size; self.storage.put_metadata(node, metadata); @@ -515,6 +505,8 @@ impl FileSystem { #[cfg(test)] mod tests { + use ic_stable_structures::{Memory, VectorMemory}; + use crate::{ error::Error, fs::{DstBuf, FdFlags, SrcBuf}, @@ -523,7 +515,10 @@ mod tests { types::{FdStat, OpenFlags}, }, storage::types::FileType, - test_utils::{read_text_file, test_fs, test_fs_transient, write_text_fd, write_text_file}, + test_utils::{ + new_vector_memory, read_text_file, test_fs, test_fs_setups, test_fs_transient, + write_text_fd, write_text_file, + }, }; use super::{Fd, FileSystem}; @@ -1259,45 +1254,67 @@ mod tests { } #[test] - fn writing_from_different_file_descriptors() { + fn writing_into_mounted_memory() { + let memory: VectorMemory = new_vector_memory(); + let mut fs = test_fs(); + let root_fd = fs.root_fd(); - let fd1 = fs - .open_or_create( - root_fd, - "f1/f2/text.txt", - FdStat::default(), - OpenFlags::CREATE, - 40, - ) - .unwrap(); - let fd2 = fs - .open_or_create( - root_fd, - "f1//f2/text.txt", - FdStat::default(), - OpenFlags::CREATE, - 44, - ) + fs.mount_memory_file("test.txt", Box::new(memory.clone())) .unwrap(); - write_text_fd(&mut fs, fd1, "abc", 1).unwrap(); - write_text_fd(&mut fs, fd2, "123", 1).unwrap(); - write_text_fd(&mut fs, fd1, "xyz", 1).unwrap(); + let content = "ABCDEFG123"; + + write_text_file(&mut fs, root_fd, "test.txt", content, 1).unwrap(); + + let mut buf = [0u8; 100]; - let content = read_text_file(&mut fs, root_fd, "/f1/f2/text.txt", 0, 9); + memory.read(0, &mut buf); - assert_eq!("123xyz", content); + println!("{:?}", buf); } #[test] - fn write_into_empty_filename_fails() { - let mut fs = test_fs(); - let root_fd = fs.root_fd(); - - let res = write_text_file(&mut fs, root_fd, "", "content123", 100); + fn writing_from_different_file_descriptors() { + for mut fs in test_fs_setups("f1/f2/text.txt") { + let root_fd = fs.root_fd(); + + let fd1 = fs + .open_or_create( + root_fd, + "f1/f2/text.txt", + FdStat::default(), + OpenFlags::CREATE, + 40, + ) + .unwrap(); + let fd2 = fs + .open_or_create( + root_fd, + "f1//f2/text.txt", + FdStat::default(), + OpenFlags::CREATE, + 44, + ) + .unwrap(); + + write_text_fd(&mut fs, fd1, "abc", 1).unwrap(); + write_text_fd(&mut fs, fd2, "123", 1).unwrap(); + write_text_fd(&mut fs, fd1, "xyz", 1).unwrap(); + + let content = read_text_file(&mut fs, root_fd, "/f1/f2/text.txt", 0, 9); + + assert_eq!("123xyz", content); + } + } - assert!(res.is_err()); + #[test] + fn write_into_empty_filename_fails() { + for mut fs in test_fs_setups("") { + let root_fd = fs.root_fd(); + let res = write_text_file(&mut fs, root_fd, "", "content123", 100); + assert!(res.is_err()); + } } } diff --git a/src/runtime/structure_helpers.rs b/src/runtime/structure_helpers.rs index 695f333..1fba20a 100644 --- a/src/runtime/structure_helpers.rs +++ b/src/runtime/structure_helpers.rs @@ -126,7 +126,6 @@ pub fn create_dir_entry( }, first_dir_entry: None, last_dir_entry: None, - mount_size: None, }, ); diff --git a/src/storage.rs b/src/storage.rs index 1a9c955..4ad0c45 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,3 +1,5 @@ +use ic_stable_structures::Memory; + use crate::{ error::Error, storage::types::{DirEntry, DirEntryIndex, FileChunkIndex, FileSize, Metadata, Node}, @@ -19,6 +21,15 @@ pub trait Storage { // Generate the next available node ID. fn new_node(&mut self) -> Node; + // mark node as mounted + fn mount_node(&mut self, node: Node, memory: Box) -> Result<(), Error>; + // mark note as not mounted + fn unmount_node(&mut self, node: Node) -> Result, Error>; + // return true if the node is mounted + fn is_mounted(&self, node: Node) -> bool; + // return mounted memory related to the node, or None + fn get_mounted_memory(&self, node: Node) -> Option<&dyn Memory>; + // Get the metadata associated with the node. fn get_metadata(&self, node: Node) -> Result; // Update the metadata associated with the node. @@ -34,6 +45,7 @@ pub trait Storage { fn rm_direntry(&mut self, node: Node, index: DirEntryIndex); // Fill the buffer contents with data of a selected file chunk. + #[cfg(test)] fn read_filechunk( &self, node: Node, @@ -53,6 +65,7 @@ pub trait Storage { // Insert of update a selected file chunk with the data provided in buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]); + // Remove file chunk from a given file node. fn rm_filechunk(&mut self, node: Node, index: FileChunkIndex); } diff --git a/src/storage/dummy.rs b/src/storage/dummy.rs index 3485174..ca7b566 100644 --- a/src/storage/dummy.rs +++ b/src/storage/dummy.rs @@ -56,6 +56,7 @@ impl Storage for DummyStorage { panic!("Not supported") } + #[cfg(test)] fn read_filechunk( &self, _node: Node, @@ -89,6 +90,29 @@ impl Storage for DummyStorage { ) -> Result { panic!("Not supported") } + + fn mount_node( + &mut self, + _node: Node, + _memory: Box, + ) -> Result<(), Error> { + panic!("Not supported") + } + + fn unmount_node( + &mut self, + _node: Node, + ) -> Result, Error> { + panic!("Not supported") + } + + fn is_mounted(&self, _node: Node) -> bool { + panic!("Not supported") + } + + fn get_mounted_memory(&self, _node: Node) -> Option<&dyn ic_stable_structures::Memory> { + panic!("Not supported") + } } #[cfg(test)] @@ -112,7 +136,6 @@ mod tests { times: Times::default(), first_dir_entry: Some(42), last_dir_entry: Some(24), - mount_size: None, }, ) } diff --git a/src/storage/stable.rs b/src/storage/stable.rs index 73f3513..0ef23c1 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -1,5 +1,6 @@ -use std::ops::Range; +use std::{collections::HashMap, ops::Range}; +use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES; use ic_stable_structures::{ memory_manager::{MemoryId, MemoryManager, VirtualMemory}, BTreeMap, Cell, Memory, @@ -32,9 +33,12 @@ pub struct StableStorage { metadata: BTreeMap>, direntry: BTreeMap<(Node, DirEntryIndex), DirEntry, VirtualMemory>, filechunk: BTreeMap<(Node, FileChunkIndex), FileChunk, VirtualMemory>, + mounted_meta: BTreeMap>, // It is not used, but is needed to keep memories alive. _memory_manager: Option>, + // active mounts + active_mounts: HashMap>, } impl StableStorage { @@ -73,12 +77,14 @@ impl StableStorage { let metadata_memory = memory_manager.get(MemoryId::new(memory_indices.start + 1u8)); let direntry_memory = memory_manager.get(MemoryId::new(memory_indices.start + 2u8)); let filechunk_memory = memory_manager.get(MemoryId::new(memory_indices.start + 3u8)); + let mounted_meta = memory_manager.get(MemoryId::new(memory_indices.start + 4u8)); Self::new_with_custom_memories( header_memory, metadata_memory, direntry_memory, filechunk_memory, + mounted_meta, ) } @@ -87,6 +93,7 @@ impl StableStorage { metadata: VirtualMemory, direntry: VirtualMemory, filechunk: VirtualMemory, + mounted_meta: VirtualMemory, ) -> Self { let default_header_value = Header { version: FS_VERSION, @@ -98,7 +105,10 @@ impl StableStorage { metadata: BTreeMap::init(metadata), direntry: BTreeMap::init(direntry), filechunk: BTreeMap::init(filechunk), + mounted_meta: BTreeMap::init(mounted_meta), + // runtime data _memory_manager: None, + active_mounts: HashMap::new(), }; let version = result.header.get().version; @@ -118,7 +128,6 @@ impl StableStorage { times: Times::default(), first_dir_entry: None, last_dir_entry: None, - mount_size: None, }; result.put_metadata(ROOT_NODE, metadata); } @@ -141,8 +150,6 @@ impl Storage for StableStorage { fn new_node(&mut self) -> Node { let mut header = self.header.get().clone(); - self.metadata.last_key_value(); - let result = header.next_node; header.next_node += 1; @@ -159,17 +166,29 @@ impl Storage for StableStorage { // Get the metadata associated with the node. fn get_metadata(&self, node: Node) -> Result { - self.metadata.get(&node).ok_or(Error::NotFound) + if self.is_mounted(node) { + self.mounted_meta.get(&node).ok_or(Error::NotFound) + } else { + self.metadata.get(&node).ok_or(Error::NotFound) + } } // Update the metadata associated with the node. fn put_metadata(&mut self, node: Node, metadata: Metadata) { - self.metadata.insert(node, metadata); + if self.is_mounted(node) { + self.mounted_meta.insert(node, metadata); + } else { + self.metadata.insert(node, metadata); + } } // Remove the metadata associated with the node. fn rm_metadata(&mut self, node: Node) { - self.metadata.remove(&node); + if self.is_mounted(node) { + self.mounted_meta.remove(&node); + } else { + self.metadata.remove(&node); + } } // Retrieve the DirEntry instance given the Node and DirEntryIndex. @@ -188,6 +207,7 @@ impl Storage for StableStorage { } // Fill the buffer contents with data of a chosen file chunk. + #[cfg(test)] fn read_filechunk( &self, node: Node, @@ -195,8 +215,15 @@ impl Storage for StableStorage { offset: FileSize, buf: &mut [u8], ) -> Result<(), Error> { - let value = self.filechunk.get(&(node, index)).ok_or(Error::NotFound)?; - buf.copy_from_slice(&value.bytes[offset as usize..offset as usize + buf.len()]); + if let Some(memory) = self.active_mounts.get(&node) { + // work with memory + let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + memory.read(address, buf); + } else { + let value = self.filechunk.get(&(node, index)).ok_or(Error::NotFound)?; + buf.copy_from_slice(&value.bytes[offset as usize..offset as usize + buf.len()]); + } + Ok(()) } @@ -212,55 +239,123 @@ impl Storage for StableStorage { return Ok(0); } - let start_index = (offset / FILE_CHUNK_SIZE as FileSize) as FileChunkIndex; + let size_read = if let Some(memory) = self.active_mounts.get(&node) { + let remainder = file_size - offset; + let to_read = remainder.min(buf.len() as FileSize); - let mut chunk_offset = offset - start_index as FileSize * FILE_CHUNK_SIZE as FileSize; + memory.read(offset, &mut buf[..to_read as usize]); + to_read + } else { + let start_index = (offset / FILE_CHUNK_SIZE as FileSize) as FileChunkIndex; - let range = (node, start_index)..(node + 1, 0); + let mut chunk_offset = offset - start_index as FileSize * FILE_CHUNK_SIZE as FileSize; - let mut size_read: FileSize = 0; - let mut remainder = file_size - offset; + let range = (node, start_index)..(node + 1, 0); - for ((nd, _idx), value) in self.filechunk.range(range) { - assert!(nd == node); + let mut size_read: FileSize = 0; + let mut remainder = file_size - offset; - // finished reading, buffer full - if size_read == buf.len() as FileSize { - break; - } + for ((nd, _idx), value) in self.filechunk.range(range) { + assert!(nd == node); - let chunk_space = FILE_CHUNK_SIZE as FileSize - chunk_offset; + // finished reading, buffer full + if size_read == buf.len() as FileSize { + break; + } - let to_read = remainder - .min(chunk_space) - .min(buf.len() as FileSize - size_read); + let chunk_space = FILE_CHUNK_SIZE as FileSize - chunk_offset; - let write_buf = &mut buf[size_read as usize..size_read as usize + to_read as usize]; + let to_read = remainder + .min(chunk_space) + .min(buf.len() as FileSize - size_read); - write_buf.copy_from_slice( - &value.bytes[chunk_offset as usize..chunk_offset as usize + to_read as usize], - ); + let write_buf = &mut buf[size_read as usize..size_read as usize + to_read as usize]; - chunk_offset = 0; + write_buf.copy_from_slice( + &value.bytes[chunk_offset as usize..chunk_offset as usize + to_read as usize], + ); - size_read += to_read; - remainder -= to_read; - } + chunk_offset = 0; + + size_read += to_read; + remainder -= to_read; + } + + size_read + }; Ok(size_read) } - // Insert of update a selected file chunk with the data provided in buffer. + // Insert of update a selected file chunk with the data provided in a buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { - let mut entry = self.filechunk.get(&(node, index)).unwrap_or_default(); - entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf); - self.filechunk.insert((node, index), entry); + if let Some(memory) = self.active_mounts.get(&node) { + // grow memory if needed + let pages_required = + (index as FileSize + buf.len() as FileSize + WASM_PAGE_SIZE_IN_BYTES - 1) + / WASM_PAGE_SIZE_IN_BYTES; + let cur_pages = memory.size(); + + if cur_pages < pages_required { + memory.grow(pages_required - cur_pages); + } + + // store data + let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + + memory.write(address, buf); + } else { + // work with stable structure + let mut entry = self.filechunk.get(&(node, index)).unwrap_or_default(); + entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf); + self.filechunk.insert((node, index), entry); + } } // Remove file chunk from a given file node. fn rm_filechunk(&mut self, node: Node, index: FileChunkIndex) { self.filechunk.remove(&(node, index)); } + + fn mount_node(&mut self, node: Node, memory: Box) -> Result<(), Error> { + if self.is_mounted(node) { + return Err(Error::IsMountedAlready); + } + + // do extra meta preparation + let mut meta = self.metadata.get(&node).ok_or(Error::NotFound)?; + + self.active_mounts.insert(node, memory); + + let new_mounted_meta = if let Some(old_mounted_meta) = self.mounted_meta.get(&node) { + // we can change here something for the new mounted meta + old_mounted_meta + } else { + // take a copy of the file meta, set size to 0 by default + meta.size = 0; + meta + }; + + self.mounted_meta.insert(node, new_mounted_meta); + + Ok(()) + } + + fn unmount_node(&mut self, node: Node) -> Result, Error> { + let memory = self.active_mounts.remove(&node); + + memory.ok_or(Error::FileIsNotMounted) + } + + fn is_mounted(&self, node: Node) -> bool { + self.active_mounts.contains_key(&node) + } + + fn get_mounted_memory(&self, node: Node) -> Option<&dyn Memory> { + let res: Option<&Box> = self.active_mounts.get(&node); + + res.map(|b| b.as_ref()) + } } #[cfg(test)] @@ -286,7 +381,6 @@ mod tests { times: Times::default(), first_dir_entry: Some(42), last_dir_entry: Some(24), - mount_size: None, }, ); let metadata = storage.get_metadata(node).unwrap(); diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 2ca0533..1d33fb1 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -1,4 +1,7 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; + +use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES; +use ic_stable_structures::Memory; use crate::{ error::Error, @@ -9,7 +12,7 @@ use crate::{ storage::Storage, }; -use super::types::FILE_CHUNK_SIZE; +use super::types::{Header, FILE_CHUNK_SIZE}; // The root node ID. const ROOT_NODE: Node = 0; @@ -17,16 +20,19 @@ const ROOT_NODE: Node = 0; const FS_TRANSIENT_VERSION: u32 = 1; // Transient storage representation. -#[derive(Debug, Default)] +#[derive(Default)] pub struct TransientStorage { + header: Header, // Node metadata information. metadata: BTreeMap, // Directory entries for each of the directory node. direntry: BTreeMap<(Node, DirEntryIndex), DirEntry>, // File contents for each of the file node. filechunk: BTreeMap<(Node, FileChunkIndex), FileChunk>, - // Next node ID. - next_node: Node, + // Mounted memory Node metadata information. + mounted_meta: BTreeMap, + // active mounts + active_mounts: HashMap>, } impl TransientStorage { @@ -40,13 +46,18 @@ impl TransientStorage { times: Times::default(), first_dir_entry: None, last_dir_entry: None, - mount_size: None, }; let mut result = Self { + header: Header { + version: 1, + next_node: ROOT_NODE + 1, + }, metadata: Default::default(), direntry: Default::default(), filechunk: Default::default(), - next_node: ROOT_NODE + 1, + + mounted_meta: Default::default(), + active_mounts: Default::default(), }; result.put_metadata(ROOT_NODE, metadata); result @@ -66,26 +77,38 @@ impl Storage for TransientStorage { // Generate the next available node ID. fn new_node(&mut self) -> Node { - let result = self.next_node; - self.next_node += 1; + let result = self.header.next_node; + self.header.next_node += 1; result } // Get the metadata associated with the node. fn get_metadata(&self, node: Node) -> Result { - let value = self.metadata.get(&node).ok_or(Error::NotFound)?; - Ok(value.clone()) + let meta = if self.is_mounted(node) { + self.mounted_meta.get(&node).ok_or(Error::NotFound)? + } else { + self.metadata.get(&node).ok_or(Error::NotFound)? + }; + + Ok(meta.clone()) } // Update the metadata associated with the node. fn put_metadata(&mut self, node: Node, metadata: Metadata) { - self.next_node = self.next_node.max(node + 1); - self.metadata.insert(node, metadata); + if self.is_mounted(node) { + self.mounted_meta.insert(node, metadata); + } else { + self.metadata.insert(node, metadata); + } } // Remove the metadata associated with the node. fn rm_metadata(&mut self, node: Node) { - self.metadata.remove(&node); + if self.is_mounted(node) { + self.mounted_meta.remove(&node); + } else { + self.metadata.remove(&node); + } } // Retrieve the DirEntry instance given the Node and DirEntryIndex. @@ -105,6 +128,7 @@ impl Storage for TransientStorage { } // Fill the buffer contents with data of a chosen file chunk. + #[cfg(test)] fn read_filechunk( &self, node: Node, @@ -112,12 +136,19 @@ impl Storage for TransientStorage { offset: FileSize, buf: &mut [u8], ) -> Result<(), Error> { - let value = self.filechunk.get(&(node, index)).ok_or(Error::NotFound)?; - buf.copy_from_slice(&value.bytes[offset as usize..offset as usize + buf.len()]); + if let Some(memory) = self.get_mounted_memory(node) { + // work with memory + let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + memory.read(address, buf); + } else { + let value = self.filechunk.get(&(node, index)).ok_or(Error::NotFound)?; + buf.copy_from_slice(&value.bytes[offset as usize..offset as usize + buf.len()]); + } + Ok(()) } - // Fill the buffer contents with data of a chosen file chunk. + // Fill the buffer contents with data fn read_range( &self, node: Node, @@ -129,54 +160,120 @@ impl Storage for TransientStorage { return Ok(0); } - let start_index = (offset / FILE_CHUNK_SIZE as FileSize) as FileChunkIndex; + let size_read = if let Some(memory) = self.get_mounted_memory(node) { + let remainder = file_size - offset; + let to_read = remainder.min(buf.len() as FileSize); - let mut chunk_offset = offset - start_index as FileSize * FILE_CHUNK_SIZE as FileSize; + memory.read(offset, &mut buf[..to_read as usize]); + to_read + } else { + let start_index = (offset / FILE_CHUNK_SIZE as FileSize) as FileChunkIndex; - let range = (node, start_index)..(node + 1, 0); + let mut chunk_offset = offset - start_index as FileSize * FILE_CHUNK_SIZE as FileSize; - let mut size_read: FileSize = 0; - let mut remainder = file_size - offset; + let range = (node, start_index)..(node + 1, 0); - for ((nd, _idx), value) in self.filechunk.range(range) { - assert!(*nd == node); + let mut size_read: FileSize = 0; + let mut remainder = file_size - offset; - // finished reading, buffer full - if size_read == buf.len() as FileSize { - break; - } + for ((nd, _idx), value) in self.filechunk.range(range) { + assert!(*nd == node); - let chunk_space = FILE_CHUNK_SIZE as FileSize - chunk_offset; + // finished reading, buffer full + if size_read == buf.len() as FileSize { + break; + } - let to_read = remainder - .min(chunk_space) - .min(buf.len() as FileSize - size_read); + let chunk_space = FILE_CHUNK_SIZE as FileSize - chunk_offset; - let write_buf = &mut buf[size_read as usize..size_read as usize + to_read as usize]; + let to_read = remainder + .min(chunk_space) + .min(buf.len() as FileSize - size_read); - write_buf.copy_from_slice( - &value.bytes[chunk_offset as usize..chunk_offset as usize + to_read as usize], - ); + let write_buf = &mut buf[size_read as usize..size_read as usize + to_read as usize]; - chunk_offset = 0; + write_buf.copy_from_slice( + &value.bytes[chunk_offset as usize..chunk_offset as usize + to_read as usize], + ); - size_read += to_read; - remainder -= to_read; - } + chunk_offset = 0; + + size_read += to_read; + remainder -= to_read; + } + + size_read + }; Ok(size_read) } // Insert of update a selected file chunk with the data provided in buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { - let entry = self.filechunk.entry((node, index)).or_default(); - entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf) + if let Some(memory) = self.get_mounted_memory(node) { + // grow memory if needed + let pages_required = + (index as FileSize + buf.len() as FileSize + WASM_PAGE_SIZE_IN_BYTES - 1) + / WASM_PAGE_SIZE_IN_BYTES; + let cur_pages = memory.size(); + + if cur_pages < pages_required { + memory.grow(pages_required - cur_pages); + } + + // work with memory + let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + memory.write(address, buf); + } else { + let entry = self.filechunk.entry((node, index)).or_default(); + entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf) + } } // Remove file chunk from a given file node. fn rm_filechunk(&mut self, node: Node, index: FileChunkIndex) { self.filechunk.remove(&(node, index)); } + + fn mount_node(&mut self, node: Node, memory: Box) -> Result<(), Error> { + if self.is_mounted(node) { + return Err(Error::IsMountedAlready); + } + + // do extra meta preparation + let mut meta = self.metadata.get(&node).ok_or(Error::NotFound)?.clone(); + + self.active_mounts.insert(node, memory); + + let new_mounted_meta = if let Some(old_mounted_meta) = self.mounted_meta.get(&node) { + // we can change here something for the new mounted meta + old_mounted_meta.clone() + } else { + // take a copy of the file meta, set size to 0 by default + meta.size = 0; + meta + }; + + self.mounted_meta.insert(node, new_mounted_meta); + + Ok(()) + } + + fn unmount_node(&mut self, node: Node) -> Result, Error> { + let memory = self.active_mounts.remove(&node); + + memory.ok_or(Error::FileIsNotMounted) + } + + fn is_mounted(&self, node: Node) -> bool { + self.active_mounts.contains_key(&node) + } + + fn get_mounted_memory(&self, node: Node) -> Option<&dyn Memory> { + let res = self.active_mounts.get(&node); + + res.map(|b| b.as_ref()) + } } #[cfg(test)] @@ -197,7 +294,6 @@ mod tests { times: Times::default(), first_dir_entry: None, last_dir_entry: None, - mount_size: None, }, ); storage.write_filechunk(node, 0, 0, &[42; 10]); diff --git a/src/storage/types.rs b/src/storage/types.rs index a54c807..2e96aea 100644 --- a/src/storage/types.rs +++ b/src/storage/types.rs @@ -76,7 +76,6 @@ pub struct Metadata { pub times: Times, pub first_dir_entry: Option, pub last_dir_entry: Option, - pub mount_size: Option, } impl ic_stable_structures::Storable for Metadata { diff --git a/src/test_utils.rs b/src/test_utils.rs index de96eef..fd01617 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -1,7 +1,14 @@ -use ic_stable_structures::DefaultMemoryImpl; +use ic_stable_structures::{DefaultMemoryImpl, VectorMemory}; use crate::{error::Error, fs::FileSystem, storage::stable::StableStorage}; +#[cfg(test)] +pub fn new_vector_memory() -> VectorMemory { + use std::{cell::RefCell, rc::Rc}; + + Rc::new(RefCell::new(Vec::new())) +} + #[cfg(test)] pub fn test_fs() -> FileSystem { let memory = DefaultMemoryImpl::default(); @@ -18,6 +25,33 @@ pub fn test_fs_transient() -> FileSystem { FileSystem::new(Box::new(storage)).unwrap() } +#[cfg(test)] +pub fn test_fs_setups(virtual_file_name: &str) -> Vec { + let mut result = Vec::new(); + + result.push(test_fs()); + + result.push(test_fs_transient()); + + if !virtual_file_name.is_empty() { + let mut fs = test_fs(); + + fs.mount_memory_file(virtual_file_name, Box::new(new_vector_memory())) + .unwrap(); + + result.push(fs); + + let mut fs = test_fs_transient(); + + fs.mount_memory_file(virtual_file_name, Box::new(new_vector_memory())) + .unwrap(); + + result.push(fs); + } + + result +} + #[cfg(test)] pub fn write_text_file( fs: &mut FileSystem, From 35f1564aa5489bd6a3f81f5e058046d0636f1e45 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Mon, 5 Aug 2024 07:49:21 +0200 Subject: [PATCH 06/13] fixed some errors, more tests are working --- src/fs.rs | 62 +++---- src/integration_tests.rs | 62 ++++++- src/runtime/file.rs | 151 ++++++++---------- src/storage/stable.rs | 13 +- src/storage/transient.rs | 8 +- .../src/demo_test_upgraded_backend/src/lib.rs | 9 +- .../src/fs_benchmark_test_backend/src/lib.rs | 25 +-- tests/fs_benchmark_test/store_stable.sh | 3 + 8 files changed, 192 insertions(+), 141 deletions(-) create mode 100755 tests/fs_benchmark_test/store_stable.sh diff --git a/src/fs.rs b/src/fs.rs index 3cf6d23..0fd9a67 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -362,7 +362,7 @@ impl FileSystem { } } - // Opens a file and return its new file descriptor. + // Opens a file and returns its new file descriptor. pub fn open(&mut self, node: Node, stat: FdStat, flags: OpenFlags) -> Result { if flags.contains(OpenFlags::EXCLUSIVE) { return Err(Error::FileAlreadyExists); @@ -966,56 +966,58 @@ mod tests { #[test] fn renumber_when_the_alternative_file_doesnt_exist() { - let mut fs = test_fs(); - let dir = fs.root_fd(); + for mut fs in test_fs_setups("file1.txt") { + let dir = fs.root_fd(); - let fd1 = fs - .open_or_create(dir, "file1.txt", FdStat::default(), OpenFlags::CREATE, 0) - .unwrap(); + let fd1 = fs + .open_or_create(dir, "file1.txt", FdStat::default(), OpenFlags::CREATE, 0) + .unwrap(); - fs.write(fd1, &[1, 2, 3, 4, 5]).unwrap(); + fs.write(fd1, &[1, 2, 3, 4, 5]).unwrap(); - let pos1 = fs.tell(fd1).unwrap(); + let pos1 = fs.tell(fd1).unwrap(); - assert!(pos1 == 5); + assert!(pos1 == 5); - let fd2 = 100; + let fd2 = 100; - fs.renumber(fd1, fd2).unwrap(); + fs.renumber(fd1, fd2).unwrap(); - let pos2_renumbered = fs.tell(fd2).unwrap(); + let pos2_renumbered = fs.tell(fd2).unwrap(); - assert!(pos1 == pos2_renumbered); + assert!(pos1 == pos2_renumbered); - let res = fs.tell(fd1); + let res = fs.tell(fd1); - assert!(res.is_err()); + assert!(res.is_err()); + } } #[test] fn set_modified_set_accessed_time() { - let mut fs = test_fs(); - let dir = fs.root_fd(); + for mut fs in test_fs_setups("") { + let dir = fs.root_fd(); - let fd1 = fs - .open_or_create(dir, "file1.txt", FdStat::default(), OpenFlags::CREATE, 111) - .unwrap(); + let fd1 = fs + .open_or_create(dir, "file1.txt", FdStat::default(), OpenFlags::CREATE, 111) + .unwrap(); - fs.write(fd1, &[1, 2, 3, 4, 5]).unwrap(); + fs.write(fd1, &[1, 2, 3, 4, 5]).unwrap(); - fs.set_accessed_time(fd1, 333).unwrap(); - fs.set_modified_time(fd1, 222).unwrap(); + fs.set_accessed_time(fd1, 333).unwrap(); + fs.set_modified_time(fd1, 222).unwrap(); - let metadata = fs.metadata(fd1).unwrap(); + let metadata = fs.metadata(fd1).unwrap(); - let times = metadata.times; + let times = metadata.times; - assert_eq!(times.created, 111); - assert_eq!(times.accessed, 333); - assert_eq!(times.modified, 222); + assert_eq!(times.accessed, 333); + assert_eq!(times.modified, 222); + assert_eq!(times.created, 111); - assert_eq!(metadata.size, 5); - assert_eq!(metadata.file_type, FileType::RegularFile); + assert_eq!(metadata.size, 5); + assert_eq!(metadata.file_type, FileType::RegularFile); + } } #[test] diff --git a/src/integration_tests.rs b/src/integration_tests.rs index 3a50f88..fcf8b6d 100644 --- a/src/integration_tests.rs +++ b/src/integration_tests.rs @@ -92,6 +92,35 @@ mod fns { .unwrap(); } + pub(crate) fn append_buffer(pic: &PocketIc, content: &str, count: u64) { + pic.update_call( + active_canister(), + Principal::anonymous(), + "append_buffer", + candid::encode_args((content, count)).unwrap(), + ) + .unwrap(); + } + + pub(crate) fn store_buffer(pic: &PocketIc, filename: &str) -> (u64, u64) { + let response = pic + .update_call( + active_canister(), + Principal::anonymous(), + "store_buffer", + candid::encode_one(filename).unwrap(), + ) + .unwrap(); + + if let WasmResult::Reply(response) = response { + let result: (u64, u64) = decode_args(&response).unwrap(); + + result + } else { + panic!("unintended call failure!"); + } + } + pub(crate) fn read_text(pic: &PocketIc, filename: &str, offset: i64, size: u64) -> String { let response = pic .query_call( @@ -304,11 +333,19 @@ fn create_1000_files() { let result = fns::list_files(&pic, ""); - let filenames = vec!["files1", "files2", "files3", "files4"]; + let filenames = vec!["stable_file.txt", "files1", "files2", "files3", "files4"]; assert_eq!(result, filenames); } +fn no_virtual_names(vec: Vec) -> Vec { + let mut v = vec; + + v.retain(|v| !(*v).eq("stable_file.txt")); + + v +} + #[test] fn long_paths_and_file_names() { let pic = setup_initial_canister(); @@ -345,6 +382,8 @@ fn long_paths_and_file_names() { let filenames = vec![long_name]; let result = fns::list_files(&pic, ""); + let result = no_virtual_names(result); + assert_eq!(result, filenames); // try reading one of the files @@ -401,3 +440,24 @@ fn large_file_read() { assert_eq!(size, 99_999_987); } + +#[test] +fn large_file_write() { + let pic = setup_initial_canister(); + + let filename = "stable_file1.txt"; + + // create large buffer + fns::append_buffer(&pic, "abcdef7890", 10_000_000); + + let (instructions, size) = fns::store_buffer(&pic, filename); + + println!("instructions {instructions}, size {size}"); + + assert!( + instructions < 14_000_000_000, + "The call should take less than 3 billion instructions" + ); + + assert_eq!(size, 100_000_000); +} diff --git a/src/runtime/file.rs b/src/runtime/file.rs index 09b2cf2..3d65ba0 100644 --- a/src/runtime/file.rs +++ b/src/runtime/file.rs @@ -223,7 +223,10 @@ fn get_chunk_infos(start: FileSize, end: FileSize) -> Vec { #[cfg(test)] mod tests { - use crate::test_utils::{test_fs, test_fs_transient}; + use crate::{ + fs::OpenFlags, + test_utils::{test_fs, test_fs_setups}, + }; use super::*; @@ -402,112 +405,84 @@ mod tests { #[test] fn read_and_write_offset_chunk() { - let mut fs = test_fs(); - let fd = fs - .create_file(fs.root_fd(), "test", FdStat::default(), 0) - .unwrap(); + for mut fs in test_fs_setups("test") { + let fd = fs + .open_or_create( + fs.root_fd(), + "test", + FdStat::default(), + OpenFlags::CREATE, + 0, + ) + .unwrap(); - let mut file = fs.get_test_file(fd); - let storage = fs.get_test_storage(); + let mut file = fs.get_test_file(fd); + let storage = fs.get_test_storage(); - for i in 0..1000 { - let buf = [(i % 256) as u8; 16]; - file.write_with_offset(i * 16, &buf, storage).unwrap(); - } + for i in 0..1000 { + let buf = [(i % 256) as u8; 16]; + file.write_with_offset(i * 16, &buf, storage).unwrap(); + } - file.seek(-1000 * 16, Whence::END, storage).unwrap(); + file.seek(-1000 * 16, Whence::END, storage).unwrap(); - for i in 0..1000 { - let mut buf = [0; 16]; - file.read_with_offset_chunk(i * 16, &mut buf, storage) - .unwrap(); + for i in 0..1000 { + let mut buf = [0; 16]; + file.read_with_offset_chunk(i * 16, &mut buf, storage) + .unwrap(); - let expected = [(i % 256) as u8; 16]; - assert_eq!(buf, expected); + let expected = [(i % 256) as u8; 16]; + assert_eq!(buf, expected); + } } } #[test] fn read_and_write_offset_vs_range() { - let mut fs = test_fs(); - let fd = fs - .create_file(fs.root_fd(), "test", FdStat::default(), 0) - .unwrap(); - - let file = fs.get_test_file(fd); - let storage = fs.get_test_storage(); - - for i in 0..1000 { - let buf = [(i % 256) as u8; 16]; - file.write_with_offset(i * 16, &buf, storage).unwrap(); - } - - for i in 0..1000 { - let mut buf1 = [0; 13]; - let len1 = file - .read_with_offset_chunk(i * 16, &mut buf1, storage) - .unwrap(); - - let mut buf2 = [0; 13]; - let len2 = file.read_with_offset(i * 16, &mut buf2, storage).unwrap(); - - assert_eq!(buf1, buf2); - assert_eq!(len1, len2); - } - - for i in 0..2050 { - let mut buf1 = [0; 5003]; - let len1 = file - .read_with_offset_chunk(i * 13, &mut buf1, storage) + for mut fs in test_fs_setups("test") { + let fd = fs + .open_or_create( + fs.root_fd(), + "test", + FdStat::default(), + OpenFlags::CREATE, + 0, + ) .unwrap(); - let mut buf2 = [0; 5003]; - let len2 = file.read_with_offset(i * 13, &mut buf2, storage).unwrap(); - - assert_eq!(buf1, buf2); - assert_eq!(len1, len2); - } - } - - #[test] - fn read_and_write_offset_vs_range_transient() { - let mut fs = test_fs_transient(); - let fd = fs - .create_file(fs.root_fd(), "test", FdStat::default(), 0) - .unwrap(); + let file = fs.get_test_file(fd); + let storage = fs.get_test_storage(); - let file = fs.get_test_file(fd); - let storage = fs.get_test_storage(); + for i in 0..1000 { + let buf = [(i % 256) as u8; 16]; + file.write_with_offset(i * 16, &buf, storage).unwrap(); + } - for i in 0..1000 { - let buf = [(i % 256) as u8; 16]; - file.write_with_offset(i * 16, &buf, storage).unwrap(); - } + for i in 0..1000 { + let mut buf1 = [0; 13]; + let len1 = file + .read_with_offset_chunk(i * 16, &mut buf1, storage) + .unwrap(); - for i in 0..1000 { - let mut buf1 = [0; 13]; - let len1 = file - .read_with_offset_chunk(i * 16, &mut buf1, storage) - .unwrap(); + let mut buf2 = [0; 13]; + let len2 = file.read_with_offset(i * 16, &mut buf2, storage).unwrap(); - let mut buf2 = [0; 13]; - let len2 = file.read_with_offset(i * 16, &mut buf2, storage).unwrap(); - - assert_eq!(buf1, buf2); - assert_eq!(len1, len2); - } + assert_eq!(buf1, buf2); + assert_eq!(len1, len2); + } - for i in 0..2050 { - let mut buf1 = [0; 5003]; - let len1 = file - .read_with_offset_chunk(i * 13, &mut buf1, storage) - .unwrap(); + for i in 0..2050 { + let mut buf1 = [0; 5003]; + let len1 = file + .read_with_offset_chunk(i * 13, &mut buf1, storage) + .unwrap(); - let mut buf2 = [0; 5003]; - let len2 = file.read_with_offset(i * 13, &mut buf2, storage).unwrap(); + let mut buf2 = [0; 5003]; + let len2 = file.read_with_offset(i * 13, &mut buf2, storage).unwrap(); - assert_eq!(buf1, buf2); - assert_eq!(len1, len2); + assert_eq!(buf1, buf2); + assert_eq!(len1, len2); + } } } } diff --git a/src/storage/stable.rs b/src/storage/stable.rs index 0ef23c1..f5625e2 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -289,20 +289,23 @@ impl Storage for StableStorage { // Insert of update a selected file chunk with the data provided in a buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { + if let Some(memory) = self.active_mounts.get(&node) { + + // work with memory diriectly // grow memory if needed - let pages_required = - (index as FileSize + buf.len() as FileSize + WASM_PAGE_SIZE_IN_BYTES - 1) - / WASM_PAGE_SIZE_IN_BYTES; + let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize + buf.len() as FileSize; + let pages_required = (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; + let cur_pages = memory.size(); if cur_pages < pages_required { - memory.grow(pages_required - cur_pages); + memory.grow(pages_required - cur_pages); } // store data let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; - + memory.write(address, buf); } else { // work with stable structure diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 1d33fb1..79e1977 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -211,10 +211,12 @@ impl Storage for TransientStorage { // Insert of update a selected file chunk with the data provided in buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { if let Some(memory) = self.get_mounted_memory(node) { + + // work with memory diriectly // grow memory if needed - let pages_required = - (index as FileSize + buf.len() as FileSize + WASM_PAGE_SIZE_IN_BYTES - 1) - / WASM_PAGE_SIZE_IN_BYTES; + let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize + buf.len() as FileSize; + let pages_required = (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; + let cur_pages = memory.size(); if cur_pages < pages_required { diff --git a/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs b/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs index 2497dcf..7779ea5 100644 --- a/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs +++ b/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs @@ -4,7 +4,6 @@ use stable_fs::{fs::{DstBuf, FdStat, FileSystem, OpenFlags, SrcBuf, Whence}, sto use ic_stable_structures::{memory_manager::{MemoryId, MemoryManager}, DefaultMemoryImpl, Memory}; - #[ic_cdk::query] fn greet(name: String) -> String { format!("Greetings, {}!", name) @@ -36,9 +35,13 @@ thread_local! { let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8); - RefCell::new( + let fs = RefCell::new( FileSystem::new(Box::new(storage)).unwrap() - ) + ); + + fs.borrow_mut().mount_memory_file("stable_file.txt", Box::new(memory_manager.get(MemoryId::new(155)))).unwrap(); + + fs }) }; } diff --git a/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs b/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs index caae120..cc1eaf3 100644 --- a/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs +++ b/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs @@ -4,7 +4,6 @@ use stable_fs::{fs::{DstBuf, FdStat, FileSystem, OpenFlags, SrcBuf, Whence}, sto use ic_stable_structures::{memory_manager::{MemoryId, MemoryManager}, DefaultMemoryImpl, Memory}; - #[ic_cdk::query] fn greet(name: String) -> String { format!("Hello, {}!", name) @@ -36,9 +35,13 @@ thread_local! { let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8); - RefCell::new( + let fs = RefCell::new( FileSystem::new(Box::new(storage)).unwrap() - ) + ); + + fs.borrow_mut().mount_memory_file("stable_file.txt", Box::new(memory_manager.get(MemoryId::new(155)))).unwrap(); + + fs }) }; } @@ -48,22 +51,22 @@ thread_local! { } #[ic_cdk::update] -pub fn append_buffer(text: String, times: usize, capa: usize) -> usize { +pub fn append_buffer(text: String, times: usize) -> usize { - BUFFER.with(|chunk| { - let mut chunk = chunk.borrow_mut(); + BUFFER.with(|buffer| { + let mut buffer = buffer.borrow_mut(); - if chunk.is_none() { - *chunk = Some(Vec::with_capacity(capa)); + if buffer.is_none() { + *buffer = Some(Vec::new()); } - let chunk = chunk.as_mut().unwrap(); + let buffer = buffer.as_mut().unwrap(); for _ in 0..times { - chunk.extend_from_slice(text.as_bytes()); + buffer.extend_from_slice(text.as_bytes()); } - chunk.len() + buffer.len() }) } diff --git a/tests/fs_benchmark_test/store_stable.sh b/tests/fs_benchmark_test/store_stable.sh new file mode 100755 index 0000000..10dfd89 --- /dev/null +++ b/tests/fs_benchmark_test/store_stable.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +dfx canister call fs_benchmark_test_backend store_buffer '("stable_file.txt")' From 3286f59c222b16407787a69aef5fe4a02976d7a0 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:05:42 +0200 Subject: [PATCH 07/13] some more tests --- src/integration_tests.rs | 28 +++++++++++++++++++ .../src/demo_test_upgraded_backend/src/lib.rs | 16 +++++------ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/integration_tests.rs b/src/integration_tests.rs index fcf8b6d..b307fb0 100644 --- a/src/integration_tests.rs +++ b/src/integration_tests.rs @@ -441,6 +441,34 @@ fn large_file_read() { assert_eq!(size, 99_999_987); } +#[test] +fn large_file_read_after_upgrade() { + let pic = setup_initial_canister(); + + let filename = "stable_file.txt"; + + // create large file + fns::append_text(&pic, "t1.txt", "abcdef7890", 10_000_000); + fns::append_text(&pic, "t2.txt", "abcdef7890", 10_000_000); + fns::append_text(&pic, "t3.txt", "abcdef7890", 10_000_000); + fns::append_text(&pic, "t4.txt", "abcdef7890", 10_000_000); + fns::append_text(&pic, filename, "abcdef7890", 10_000_000); + + // do upgrade + upgrade_canister(&pic); + + let (instructions, size) = fns::read_bytes(&pic, filename, 13, 100_000_000); + + println!("instructions {instructions}, size {size}"); + + assert!( + instructions < 3_000_000_000, + "The call should take less than 3 billion instructions" + ); + + assert_eq!(size, 99_999_987); +} + #[test] fn large_file_write() { let pic = setup_initial_canister(); diff --git a/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs b/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs index 7779ea5..96ada20 100644 --- a/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs +++ b/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs @@ -51,22 +51,22 @@ thread_local! { } #[ic_cdk::update] -pub fn append_buffer(text: String, times: usize, capa: usize) -> usize { +pub fn append_buffer(text: String, times: usize) -> usize { - BUFFER.with(|chunk| { - let mut chunk = chunk.borrow_mut(); + BUFFER.with(|buffer| { + let mut buffer = buffer.borrow_mut(); - if chunk.is_none() { - *chunk = Some(Vec::with_capacity(capa)); + if buffer.is_none() { + *buffer = Some(Vec::new()); } - let chunk = chunk.as_mut().unwrap(); + let buffer = buffer.as_mut().unwrap(); for _ in 0..times { - chunk.extend_from_slice(text.as_bytes()); + buffer.extend_from_slice(text.as_bytes()); } - chunk.len() + buffer.len() }) } From 206cf5e24476957c8cc60ea5bd57094e3d935133 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:10:23 +0200 Subject: [PATCH 08/13] clean-up --- src/storage/stable.rs | 13 +++++++------ src/storage/transient.rs | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/storage/stable.rs b/src/storage/stable.rs index f5625e2..10920d9 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -289,23 +289,24 @@ impl Storage for StableStorage { // Insert of update a selected file chunk with the data provided in a buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { - if let Some(memory) = self.active_mounts.get(&node) { - // work with memory diriectly // grow memory if needed - let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize + buf.len() as FileSize; - let pages_required = (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; + let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + + offset as FileSize + + buf.len() as FileSize; + let pages_required = + (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; let cur_pages = memory.size(); if cur_pages < pages_required { - memory.grow(pages_required - cur_pages); + memory.grow(pages_required - cur_pages); } // store data let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; - + memory.write(address, buf); } else { // work with stable structure diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 79e1977..aedda5b 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -211,11 +211,13 @@ impl Storage for TransientStorage { // Insert of update a selected file chunk with the data provided in buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { if let Some(memory) = self.get_mounted_memory(node) { - // work with memory diriectly // grow memory if needed - let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize + buf.len() as FileSize; - let pages_required = (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; + let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + + offset as FileSize + + buf.len() as FileSize; + let pages_required = + (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; let cur_pages = memory.size(); From 84c9699a392d68f2fddb27ec524b0b8d35b21139 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:39:11 +0200 Subject: [PATCH 09/13] clean-up --- .github/workflows/ci.yml | 2 +- Cargo.lock | 2 +- Cargo.toml | 2 +- src/storage/stable.rs | 2 -- src/storage/transient.rs | 4 ++-- tests/fs_benchmark_test/Cargo.lock | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 670ed26..28dfe3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Tests -on: [push, pull_request] +on: [push] env: CARGO_TERM_COLOR: always diff --git a/Cargo.lock b/Cargo.lock index 371e854..4a09c69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,7 +1336,7 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "stable-fs" -version = "0.5.1" +version = "0.6.0" dependencies = [ "bitflags 2.4.1", "candid", diff --git a/Cargo.toml b/Cargo.toml index e07917e..60e9db2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stable-fs" -version = "0.5.1" +version = "0.6.0" edition = "2021" description = "A Simple File system implementing WASI endpoints and using the stable structures of the Internet Computer" keywords = ["ic", "internet-computer", "file-system"] diff --git a/src/storage/stable.rs b/src/storage/stable.rs index 10920d9..ea54fbe 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -290,7 +290,6 @@ impl Storage for StableStorage { // Insert of update a selected file chunk with the data provided in a buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { if let Some(memory) = self.active_mounts.get(&node) { - // work with memory diriectly // grow memory if needed let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize @@ -309,7 +308,6 @@ impl Storage for StableStorage { memory.write(address, buf); } else { - // work with stable structure let mut entry = self.filechunk.get(&(node, index)).unwrap_or_default(); entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf); self.filechunk.insert((node, index), entry); diff --git a/src/storage/transient.rs b/src/storage/transient.rs index aedda5b..566dadf 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -211,7 +211,7 @@ impl Storage for TransientStorage { // Insert of update a selected file chunk with the data provided in buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { if let Some(memory) = self.get_mounted_memory(node) { - // work with memory diriectly + // grow memory if needed let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize @@ -225,7 +225,7 @@ impl Storage for TransientStorage { memory.grow(pages_required - cur_pages); } - // work with memory + // store data let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; memory.write(address, buf); } else { diff --git a/tests/fs_benchmark_test/Cargo.lock b/tests/fs_benchmark_test/Cargo.lock index c6b9a31..3088c34 100644 --- a/tests/fs_benchmark_test/Cargo.lock +++ b/tests/fs_benchmark_test/Cargo.lock @@ -620,7 +620,7 @@ dependencies = [ [[package]] name = "stable-fs" -version = "0.5.1" +version = "0.6.0" dependencies = [ "bitflags", "ciborium", From 3a488575faf7e774a1e974f0c4eafea253b0b73a Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:39:32 +0200 Subject: [PATCH 10/13] clean-up --- src/storage/transient.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 566dadf..3864185 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -211,7 +211,6 @@ impl Storage for TransientStorage { // Insert of update a selected file chunk with the data provided in buffer. fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { if let Some(memory) = self.get_mounted_memory(node) { - // grow memory if needed let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize From 5d8be0b97c7b7e3d31e708a30c865284accb97e7 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Tue, 6 Aug 2024 00:43:25 +0200 Subject: [PATCH 11/13] mount init and store added with tests --- src/fs.rs | 106 ++++++++++++++++++- src/runtime.rs | 2 +- src/runtime/file.rs | 109 +------------------- src/runtime/structure_helpers.rs | 103 ++++++++++++++++++- src/storage.rs | 15 ++- src/storage/dummy.rs | 34 +++---- src/storage/stable.rs | 144 +++++++++++++++++++++----- src/storage/transient.rs | 152 ++++++++++++++++++++++------ src/storage/types.rs | 8 ++ tests/demo_test_upgraded/Cargo.lock | 2 +- 10 files changed, 493 insertions(+), 182 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index 0fd9a67..7525dde 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -100,6 +100,8 @@ impl FileSystem { } } + // mounte memory file on the top of the given file name, if the file does not exist, it will be created. + // The method fails if the file system couldn't create the file. pub fn mount_memory_file( &mut self, filename: &str, @@ -122,6 +124,24 @@ impl FileSystem { Ok(()) } + // initialize mounted memory with the data stored in the host file + pub fn init_memory_file(&mut self, filename: &str) -> Result<(), Error> { + // create a file for the mount + let fd = self.open_or_create( + self.root_fd, + filename, + FdStat::default(), + OpenFlags::empty(), + 0, + )?; + + let node = self.get_node(fd)?; + self.close(fd)?; + + self.storage.init_mounted_memory(node) + } + + // helper method to set the size of the mounted memory pub fn set_memory_file_size( &mut self, filename: &str, @@ -144,6 +164,42 @@ impl FileSystem { Ok(()) } + // store content of the currently active memory file to the file system + pub fn store_memory_file(&mut self, filename: &str) -> Result<(), Error> { + // create a file for the mount + let fd = self.open_or_create( + self.root_fd, + filename, + FdStat::default(), + OpenFlags::empty(), + 0, + )?; + + let node = self.get_node(fd)?; + self.close(fd)?; + + self.storage.store_mounted_memory(node) + } + + // Unmount memory file, the system will continue to work with its own memory + pub fn unmount_memory_file(&mut self, filename: &str) -> Result, Error> { + // create a file for the mount + let fd = self.open_or_create( + self.root_fd, + filename, + FdStat::default(), + OpenFlags::empty(), + 0, + )?; + + let node = self.get_node(fd)?; + self.close(fd)?; + + let memory = self.storage.unmount_node(node)?; + + Ok(memory) + } + // Get dir entry for a given directory and the directory index. pub fn get_direntry(&self, fd: Fd, index: DirEntryIndex) -> Result { self.get_dir(fd)?.get_entry(index, self.storage.as_ref()) @@ -514,7 +570,7 @@ mod tests { structure_helpers::find_node, types::{FdStat, OpenFlags}, }, - storage::types::FileType, + storage::types::{FileSize, FileType}, test_utils::{ new_vector_memory, read_text_file, test_fs, test_fs_setups, test_fs_transient, write_text_fd, write_text_file, @@ -1277,6 +1333,54 @@ mod tests { println!("{:?}", buf); } + #[test] + fn mounted_memory_store_and_init_roundtrip() { + for mut fs in test_fs_setups("") { + let memory1: VectorMemory = new_vector_memory(); + let memory2: VectorMemory = new_vector_memory(); + + let file_name = "test.txt"; + + fs.mount_memory_file(file_name, Box::new(memory1.clone())) + .unwrap(); + + let content = "ABCDEFG123"; + let len = content.len(); + let count = 1000; + + memory1.grow(5); + + // fill up memory with some data + for i in 0..count { + memory1.write(i as u64 * len as u64, content.as_bytes()); + } + + fs.set_memory_file_size(file_name, len as FileSize * count as FileSize) + .unwrap(); + + // store memory into a file + fs.store_memory_file(file_name).unwrap(); + + fs.unmount_memory_file(file_name).unwrap(); + + fs.mount_memory_file(file_name, Box::new(memory2.clone())) + .unwrap(); + + // init new memory into a file + fs.init_memory_file(file_name).unwrap(); + + let mut buf1 = [0u8; 10]; + let mut buf2 = [0u8; 10]; + + for i in 0..count { + memory1.read(i as u64 * len as u64, &mut buf1); + memory2.read(i as u64 * len as u64, &mut buf2); + + assert_eq!(buf1, buf2); + } + } + } + #[test] fn writing_from_different_file_descriptors() { for mut fs in test_fs_setups("f1/f2/text.txt") { diff --git a/src/runtime.rs b/src/runtime.rs index e246fff..502e0c2 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -1,5 +1,5 @@ pub mod dir; pub mod fd; pub mod file; -pub mod structure_helpers; +pub(crate) mod structure_helpers; pub mod types; diff --git a/src/runtime/file.rs b/src/runtime/file.rs index 3d65ba0..eb6ab8e 100644 --- a/src/runtime/file.rs +++ b/src/runtime/file.rs @@ -2,7 +2,7 @@ use crate::{ error::Error, runtime::types::{FdFlags, FdStat, Whence}, storage::{ - types::{FileChunkIndex, FileSize, FileType, Node, FILE_CHUNK_SIZE}, + types::{FileSize, FileType, Node}, Storage, }, }; @@ -126,6 +126,8 @@ impl File { buf: &mut [u8], storage: &mut dyn Storage, ) -> Result { + use super::structure_helpers::get_chunk_infos; + if buf.is_empty() { return Ok(0 as FileSize); } @@ -155,26 +157,7 @@ impl File { buf: &[u8], storage: &mut dyn Storage, ) -> Result { - let mut metadata = storage.get_metadata(self.node)?; - let end = offset + buf.len() as FileSize; - let chunk_infos = get_chunk_infos(offset, end); - let mut written_size = 0; - for chunk in chunk_infos.into_iter() { - storage.write_filechunk( - self.node, - chunk.index, - chunk.offset, - &buf[written_size..written_size + chunk.len as usize], - ); - written_size += chunk.len as usize; - } - - if end > metadata.size { - metadata.size = end; - storage.put_metadata(self.node, metadata) - } - - Ok(written_size as FileSize) + storage.write_with_offset(self.node, offset, buf) } // Truncate file to 0 size. @@ -186,41 +169,6 @@ impl File { } } -#[derive(Debug, PartialEq, Eq)] -struct ChunkHandle { - index: FileChunkIndex, - offset: FileSize, - len: FileSize, -} - -fn offset_to_file_chunk_index(offset: FileSize) -> FileChunkIndex { - (offset / FILE_CHUNK_SIZE as FileSize) as FileChunkIndex -} - -fn file_chunk_index_to_offset(index: FileChunkIndex) -> FileSize { - index as FileSize * FILE_CHUNK_SIZE as FileSize -} - -fn get_chunk_infos(start: FileSize, end: FileSize) -> Vec { - let mut result = vec![]; - let start_index = offset_to_file_chunk_index(start); - let end_index = offset_to_file_chunk_index(end); - for index in start_index..=end_index { - let start_of_chunk = file_chunk_index_to_offset(index); - assert!(start_of_chunk <= end); - let start_in_chunk = start_of_chunk.max(start) - start_of_chunk; - let end_in_chunk = (start_of_chunk + FILE_CHUNK_SIZE as FileSize).min(end) - start_of_chunk; - if start_in_chunk < end_in_chunk { - result.push(ChunkHandle { - index, - offset: start_in_chunk, - len: end_in_chunk - start_in_chunk, - }); - } - } - result -} - #[cfg(test)] mod tests { use crate::{ @@ -230,55 +178,6 @@ mod tests { use super::*; - #[test] - fn get_chunk_infos_parital() { - let chunks = get_chunk_infos( - FILE_CHUNK_SIZE as FileSize - 1, - 2 * FILE_CHUNK_SIZE as FileSize + 1, - ); - assert_eq!( - chunks[0], - ChunkHandle { - index: 0, - offset: FILE_CHUNK_SIZE as FileSize - 1, - len: 1 - } - ); - assert_eq!( - chunks[1], - ChunkHandle { - index: 1, - offset: 0, - len: FILE_CHUNK_SIZE as FileSize, - } - ); - - assert_eq!( - chunks[2], - ChunkHandle { - index: 2, - offset: 0, - len: 1, - } - ); - } - - #[test] - fn get_chunk_infos_full() { - let chunks = get_chunk_infos(0, 10 * FILE_CHUNK_SIZE as FileSize); - #[allow(clippy::needless_range_loop)] - for i in 0..10 { - assert_eq!( - chunks[i], - ChunkHandle { - index: i as FileChunkIndex, - offset: 0, - len: FILE_CHUNK_SIZE as FileSize, - } - ); - } - } - #[test] fn seek_and_tell() { let mut fs = test_fs(); diff --git a/src/runtime/structure_helpers.rs b/src/runtime/structure_helpers.rs index 1fba20a..2309b33 100644 --- a/src/runtime/structure_helpers.rs +++ b/src/runtime/structure_helpers.rs @@ -1,7 +1,13 @@ +use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES; +use ic_stable_structures::Memory; + use crate::{ error::Error, storage::{ - types::{DirEntry, DirEntryIndex, FileName, FileType, Metadata, Node, Times}, + types::{ + ChunkHandle, DirEntry, DirEntryIndex, FileChunkIndex, FileName, FileSize, FileType, + Metadata, Node, Times, FILE_CHUNK_SIZE, + }, Storage, }, }; @@ -383,6 +389,44 @@ pub fn rm_dir_entry( Ok((removed_dir_entry_node, removed_metadata)) } +pub fn grow_memory(memory: &dyn Memory, max_address: FileSize) { + let pages_required = (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; + + let cur_pages = memory.size(); + + if cur_pages < pages_required { + memory.grow(pages_required - cur_pages); + } +} + +pub fn offset_to_file_chunk_index(offset: FileSize) -> FileChunkIndex { + (offset / FILE_CHUNK_SIZE as FileSize) as FileChunkIndex +} + +pub fn file_chunk_index_to_offset(index: FileChunkIndex) -> FileSize { + index as FileSize * FILE_CHUNK_SIZE as FileSize +} + +pub fn get_chunk_infos(start: FileSize, end: FileSize) -> Vec { + let mut result = vec![]; + let start_index = offset_to_file_chunk_index(start); + let end_index = offset_to_file_chunk_index(end); + for index in start_index..=end_index { + let start_of_chunk = file_chunk_index_to_offset(index); + assert!(start_of_chunk <= end); + let start_in_chunk = start_of_chunk.max(start) - start_of_chunk; + let end_in_chunk = (start_of_chunk + FILE_CHUNK_SIZE as FileSize).min(end) - start_of_chunk; + if start_in_chunk < end_in_chunk { + result.push(ChunkHandle { + index, + offset: start_in_chunk, + len: end_in_chunk - start_in_chunk, + }); + } + } + result +} + #[cfg(test)] mod tests { @@ -390,10 +434,63 @@ mod tests { use crate::{ error::Error, - runtime::structure_helpers::{create_path, find_node}, - storage::{stable::StableStorage, types::FileType, Storage}, + runtime::structure_helpers::{create_path, find_node, get_chunk_infos}, + storage::{ + stable::StableStorage, + types::{ChunkHandle, FileChunkIndex, FileSize, FileType, FILE_CHUNK_SIZE}, + Storage, + }, }; + #[test] + fn get_chunk_infos_parital() { + let chunks = get_chunk_infos( + FILE_CHUNK_SIZE as FileSize - 1, + 2 * FILE_CHUNK_SIZE as FileSize + 1, + ); + assert_eq!( + chunks[0], + ChunkHandle { + index: 0, + offset: FILE_CHUNK_SIZE as FileSize - 1, + len: 1 + } + ); + assert_eq!( + chunks[1], + ChunkHandle { + index: 1, + offset: 0, + len: FILE_CHUNK_SIZE as FileSize, + } + ); + + assert_eq!( + chunks[2], + ChunkHandle { + index: 2, + offset: 0, + len: 1, + } + ); + } + + #[test] + fn get_chunk_infos_full() { + let chunks = get_chunk_infos(0, 10 * FILE_CHUNK_SIZE as FileSize); + #[allow(clippy::needless_range_loop)] + for i in 0..10 { + assert_eq!( + chunks[i], + ChunkHandle { + index: i as FileChunkIndex, + offset: 0, + len: FILE_CHUNK_SIZE as FileSize, + } + ); + } + } + #[test] fn create_path_with_subfolders() { let mut storage_box = Box::new(StableStorage::new(DefaultMemoryImpl::default())); diff --git a/src/storage.rs b/src/storage.rs index 4ad0c45..4a4eb53 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -30,6 +30,11 @@ pub trait Storage { // return mounted memory related to the node, or None fn get_mounted_memory(&self, node: Node) -> Option<&dyn Memory>; + // initialize memory with the contents from file + fn init_mounted_memory(&mut self, node: Node) -> Result<(), Error>; + // store mounted memory state back to host file + fn store_mounted_memory(&mut self, node: Node) -> Result<(), Error>; + // Get the metadata associated with the node. fn get_metadata(&self, node: Node) -> Result; // Update the metadata associated with the node. @@ -64,7 +69,15 @@ pub trait Storage { ) -> Result; // Insert of update a selected file chunk with the data provided in buffer. - fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]); + //fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]); + + // Write file at the current file cursor, the cursor position will NOT be updated after reading. + fn write_with_offset( + &mut self, + node: Node, + offset: FileSize, + buf: &[u8], + ) -> Result; // Remove file chunk from a given file node. fn rm_filechunk(&mut self, node: Node, index: FileChunkIndex); diff --git a/src/storage/dummy.rs b/src/storage/dummy.rs index ca7b566..d7ae5b2 100644 --- a/src/storage/dummy.rs +++ b/src/storage/dummy.rs @@ -67,16 +67,6 @@ impl Storage for DummyStorage { panic!("Not supported") } - fn write_filechunk( - &mut self, - _node: Node, - _index: FileChunkIndex, - _offset_in_first_chunk: FileSize, - _buf: &[u8], - ) { - panic!("Not supported") - } - fn rm_filechunk(&mut self, _node: Node, _index: FileChunkIndex) { panic!("Not supported") } @@ -113,6 +103,23 @@ impl Storage for DummyStorage { fn get_mounted_memory(&self, _node: Node) -> Option<&dyn ic_stable_structures::Memory> { panic!("Not supported") } + + fn init_mounted_memory(&mut self, _node: Node) -> Result<(), Error> { + panic!("Not supported") + } + + fn store_mounted_memory(&mut self, _node: Node) -> Result<(), Error> { + panic!("Not supported") + } + + fn write_with_offset( + &mut self, + _node: Node, + _offset: FileSize, + _buf: &[u8], + ) -> Result { + panic!("Not supported") + } } #[cfg(test)] @@ -205,13 +212,6 @@ mod tests { let _ = storage.read_filechunk(0, 0, 0, &mut []); } - #[test] - #[should_panic] - fn write_filechunk_panic() { - let mut storage = DummyStorage::new(); - storage.write_filechunk(0, 0, 0, &[]); - } - #[test] #[should_panic] fn rm_filechunk_panic() { diff --git a/src/storage/stable.rs b/src/storage/stable.rs index ea54fbe..b80ff72 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -6,7 +6,10 @@ use ic_stable_structures::{ BTreeMap, Cell, Memory, }; -use crate::error::Error; +use crate::{ + error::Error, + runtime::structure_helpers::{get_chunk_infos, grow_memory}, +}; use super::{ types::{ @@ -138,6 +141,27 @@ impl StableStorage { result } + + // Insert of update a selected file chunk with the data provided in a buffer. + fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { + if let Some(memory) = self.get_mounted_memory(node) { + // grow memory if needed + let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + + offset as FileSize + + buf.len() as FileSize; + + grow_memory(memory, max_address); + + // store data + let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + + memory.write(address, buf); + } else { + let mut entry = self.filechunk.get(&(node, index)).unwrap_or_default(); + entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf); + self.filechunk.insert((node, index), entry); + } + } } impl Storage for StableStorage { @@ -287,31 +311,33 @@ impl Storage for StableStorage { Ok(size_read) } - // Insert of update a selected file chunk with the data provided in a buffer. - fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { - if let Some(memory) = self.active_mounts.get(&node) { - // grow memory if needed - let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize - + offset as FileSize - + buf.len() as FileSize; - let pages_required = - (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; - - let cur_pages = memory.size(); - - if cur_pages < pages_required { - memory.grow(pages_required - cur_pages); - } - - // store data - let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + // Write file at the current file cursor, the cursor position will NOT be updated after reading. + fn write_with_offset( + &mut self, + node: Node, + offset: FileSize, + buf: &[u8], + ) -> Result { + let mut metadata = self.get_metadata(node)?; + let end = offset + buf.len() as FileSize; + let chunk_infos = get_chunk_infos(offset, end); + let mut written_size = 0; + for chunk in chunk_infos.into_iter() { + self.write_filechunk( + node, + chunk.index, + chunk.offset, + &buf[written_size..written_size + chunk.len as usize], + ); + written_size += chunk.len as usize; + } - memory.write(address, buf); - } else { - let mut entry = self.filechunk.get(&(node, index)).unwrap_or_default(); - entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf); - self.filechunk.insert((node, index), entry); + if end > metadata.size { + metadata.size = end; + self.put_metadata(node, metadata) } + + Ok(written_size as FileSize) } // Remove file chunk from a given file node. @@ -358,6 +384,76 @@ impl Storage for StableStorage { res.map(|b| b.as_ref()) } + + fn init_mounted_memory(&mut self, node: Node) -> Result<(), Error> { + // temporary disable mount to activate access to the original file + let memory = self.unmount_node(node)?; + + let meta = self.get_metadata(node)?; + let file_size = meta.size; + println!("stored file size {file_size}"); + + // grow memory if needed + grow_memory(memory.as_ref(), file_size); + + let mut remainder = file_size; + + let mut buf = [0u8; WASM_PAGE_SIZE_IN_BYTES as usize]; + + let mut offset = 0; + + while remainder > 0 { + let to_read = remainder.min(buf.len() as FileSize); + + self.read_range(node, offset, file_size, &mut buf[..to_read as usize])?; + + memory.write(offset, &buf[..to_read as usize]); + + offset += to_read; + remainder -= to_read; + } + + self.mount_node(node, memory)?; + + self.put_metadata(node, meta); + + Ok(()) + } + + fn store_mounted_memory(&mut self, node: Node) -> Result<(), Error> { + // get current size of the mounted memory + let meta = self.get_metadata(node)?; + let file_size = meta.size; + + // temporary disable mount to activate access to the original file + let memory = self.unmount_node(node)?; + + // grow memory if needed + grow_memory(memory.as_ref(), file_size); + + let mut remainder = file_size; + + let mut buf = [0u8; WASM_PAGE_SIZE_IN_BYTES as usize]; + + let mut offset = 0; + + while remainder > 0 { + let to_read = remainder.min(buf.len() as FileSize); + + memory.read(offset, &mut buf[..to_read as usize]); + + self.write_with_offset(node, offset, &buf[..to_read as usize])?; + + offset += to_read; + remainder -= to_read; + } + + self.put_metadata(node, meta); + + self.mount_node(node, memory)?; + + Ok(()) + } } #[cfg(test)] diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 3864185..65ca43f 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -5,11 +5,14 @@ use ic_stable_structures::Memory; use crate::{ error::Error, - storage::types::{ - DirEntry, DirEntryIndex, FileChunk, FileChunkIndex, FileSize, FileType, Metadata, Node, - Times, + runtime::structure_helpers::{get_chunk_infos, grow_memory}, + storage::{ + types::{ + DirEntry, DirEntryIndex, FileChunk, FileChunkIndex, FileSize, FileType, Metadata, Node, + Times, + }, + Storage, }, - storage::Storage, }; use super::types::{Header, FILE_CHUNK_SIZE}; @@ -62,6 +65,25 @@ impl TransientStorage { result.put_metadata(ROOT_NODE, metadata); result } + + // Insert of update a selected file chunk with the data provided in buffer. + fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { + if let Some(memory) = self.get_mounted_memory(node) { + // grow memory if needed + let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize + + offset as FileSize + + buf.len() as FileSize; + + grow_memory(memory, max_address); + + // store data + let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; + memory.write(address, buf); + } else { + let entry = self.filechunk.entry((node, index)).or_default(); + entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf) + } + } } impl Storage for TransientStorage { @@ -208,31 +230,6 @@ impl Storage for TransientStorage { Ok(size_read) } - // Insert of update a selected file chunk with the data provided in buffer. - fn write_filechunk(&mut self, node: Node, index: FileChunkIndex, offset: FileSize, buf: &[u8]) { - if let Some(memory) = self.get_mounted_memory(node) { - // grow memory if needed - let max_address = index as FileSize * FILE_CHUNK_SIZE as FileSize - + offset as FileSize - + buf.len() as FileSize; - let pages_required = - (max_address + WASM_PAGE_SIZE_IN_BYTES - 1) / WASM_PAGE_SIZE_IN_BYTES; - - let cur_pages = memory.size(); - - if cur_pages < pages_required { - memory.grow(pages_required - cur_pages); - } - - // store data - let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; - memory.write(address, buf); - } else { - let entry = self.filechunk.entry((node, index)).or_default(); - entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf) - } - } - // Remove file chunk from a given file node. fn rm_filechunk(&mut self, node: Node, index: FileChunkIndex) { self.filechunk.remove(&(node, index)); @@ -277,6 +274,103 @@ impl Storage for TransientStorage { res.map(|b| b.as_ref()) } + + fn init_mounted_memory(&mut self, node: Node) -> Result<(), Error> { + // temporary disable mount to activate access to the original file + let memory = self.unmount_node(node)?; + + let meta = self.get_metadata(node)?; + let file_size = meta.size; + + // grow memory if needed + grow_memory(memory.as_ref(), file_size); + + let mut remainder = file_size; + + let mut buf = [0u8; WASM_PAGE_SIZE_IN_BYTES as usize]; + + let mut offset = 0; + + while remainder > 0 { + let to_read = remainder.min(buf.len() as FileSize); + + self.read_range(node, offset, file_size, &mut buf[..to_read as usize])?; + + memory.write(offset, &buf[..to_read as usize]); + + offset += to_read; + remainder -= to_read; + } + + self.mount_node(node, memory)?; + + self.put_metadata(node, meta); + + Ok(()) + } + + fn store_mounted_memory(&mut self, node: Node) -> Result<(), Error> { + // get current size of the mounted memory + let meta = self.get_metadata(node)?; + let file_size = meta.size; + + // temporary disable mount to activate access to the original file + let memory = self.unmount_node(node)?; + + // grow memory if needed + grow_memory(memory.as_ref(), file_size); + + let mut remainder = file_size; + + let mut buf = [0u8; WASM_PAGE_SIZE_IN_BYTES as usize]; + + let mut offset = 0; + + while remainder > 0 { + let to_read = remainder.min(buf.len() as FileSize); + + memory.read(offset, &mut buf[..to_read as usize]); + + self.write_with_offset(node, offset, &buf[..to_read as usize])?; + + offset += to_read; + remainder -= to_read; + } + + self.put_metadata(node, meta); + + self.mount_node(node, memory)?; + + Ok(()) + } + + fn write_with_offset( + &mut self, + node: Node, + offset: FileSize, + buf: &[u8], + ) -> Result { + let mut metadata = self.get_metadata(node)?; + let end = offset + buf.len() as FileSize; + let chunk_infos = get_chunk_infos(offset, end); + let mut written_size = 0; + for chunk in chunk_infos.into_iter() { + self.write_filechunk( + node, + chunk.index, + chunk.offset, + &buf[written_size..written_size + chunk.len as usize], + ); + written_size += chunk.len as usize; + } + + if end > metadata.size { + metadata.size = end; + self.put_metadata(node, metadata) + } + + Ok(written_size as FileSize) + } } #[cfg(test)] diff --git a/src/storage/types.rs b/src/storage/types.rs index 2e96aea..f1baddc 100644 --- a/src/storage/types.rs +++ b/src/storage/types.rs @@ -15,6 +15,14 @@ pub type FileSize = u64; // An index of a file chunk. pub type FileChunkIndex = u32; +// A handle used for writing files in chunks +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct ChunkHandle { + pub index: FileChunkIndex, + pub offset: FileSize, + pub len: FileSize, +} + // A file consists of multiple file chunks. #[derive(Clone, Debug)] pub struct FileChunk { diff --git a/tests/demo_test_upgraded/Cargo.lock b/tests/demo_test_upgraded/Cargo.lock index 6eea1bf..909c616 100644 --- a/tests/demo_test_upgraded/Cargo.lock +++ b/tests/demo_test_upgraded/Cargo.lock @@ -620,7 +620,7 @@ dependencies = [ [[package]] name = "stable-fs" -version = "0.5.1" +version = "0.6.0" dependencies = [ "bitflags", "ciborium", From 66dc54539da390af01c8be7e0c560e13346e8ce5 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Wed, 7 Aug 2024 02:54:27 +0200 Subject: [PATCH 12/13] clean-up --- src/error.rs | 4 ++-- src/fs.rs | 38 +++++++++----------------------------- src/storage/stable.rs | 4 ++-- src/storage/transient.rs | 4 ++-- 4 files changed, 15 insertions(+), 35 deletions(-) diff --git a/src/error.rs b/src/error.rs index 256e3bd..3552834 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,8 +9,8 @@ pub enum Error { InvalidOpenFlags, InvalidFdFlags, FileAlreadyExists, - FileIsNotMounted, - IsMountedAlready, + MemoryFileIsNotMounted, + MemoryFileIsMountedAlready, NameTooLong, DirectoryNotEmpty, ExpectedToRemoveFile, diff --git a/src/fs.rs b/src/fs.rs index 7525dde..67c37f5 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -100,8 +100,8 @@ impl FileSystem { } } - // mounte memory file on the top of the given file name, if the file does not exist, it will be created. - // The method fails if the file system couldn't create the file. + // mount memory on the top of the given host file name, if the file does not exist, it will be created. + // The method fails if the file system could not open or create the file. pub fn mount_memory_file( &mut self, filename: &str, @@ -141,29 +141,6 @@ impl FileSystem { self.storage.init_mounted_memory(node) } - // helper method to set the size of the mounted memory - pub fn set_memory_file_size( - &mut self, - filename: &str, - new_size: FileSize, - ) -> Result<(), Error> { - let root_node = self.get_node(self.root_fd())?; - - let node = find_node(root_node, filename, self.storage.as_ref())?; - - if !self.storage.is_mounted(node) { - return Err(Error::FileIsNotMounted); - } - - let mut metadata = self.metadata_from_node(node)?; - - metadata.size = new_size; - - self.storage.put_metadata(node, metadata); - - Ok(()) - } - // store content of the currently active memory file to the file system pub fn store_memory_file(&mut self, filename: &str) -> Result<(), Error> { // create a file for the mount @@ -181,7 +158,7 @@ impl FileSystem { self.storage.store_mounted_memory(node) } - // Unmount memory file, the system will continue to work with its own memory + // Unmount memory, the system will continue to work with the file in normal mode. pub fn unmount_memory_file(&mut self, filename: &str) -> Result, Error> { // create a file for the mount let fd = self.open_or_create( @@ -200,7 +177,7 @@ impl FileSystem { Ok(memory) } - // Get dir entry for a given directory and the directory index. + // Get directory entry for a given directory file descriptor and the entry index. pub fn get_direntry(&self, fd: Fd, index: DirEntryIndex) -> Result { self.get_dir(fd)?.get_entry(index, self.storage.as_ref()) } @@ -1355,8 +1332,11 @@ mod tests { memory1.write(i as u64 * len as u64, content.as_bytes()); } - fs.set_memory_file_size(file_name, len as FileSize * count as FileSize) - .unwrap(); + let fd = fs.open_or_create(fs.root_fd, file_name, FdStat::default(), OpenFlags::empty(), 0).unwrap(); + let mut metadata = fs.metadata(fd).unwrap(); + metadata.size = len as FileSize * count as FileSize; + fs.set_metadata(fd, metadata).unwrap(); + fs.close(fd).unwrap(); // store memory into a file fs.store_memory_file(file_name).unwrap(); diff --git a/src/storage/stable.rs b/src/storage/stable.rs index b80ff72..f90a2ab 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -347,7 +347,7 @@ impl Storage for StableStorage { fn mount_node(&mut self, node: Node, memory: Box) -> Result<(), Error> { if self.is_mounted(node) { - return Err(Error::IsMountedAlready); + return Err(Error::MemoryFileIsMountedAlready); } // do extra meta preparation @@ -372,7 +372,7 @@ impl Storage for StableStorage { fn unmount_node(&mut self, node: Node) -> Result, Error> { let memory = self.active_mounts.remove(&node); - memory.ok_or(Error::FileIsNotMounted) + memory.ok_or(Error::MemoryFileIsNotMounted) } fn is_mounted(&self, node: Node) -> bool { diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 65ca43f..066ffda 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -237,7 +237,7 @@ impl Storage for TransientStorage { fn mount_node(&mut self, node: Node, memory: Box) -> Result<(), Error> { if self.is_mounted(node) { - return Err(Error::IsMountedAlready); + return Err(Error::MemoryFileIsMountedAlready); } // do extra meta preparation @@ -262,7 +262,7 @@ impl Storage for TransientStorage { fn unmount_node(&mut self, node: Node) -> Result, Error> { let memory = self.active_mounts.remove(&node); - memory.ok_or(Error::FileIsNotMounted) + memory.ok_or(Error::MemoryFileIsNotMounted) } fn is_mounted(&self, node: Node) -> bool { From 21db75717b0a7aa3a3d8c1bb905a5eed4798fb10 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:23:47 +0200 Subject: [PATCH 13/13] formatting --- src/fs.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/fs.rs b/src/fs.rs index 67c37f5..02690c9 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1332,7 +1332,15 @@ mod tests { memory1.write(i as u64 * len as u64, content.as_bytes()); } - let fd = fs.open_or_create(fs.root_fd, file_name, FdStat::default(), OpenFlags::empty(), 0).unwrap(); + let fd = fs + .open_or_create( + fs.root_fd, + file_name, + FdStat::default(), + OpenFlags::empty(), + 0, + ) + .unwrap(); let mut metadata = fs.metadata(fd).unwrap(); metadata.size = len as FileSize * count as FileSize; fs.set_metadata(fd, metadata).unwrap();