From adb29958dba22cb0f51e8534b9bb59cb0c1f0c02 Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Mon, 12 Aug 2024 07:29:05 +0200 Subject: [PATCH] improved texts, clean-up --- src/integration_tests.rs | 10 ++- src/runtime/file.rs | 2 +- src/storage/stable.rs | 64 +++++++++++-------- src/storage/transient.rs | 25 +------- src/storage/types.rs | 9 ++- .../src/demo_test_upgraded_backend/src/lib.rs | 2 +- .../src/fs_benchmark_test_backend/src/lib.rs | 2 +- tests/fs_benchmark_test/store_stable.sh | 2 +- 8 files changed, 54 insertions(+), 62 deletions(-) diff --git a/src/integration_tests.rs b/src/integration_tests.rs index e218278..49a0a58 100644 --- a/src/integration_tests.rs +++ b/src/integration_tests.rs @@ -333,7 +333,7 @@ fn create_1000_files() { let result = fns::list_files(&pic, ""); - let filenames = vec!["stable_file.txt", "files1", "files2", "files3", "files4"]; + let filenames = vec!["mount_file.txt", "files1", "files2", "files3", "files4"]; assert_eq!(result, filenames); } @@ -341,7 +341,7 @@ fn create_1000_files() { fn no_virtual_names(vec: Vec) -> Vec { let mut v = vec; - v.retain(|v| !(*v).eq("stable_file.txt")); + v.retain(|v| !(*v).eq("mount_file.txt")); v } @@ -445,7 +445,7 @@ fn large_file_read() { fn large_file_read_after_upgrade() { let pic = setup_initial_canister(); - let filename = "stable_file.txt"; + let filename = "mount_file.txt"; // create large file fns::append_text(&pic, "t1.txt", "abcdef7890", 10_000_000); @@ -473,7 +473,7 @@ fn large_file_read_after_upgrade() { fn large_mounted_file_write() { let pic = setup_initial_canister(); - let filename = "stable_file.txt"; + let filename = "mount_file.txt"; // create large buffer fns::append_buffer(&pic, "abcdef7890", 10_000_000); @@ -510,5 +510,3 @@ fn large_file_write() { assert_eq!(size, 100_000_000); } - - diff --git a/src/runtime/file.rs b/src/runtime/file.rs index f73e70a..48932db 100644 --- a/src/runtime/file.rs +++ b/src/runtime/file.rs @@ -139,7 +139,7 @@ impl File { mod tests { use crate::{ fs::OpenFlags, - test_utils::{test_fs, test_fs_setups, test_fs_transient}, + test_utils::{test_fs, test_fs_setups}, }; use super::*; diff --git a/src/storage/stable.rs b/src/storage/stable.rs index ff5646c..51436f2 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -167,12 +167,25 @@ impl ChunkPtrAllocator { #[cfg(test)] self.check_free(ptr); - println!("release: {}", ptr / FILE_CHUNK_SIZE as u64); - self.push_ptr(ptr); } } + +struct StorageMemories { + header_memory: VirtualMemory, + metadata_memory: VirtualMemory, + direntry_memory: VirtualMemory, + filechunk_memory: VirtualMemory, + mounted_meta_memory: VirtualMemory, + + v2_chunk_ptr_memory: VirtualMemory, + v2_chunks_memory: VirtualMemory, + v2_allocator_memory: VirtualMemory, + +} + + #[repr(C)] pub struct StableStorage { header: Cell>, @@ -227,51 +240,48 @@ impl StableStorage { let metadata_memory = memory_manager.get(MemoryId::new(memory_indices.start + 1u8)); let direntry_memory = memory_manager.get(MemoryId::new(memory_indices.start + 2u8)); let filechunk_memory = memory_manager.get(MemoryId::new(memory_indices.start + 3u8)); - let mounted_meta = memory_manager.get(MemoryId::new(memory_indices.start + 4u8)); + let mounted_meta_memory = memory_manager.get(MemoryId::new(memory_indices.start + 4u8)); - let v2_chunk_ptr = memory_manager.get(MemoryId::new(memory_indices.start + 5u8)); - let v2_chunks = memory_manager.get(MemoryId::new(memory_indices.start + 7u8)); + let v2_chunk_ptr_memory = memory_manager.get(MemoryId::new(memory_indices.start + 5u8)); + let v2_chunks_memory = memory_manager.get(MemoryId::new(memory_indices.start + 7u8)); let v2_allocator_memory = memory_manager.get(MemoryId::new(memory_indices.start + 6u8)); - Self::new_with_custom_memories( + let memories = StorageMemories { header_memory, metadata_memory, direntry_memory, filechunk_memory, - mounted_meta, - v2_chunk_ptr, - v2_chunks, + mounted_meta_memory, + v2_chunk_ptr_memory, + v2_chunks_memory, v2_allocator_memory, + }; + + Self::new_with_custom_memories( + memories ) } fn new_with_custom_memories( - header: VirtualMemory, - metadata: VirtualMemory, - direntry: VirtualMemory, - filechunk: VirtualMemory, - mounted_meta: VirtualMemory, - - v2_chunk_ptr: VirtualMemory, - v2_chunks: VirtualMemory, - v2_allocator_memory: VirtualMemory, + memories: StorageMemories + ) -> Self { let default_header_value = Header { version: FS_VERSION, next_node: ROOT_NODE + 1, }; - let v2_allocator = ChunkPtrAllocator::new(v2_allocator_memory); + let v2_allocator = ChunkPtrAllocator::new(memories.v2_allocator_memory); let mut result = Self { - header: Cell::init(header, default_header_value).unwrap(), - metadata: BTreeMap::init(metadata), - direntry: BTreeMap::init(direntry), - filechunk: BTreeMap::init(filechunk), - mounted_meta: BTreeMap::init(mounted_meta), - - v2_chunk_ptr: BTreeMap::init(v2_chunk_ptr), - v2_chunks, + header: Cell::init(memories.header_memory, default_header_value).unwrap(), + metadata: BTreeMap::init(memories.metadata_memory), + direntry: BTreeMap::init(memories.direntry_memory), + filechunk: BTreeMap::init(memories.filechunk_memory), + mounted_meta: BTreeMap::init(memories.mounted_meta_memory), + + v2_chunk_ptr: BTreeMap::init(memories.v2_chunk_ptr_memory), + v2_chunks: memories.v2_chunks_memory, v2_allocator, // runtime data diff --git a/src/storage/transient.rs b/src/storage/transient.rs index 0be2f8e..0d4f694 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -34,7 +34,7 @@ pub struct TransientStorage { filechunk: BTreeMap<(Node, FileChunkIndex), FileChunk>, // Mounted memory Node metadata information. mounted_meta: BTreeMap, - // active mounts + // Active mounts. active_mounts: HashMap>, } @@ -53,7 +53,7 @@ impl TransientStorage { let mut result = Self { header: Header { version: 1, - next_node: ROOT_NODE + 1, + next_node: ROOT_NODE + 1 }, metadata: Default::default(), direntry: Default::default(), @@ -84,27 +84,6 @@ impl TransientStorage { entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf) } } - - // Fill the buffer contents with data of a chosen file chunk. - #[cfg(test)] - fn read_filechunk( - &self, - node: Node, - index: FileChunkIndex, - offset: FileSize, - buf: &mut [u8], - ) -> Result<(), Error> { - if let Some(memory) = self.get_mounted_memory(node) { - // work with memory - let address = index as FileSize * FILE_CHUNK_SIZE as FileSize + offset as FileSize; - memory.read(address, buf); - } else { - let value = self.filechunk.get(&(node, index)).ok_or(Error::NotFound)?; - buf.copy_from_slice(&value.bytes[offset as usize..offset as usize + buf.len()]); - } - - Ok(()) - } } impl Storage for TransientStorage { diff --git a/src/storage/types.rs b/src/storage/types.rs index c1366fd..daa101f 100644 --- a/src/storage/types.rs +++ b/src/storage/types.rs @@ -2,8 +2,13 @@ use crate::error::Error; use ic_stable_structures::storable::Bound; use serde::{Deserialize, Serialize}; -//pub const FILE_CHUNK_SIZE: usize = 4096; -pub const FILE_CHUNK_SIZE: usize = 8192; +//pub const FILE_CHUNK_SIZE: usize = 1024; +//pub const FILE_CHUNK_SIZE: usize = 2048; +pub const FILE_CHUNK_SIZE: usize = 4096; +//pub const FILE_CHUNK_SIZE: usize = 8192; +//pub const FILE_CHUNK_SIZE: usize = 16384; +//pub const FILE_CHUNK_SIZE: usize = 32768; + pub const MAX_FILE_NAME: usize = 255; // The unique identifier of a node, which can be a file or a directory. diff --git a/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs b/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs index 96ada20..b831975 100644 --- a/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs +++ b/tests/demo_test_upgraded/src/demo_test_upgraded_backend/src/lib.rs @@ -39,7 +39,7 @@ thread_local! { FileSystem::new(Box::new(storage)).unwrap() ); - fs.borrow_mut().mount_memory_file("stable_file.txt", Box::new(memory_manager.get(MemoryId::new(155)))).unwrap(); + fs.borrow_mut().mount_memory_file("mount_file.txt", Box::new(memory_manager.get(MemoryId::new(155)))).unwrap(); fs }) diff --git a/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs b/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs index cc1eaf3..8d84d76 100644 --- a/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs +++ b/tests/fs_benchmark_test/src/fs_benchmark_test_backend/src/lib.rs @@ -39,7 +39,7 @@ thread_local! { FileSystem::new(Box::new(storage)).unwrap() ); - fs.borrow_mut().mount_memory_file("stable_file.txt", Box::new(memory_manager.get(MemoryId::new(155)))).unwrap(); + fs.borrow_mut().mount_memory_file("mount_file.txt", Box::new(memory_manager.get(MemoryId::new(155)))).unwrap(); fs }) diff --git a/tests/fs_benchmark_test/store_stable.sh b/tests/fs_benchmark_test/store_stable.sh index 10dfd89..dcd49fc 100755 --- a/tests/fs_benchmark_test/store_stable.sh +++ b/tests/fs_benchmark_test/store_stable.sh @@ -1,3 +1,3 @@ #!/bin/bash -dfx canister call fs_benchmark_test_backend store_buffer '("stable_file.txt")' +dfx canister call fs_benchmark_test_backend store_buffer '("mount_file.txt")'