From b844377d9c9a918745f19b59e1424700e80820ff Mon Sep 17 00:00:00 2001 From: wasm-forge <122647775+wasm-forge@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:35:35 +0100 Subject: [PATCH] add maximum size control --- .gitignore | 3 +- src/error.rs | 7 ++ src/fs.rs | 25 +++-- src/runtime/file.rs | 2 +- src/runtime/structure_helpers.rs | 12 +-- src/storage.rs | 5 +- src/storage/chunk_iterator.rs | 29 ++--- src/storage/dummy.rs | 34 +++--- src/storage/metadata_provider.rs | 47 ++++----- src/storage/stable.rs | 176 +++++++++++++++++++++++-------- src/storage/transient.rs | 118 ++++++++++++++++----- src/storage/types.rs | 5 +- 12 files changed, 322 insertions(+), 141 deletions(-) diff --git a/.gitignore b/.gitignore index 8057107..6de82ca 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ lcov.info tarpaulin-report.html -tests/svg/* \ No newline at end of file +tests/svg/* +.codegpt \ No newline at end of file diff --git a/src/error.rs b/src/error.rs index 3a279fb..3dc5e2d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ #[derive(Debug, PartialEq, Eq)] pub enum Error { NotFound, + Unimplemented, InvalidOffset, InvalidFileType, InvalidFileName, @@ -19,4 +20,10 @@ pub enum Error { CannotRemoveMountedMemoryFile, IncompatibleChunkSize, InvalidMagicMarker, + + MaxFileSizeExceeded, + NoSpaceLeftOnDevice, + OperationNotSupported, + MetadataUpdateInvalid, + TooManyOpenFiles, } diff --git a/src/fs.rs b/src/fs.rs index 4e3987b..840486f 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -317,9 +317,22 @@ impl FileSystem { } // update metadata of a given file descriptor - pub fn set_metadata(&mut self, fd: Fd, metadata: Metadata) -> Result<(), Error> { + pub fn set_metadata(&mut self, fd: Fd, metadata: &Metadata) -> Result<(), Error> { let node = self.get_node(fd)?; - self.storage.put_metadata(node, metadata); + self.storage.put_metadata(node, metadata)?; + + Ok(()) + } + + // Set maxmum file size limit in bytes. Reading, writing, and setting the cursor above the limit will result in error. + // Use this feature to limit, how much memory can be consumed by the mounted memory files. + pub fn set_file_size_limit(&mut self, fd: Fd, max_size: FileSize) -> Result<(), Error> { + let node = self.get_node(fd)?; + let mut metadata = self.storage.get_metadata(node)?; + + metadata.maximum_size_allowed = Some(max_size); + + self.storage.put_metadata(node, &metadata)?; Ok(()) } @@ -331,7 +344,7 @@ impl FileSystem { metadata.times.accessed = time; - self.storage.put_metadata(node, metadata); + self.storage.put_metadata(node, &metadata)?; Ok(()) } @@ -343,7 +356,7 @@ impl FileSystem { metadata.times.modified = time; - self.storage.put_metadata(node, metadata); + self.storage.put_metadata(node, &metadata)?; Ok(()) } @@ -1488,7 +1501,7 @@ mod tests { .unwrap(); let mut metadata = fs.metadata(fd).unwrap(); metadata.size = len as FileSize * count as FileSize; - fs.set_metadata(fd, metadata).unwrap(); + fs.set_metadata(fd, &metadata).unwrap(); fs.close(fd).unwrap(); // store memory into a file @@ -1570,7 +1583,7 @@ mod tests { let mut meta = fs.metadata(fd).unwrap(); meta.size = size; - fs.set_metadata(fd, meta).unwrap(); + fs.set_metadata(fd, &meta).unwrap(); fd } diff --git a/src/runtime/file.rs b/src/runtime/file.rs index 48932db..739e9ae 100644 --- a/src/runtime/file.rs +++ b/src/runtime/file.rs @@ -130,7 +130,7 @@ impl File { pub fn truncate(&self, storage: &mut dyn Storage) -> Result<(), Error> { let mut metadata = storage.get_metadata(self.node)?; metadata.size = 0; - storage.put_metadata(self.node, metadata); + storage.put_metadata(self.node, &metadata)?; Ok(()) } } diff --git a/src/runtime/structure_helpers.rs b/src/runtime/structure_helpers.rs index cb49700..fd00aaf 100644 --- a/src/runtime/structure_helpers.rs +++ b/src/runtime/structure_helpers.rs @@ -121,7 +121,7 @@ pub fn create_hard_link( } metadata.link_count += 1; - storage.put_metadata(node, metadata); + storage.put_metadata(node, &metadata)?; add_dir_entry(dir_node, node, leaf_name.as_bytes(), storage)?; @@ -149,7 +149,7 @@ pub fn create_dir_entry( storage.put_metadata( node, - Metadata { + &Metadata { node, file_type: entry_type, link_count: 1, @@ -164,7 +164,7 @@ pub fn create_dir_entry( chunk_type, maximum_size_allowed: None, }, - ); + )?; add_dir_entry(parent_dir_node, node, entry_name, storage)?; @@ -326,7 +326,7 @@ pub fn add_dir_entry( } metadata.size += 1; - storage.put_metadata(parent_dir_node, metadata); + storage.put_metadata(parent_dir_node, &metadata)?; Ok(()) } @@ -418,13 +418,13 @@ pub fn rm_dir_entry( parent_dir_metadata.size -= 1; // update parent metadata - storage.put_metadata(parent_dir_node, parent_dir_metadata); + storage.put_metadata(parent_dir_node, &parent_dir_metadata)?; // remove the entry storage.rm_direntry(parent_dir_node, removed_entry_index); removed_metadata.link_count -= 1; - storage.put_metadata(removed_metadata.node, removed_metadata.clone()); + storage.put_metadata(removed_metadata.node, &removed_metadata)?; Ok((removed_dir_entry_node, removed_metadata)) } diff --git a/src/storage.rs b/src/storage.rs index fb9bd51..1a29254 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -45,7 +45,7 @@ pub trait Storage { // Get the metadata associated with the node. fn get_metadata(&self, node: Node) -> Result; // Update the metadata associated with the node. - fn put_metadata(&mut self, node: Node, metadata: Metadata); + fn put_metadata(&mut self, node: Node, metadata: &Metadata) -> Result<(), Error>; // Retrieve the DirEntry instance given the Node and DirEntryIndex. fn get_direntry(&self, node: Node, index: DirEntryIndex) -> Result; @@ -65,6 +65,9 @@ pub trait Storage { // Write file at the current file cursor, the cursor position will NOT be updated after reading. fn write(&mut self, node: Node, offset: FileSize, buf: &[u8]) -> Result; + // delete chunks to match the new file size specified + fn resize_file(&mut self, node: Node, new_size: FileSize) -> Result<(), Error>; + // remove all file chunks fn rm_file(&mut self, node: Node) -> Result<(), Error>; diff --git a/src/storage/chunk_iterator.rs b/src/storage/chunk_iterator.rs index 7559d55..773abd0 100644 --- a/src/storage/chunk_iterator.rs +++ b/src/storage/chunk_iterator.rs @@ -91,20 +91,23 @@ mod tests { fn create_file_with_size(size: FileSize, storage: &mut StableStorage) -> Node { let node = storage.new_node(); - storage.put_metadata( - node, - Metadata { + storage + .put_metadata( node, - file_type: FileType::RegularFile, - link_count: 1, - size, - times: Times::default(), - first_dir_entry: Some(42), - last_dir_entry: Some(24), - chunk_type: Some(storage.chunk_type()), - maximum_size_allowed: None, - }, - ); + &Metadata { + node, + file_type: FileType::RegularFile, + link_count: 1, + size, + times: Times::default(), + first_dir_entry: Some(42), + last_dir_entry: Some(24), + chunk_type: Some(storage.chunk_type()), + maximum_size_allowed: None, + }, + ) + .unwrap(); + node } diff --git a/src/storage/dummy.rs b/src/storage/dummy.rs index 165cebc..d402ded 100644 --- a/src/storage/dummy.rs +++ b/src/storage/dummy.rs @@ -36,7 +36,7 @@ impl Storage for DummyStorage { panic!("Not supported") } - fn put_metadata(&mut self, _node: Node, _metadata: Metadata) { + fn put_metadata(&mut self, _node: Node, _metadata: &Metadata) -> Result<(), Error> { panic!("Not supported") } @@ -114,6 +114,10 @@ impl Storage for DummyStorage { fn flush(&mut self, _node: Node) { panic!("Not supported") } + + fn resize_file(&mut self, _node: Node, _new_size: FileSize) -> Result<(), Error> { + panic!("Not supported") + } } #[cfg(test)] @@ -127,20 +131,22 @@ mod tests { fn put_metadata_panic() { let mut storage = DummyStorage::new(); let node = storage.new_node(); - storage.put_metadata( - node, - Metadata { + storage + .put_metadata( node, - file_type: FileType::RegularFile, - link_count: 1, - size: 10, - times: Times::default(), - first_dir_entry: Some(42), - last_dir_entry: Some(24), - chunk_type: None, - maximum_size_allowed: None, - }, - ) + &Metadata { + node, + file_type: FileType::RegularFile, + link_count: 1, + size: 10, + times: Times::default(), + first_dir_entry: Some(42), + last_dir_entry: Some(24), + chunk_type: None, + maximum_size_allowed: None, + }, + ) + .unwrap(); } #[test] diff --git a/src/storage/metadata_provider.rs b/src/storage/metadata_provider.rs index c1589ed..90aaef3 100644 --- a/src/storage/metadata_provider.rs +++ b/src/storage/metadata_provider.rs @@ -7,8 +7,7 @@ use super::types::{Metadata, Node}; use crate::fs::FileSize; use crate::runtime::structure_helpers::{grow_memory, read_obj, write_obj}; use crate::storage::ptr_cache::PtrCache; -use crate::storage::stable::ZEROES; -use crate::storage::Error; +use crate::storage::types::ZEROES; use ic_stable_structures::memory_manager::VirtualMemory; use ic_stable_structures::BTreeMap; use ic_stable_structures::Memory; @@ -64,11 +63,6 @@ impl MetadataCache { let meta = (*self.meta).borrow(); meta.get(&node).cloned() } - - pub fn get_ptr(&self, node: Node) -> std::option::Option { - let meta = (*self.meta).borrow(); - meta.get(&node).and_then(|x| x.1) - } } pub(crate) struct MetadataProvider { @@ -119,32 +113,29 @@ impl MetadataProvider { write_obj(v2_chunks, ptr, meta); } + // try to get metadata and the data pointer, or return None if not found. pub(crate) fn get_metadata( &self, node: Node, is_mounted: bool, v2_chunk_ptr: &BTreeMap<(Node, FileChunkIndex), FileChunkPtr, VirtualMemory>, v2_chunks: &VirtualMemory, - ) -> Result { + ) -> Option<(Metadata, Option)> { let (meta_index, meta_storage, meta_cache) = if is_mounted { ( MOUNTED_METADATA_CHUNK_INDEX, - &self.metadata, - &self.meta_cache, - ) - } else { - ( - METADATA_CHUNK_INDEX, &self.mounted_meta, &self.mounted_meta_cache, ) + } else { + (METADATA_CHUNK_INDEX, &self.metadata, &self.meta_cache) }; // try to get meta from cache let meta_rec = meta_cache.get(node); if let Some(meta_rec) = meta_rec { - return Ok(meta_rec.0); + return Some((meta_rec.0.clone(), meta_rec.1)); } // meta not found in cache, try to get it from the file chunks @@ -157,14 +148,14 @@ impl MetadataProvider { // update cache meta_cache.update(node, &meta, Some(meta_ptr)); - return Ok(meta); + return Some((meta, Some(meta_ptr))); } // meta not found in chunks, try to get it from the storage - let meta_found: Result = meta_storage.get(&node).ok_or(Error::NotFound); + let meta_found = meta_storage.get(&node); - if meta_found.is_err() { - // root node is not found on the new file system, just return the generated one. + if meta_found.is_none() { + // if root node is not found on the new file system, just return the generated one. if node == ROOT_NODE { let metadata = Metadata { node: ROOT_NODE, @@ -178,24 +169,26 @@ impl MetadataProvider { maximum_size_allowed: None, }; - return Ok(metadata); + return Some((metadata, None)); } } - // return error, if not found + // return None, if no metadata was found under given node let metadata = meta_found?; // update cache meta_cache.update(node, &metadata, None); - Ok(metadata) + Some((metadata, None)) } + // put new metadata value, while overwriting the existing record, at this point the metadata should already be validated pub(crate) fn put_metadata( &mut self, node: u64, is_mounted: bool, metadata: &Metadata, + meta_ptr: Option, v2_chunk_ptr: &mut BTreeMap<(Node, FileChunkIndex), FileChunkPtr, VirtualMemory>, v2_chunks: &mut VirtualMemory, v2_allocator: &mut ChunkPtrAllocator, @@ -203,15 +196,13 @@ impl MetadataProvider { assert_eq!(node, metadata.node, "Node does not match metadata.node!"); let (meta_index, meta_cache) = if is_mounted { - (MOUNTED_METADATA_CHUNK_INDEX, &self.meta_cache) + (MOUNTED_METADATA_CHUNK_INDEX, &self.mounted_meta_cache) } else { - (METADATA_CHUNK_INDEX, &self.mounted_meta_cache) + (METADATA_CHUNK_INDEX, &self.meta_cache) }; - // try to get meta pointer from cache or else chunk pointers or else create a new chunk - let mut meta_ptr = if let Some(meta_ptr) = meta_cache.get_ptr(node) { - meta_ptr - } else if let Some(meta_ptr) = v2_chunk_ptr.get(&(node, meta_index)) { + // create a new meta pointer if it was not + let meta_ptr = if let Some(meta_ptr) = meta_ptr { meta_ptr } else { let meta_ptr = v2_allocator.allocate(); diff --git a/src/storage/stable.rs b/src/storage/stable.rs index 0ad385e..c16b016 100644 --- a/src/storage/stable.rs +++ b/src/storage/stable.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, ops::Range}; +use crate::storage::types::ZEROES; use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES; use ic_stable_structures::{ memory_manager::{MemoryId, MemoryManager, VirtualMemory}, @@ -27,7 +28,7 @@ use super::{ ptr_cache::PtrCache, types::{ DirEntry, DirEntryIndex, FileChunk, FileChunkIndex, FileChunkPtr, FileSize, FileType, - Header, Metadata, Node, Times, FILE_CHUNK_SIZE_V1, MAX_FILE_CHUNK_SIZE_V2, + Header, Metadata, Node, Times, FILE_CHUNK_SIZE_V1, MAX_FILE_CHUNK_INDEX, MAX_FILE_SIZE, }, Storage, }; @@ -43,8 +44,6 @@ const MAX_MEMORY_INDEX: u8 = 254; // the number of memory indices used by the file system (currently 8 plus some reserved ids) const MEMORY_INDEX_COUNT: u8 = 10; -pub const ZEROES: [u8; MAX_FILE_CHUNK_SIZE_V2] = [0u8; MAX_FILE_CHUNK_SIZE_V2]; - // index containing cached metadata (deprecated) const MOUNTED_META_PTR: u64 = 16; @@ -202,7 +201,7 @@ impl StableStorage { fn init_size_from_cache_journal(&mut self, journal: &VirtualMemory) { // re-define old Metadata type for correct reading #[derive(Clone, Default, PartialEq)] - pub struct MetadataDep { + pub struct MetadataLegacy { pub node: Node, pub file_type: FileType, pub link_count: u64, @@ -216,7 +215,7 @@ impl StableStorage { // try recover stored mounted metadata (if any) if journal.size() > 0 { let mut mounted_node = 0u64; - let mut mounted_meta = MetadataDep::default(); + let mut mounted_meta = MetadataLegacy::default(); read_obj(journal, MOUNTED_META_PTR, &mut mounted_node); @@ -224,7 +223,7 @@ impl StableStorage { let meta_read = Metadata { node: mounted_meta.node, - file_type: mounted_meta.file_type, + file_type: FileType::RegularFile, link_count: mounted_meta.link_count, size: mounted_meta.size, times: mounted_meta.times, @@ -240,6 +239,7 @@ impl StableStorage { mounted_node, true, &meta_read, + None, &mut self.v2_chunk_ptr, &mut self.v2_chunks, &mut self.v2_allocator, @@ -540,10 +540,6 @@ impl StableStorage { Ok(size_read) } - fn flush_mounted_meta(&mut self) { - //self.meta_provider.flush_mounted_meta::(); - } - fn use_v2(&mut self, metadata: &Metadata, node: u64) -> bool { // decide if we use v2 chunks for reading/writing let use_v2 = match metadata.chunk_type { @@ -564,6 +560,25 @@ impl StableStorage { }; use_v2 } + + fn validate_metadata_update( + old_meta: Option<&Metadata>, + new_meta: &Metadata, + ) -> Result<(), Error> { + if let Some(old_meta) = old_meta { + if old_meta.node != new_meta.node { + return Err(Error::MetadataUpdateInvalid); + } + } + + if let Some(max_size) = new_meta.maximum_size_allowed { + if new_meta.size > max_size { + return Err(Error::MaxFileSizeExceeded); + } + } + + Ok(()) + } } impl Storage for StableStorage { @@ -592,24 +607,50 @@ impl Storage for StableStorage { // Get the metadata associated with the node. fn get_metadata(&self, node: Node) -> Result { - self.meta_provider.get_metadata( - node, - self.is_mounted(node), - &self.v2_chunk_ptr, - &self.v2_chunks, - ) + self.meta_provider + .get_metadata( + node, + self.is_mounted(node), + &self.v2_chunk_ptr, + &self.v2_chunks, + ) + .map(|x| x.0) + .ok_or(Error::NotFound) } // Update the metadata associated with the node. - fn put_metadata(&mut self, node: Node, metadata: Metadata) { + fn put_metadata(&mut self, node: Node, metadata: &Metadata) -> Result<(), Error> { + let is_mounted = self.is_mounted(node); + + let meta_rec = + self.meta_provider + .get_metadata(node, is_mounted, &self.v2_chunk_ptr, &self.v2_chunks); + + let (old_meta, meta_ptr) = match meta_rec.as_ref() { + Some((m, p)) => (Some(m), p.clone()), + None => (None, None), + }; + + Self::validate_metadata_update(old_meta, &metadata)?; + + if let Some(old_meta) = old_meta { + // if the size was reduced, we need to delete the file chunks above the file size + if metadata.size < old_meta.size { + self.resize_file(node, metadata.size)?; + } + } + self.meta_provider.put_metadata( node, - self.is_mounted(node), - &metadata, + is_mounted, + metadata, + meta_ptr, &mut self.v2_chunk_ptr, &mut self.v2_chunks, &mut self.v2_allocator, ); + + Ok(()) } // Retrieve the DirEntry instance given the Node and DirEntryIndex. @@ -631,7 +672,8 @@ impl Storage for StableStorage { fn read(&mut self, node: Node, offset: FileSize, buf: &mut [u8]) -> Result { let metadata = self.get_metadata(node)?; - let file_size = metadata.size; + let max_size = metadata.maximum_size_allowed.unwrap_or(MAX_FILE_SIZE); + let file_size = metadata.size.min(max_size); if offset >= file_size { return Ok(0); @@ -663,6 +705,12 @@ impl Storage for StableStorage { fn write(&mut self, node: Node, offset: FileSize, buf: &[u8]) -> Result { let mut metadata = self.get_metadata(node)?; + let max_size = metadata.maximum_size_allowed.unwrap_or(MAX_FILE_SIZE); + + if offset + buf.len() as FileSize > max_size { + return Err(Error::MaxFileSizeExceeded); + } + let written_size = if let Some(memory) = self.get_mounted_memory(node) { self.write_mounted(memory, offset, buf); @@ -697,21 +745,27 @@ impl Storage for StableStorage { let end = offset + buf.len() as FileSize; if end > metadata.size { metadata.size = end; - self.put_metadata(node, metadata); + self.put_metadata(node, &metadata)?; } Ok(written_size) } - // - fn rm_file(&mut self, node: Node) -> Result<(), Error> { + fn resize_file(&mut self, node: Node, new_size: FileSize) -> Result<(), Error> { if self.is_mounted(node) { - return Err(Error::CannotRemoveMountedMemoryFile); + // for the mounted node we only update size in the metadata (no need to delete chunks) + return Ok(()); } // delete v1 chunks - let range = (node, 0)..(node + 1, 0); + let chunk_size = FILE_CHUNK_SIZE_V1; + + let first_deletable_index = (new_size.div_ceil(chunk_size as FileSize)) as FileChunkIndex; + + let range = (node, first_deletable_index)..(node + 1, 0); + let mut chunks: Vec<(Node, FileChunkIndex)> = Vec::new(); + for (k, _v) in self.filechunk.range(range) { chunks.push(k); } @@ -721,8 +775,25 @@ impl Storage for StableStorage { self.filechunk.remove(&(node, idx)); } + // fill with zeros the last chunk memory above the file size + if first_deletable_index > 0 { + let offset = new_size as FileSize % chunk_size as FileSize; + + self.write_filechunk_v1( + node, + first_deletable_index - 1, + offset, + &ZEROES[0..(chunk_size - offset as usize)], + ); + } + // delete v2 chunks - let range = (node, 0)..(node + 1, 0); + + let chunk_size = self.chunk_size(); + + let first_deletable_index = (new_size.div_ceil(chunk_size as FileSize)) as FileChunkIndex; + + let range = (node, first_deletable_index)..(node, MAX_FILE_CHUNK_INDEX); let mut chunks: Vec<(Node, FileChunkIndex)> = Vec::new(); for (k, _v) in self.v2_chunk_ptr.range(range) { chunks.push(k); @@ -737,6 +808,23 @@ impl Storage for StableStorage { } } + // fill with zeros the last chunk memory above the file size + if first_deletable_index > 0 { + let offset = new_size as FileSize % chunk_size as FileSize; + self.write_chunks_v2(node, new_size, &ZEROES[0..(chunk_size - offset as usize)])?; + } + + Ok(()) + } + + // + fn rm_file(&mut self, node: Node) -> Result<(), Error> { + if self.is_mounted(node) { + return Err(Error::CannotRemoveMountedMemoryFile); + } + + self.resize_file(node, 0)?; + self.meta_provider.rm_file(node, &mut self.ptr_cache); Ok(()) @@ -761,7 +849,7 @@ impl Storage for StableStorage { file_meta.size = 0; // update mounted metadata - self.put_metadata(node, file_meta); + self.put_metadata(node, &file_meta)?; }; Ok(()) @@ -812,7 +900,7 @@ impl Storage for StableStorage { self.mount_node(node, memory)?; - self.put_metadata(node, meta); + self.put_metadata(node, &meta)?; Ok(()) } @@ -848,7 +936,7 @@ impl Storage for StableStorage { remainder -= to_read; } - self.put_metadata(node, meta); + self.put_metadata(node, &meta)?; self.mount_node(node, memory)?; @@ -872,7 +960,7 @@ impl Storage for StableStorage { } fn flush(&mut self, _node: Node) { - self.flush_mounted_meta(); + // nothing to flush, the system immediately stores data on write } } @@ -889,20 +977,22 @@ mod tests { fn read_and_write_filechunk() { let mut storage = StableStorage::new(DefaultMemoryImpl::default()); let node = storage.new_node(); - storage.put_metadata( - node, - Metadata { + storage + .put_metadata( node, - file_type: FileType::RegularFile, - link_count: 1, - size: 10, - times: Times::default(), - first_dir_entry: Some(42), - last_dir_entry: Some(24), - chunk_type: Some(storage.chunk_type()), - maximum_size_allowed: None, - }, - ); + &Metadata { + node, + file_type: FileType::RegularFile, + link_count: 1, + size: 10, + times: Times::default(), + first_dir_entry: Some(42), + last_dir_entry: Some(24), + chunk_type: Some(storage.chunk_type()), + maximum_size_allowed: None, + }, + ) + .unwrap(); let metadata = storage.get_metadata(node).unwrap(); assert_eq!(metadata.node, node); assert_eq!(metadata.file_type, FileType::RegularFile); diff --git a/src/storage/transient.rs b/src/storage/transient.rs index fe405d8..040164a 100644 --- a/src/storage/transient.rs +++ b/src/storage/transient.rs @@ -10,7 +10,7 @@ use crate::{ storage::{ types::{ DirEntry, DirEntryIndex, FileChunk, FileChunkIndex, FileSize, FileType, Metadata, Node, - Times, + Times, ZEROES, }, Storage, }, @@ -53,6 +53,7 @@ impl TransientStorage { chunk_type: None, maximum_size_allowed: None, }; + let mut result = Self { header: Header { version: 1, @@ -65,7 +66,11 @@ impl TransientStorage { mounted_meta: Default::default(), active_mounts: Default::default(), }; - result.put_metadata(ROOT_NODE, metadata); + + result + .put_metadata(ROOT_NODE, &metadata) + .expect("Failed to create metadata"); + result } @@ -87,6 +92,25 @@ impl TransientStorage { entry.bytes[offset as usize..offset as usize + buf.len()].copy_from_slice(buf) } } + + fn validate_metadata_update( + old_meta: Option<&Metadata>, + new_meta: &Metadata, + ) -> Result<(), Error> { + if let Some(max_size) = new_meta.maximum_size_allowed { + if new_meta.size > max_size { + return Err(Error::MaxFileSizeExceeded); + } + } + + if let Some(old_meta) = old_meta { + if old_meta.node != new_meta.node { + return Err(Error::MetadataUpdateInvalid); + } + } + + Ok(()) + } } impl Storage for TransientStorage { @@ -119,12 +143,22 @@ impl Storage for TransientStorage { } // Update the metadata associated with the node. - fn put_metadata(&mut self, node: Node, metadata: Metadata) { - if self.is_mounted(node) { - self.mounted_meta.insert(node, metadata); + fn put_metadata(&mut self, node: Node, metadata: &Metadata) -> Result<(), Error> { + let meta_storage = if self.is_mounted(node) { + &mut self.mounted_meta + } else { + &mut self.metadata + }; + + if let Some(existing) = meta_storage.get(&node) { + Self::validate_metadata_update(Some(existing), metadata)?; } else { - self.metadata.insert(node, metadata); + Self::validate_metadata_update(None, metadata)?; } + + meta_storage.insert(node, metadata.clone()); + + Ok(()) } // Retrieve the DirEntry instance given the Node and DirEntryIndex. @@ -226,11 +260,10 @@ impl Storage for TransientStorage { Ok(size_read) } - // - fn rm_file(&mut self, node: Node) -> Result<(), Error> { - if self.is_mounted(node) { - return Err(Error::CannotRemoveMountedMemoryFile); - } + fn resize_file(&mut self, node: Node, new_size: FileSize) -> Result<(), Error> { + let chunk_size = FILE_CHUNK_SIZE_V1; + + let first_deletable_index = (new_size.div_ceil(chunk_size as FileSize)) as FileChunkIndex; let range = (node, 0)..(node + 1, 0); @@ -245,6 +278,28 @@ impl Storage for TransientStorage { self.filechunk.remove(&(node, idx)); } + // fill with zeros the last chunk memory above the file size + if first_deletable_index > 0 { + let offset = new_size as FileSize % chunk_size as FileSize; + + self.write_filechunk( + node, + first_deletable_index - 1, + offset, + &ZEROES[0..(chunk_size - offset as usize)], + ); + } + + Ok(()) + } + + fn rm_file(&mut self, node: Node) -> Result<(), Error> { + if self.is_mounted(node) { + return Err(Error::CannotRemoveMountedMemoryFile); + } + + self.resize_file(node, 0)?; + // remove metadata self.mounted_meta.remove(&node); self.metadata.remove(&node); @@ -321,7 +376,7 @@ impl Storage for TransientStorage { self.mount_node(node, memory)?; - self.put_metadata(node, meta); + self.put_metadata(node, &meta)?; Ok(()) } @@ -354,7 +409,7 @@ impl Storage for TransientStorage { remainder -= to_read; } - self.put_metadata(node, meta); + self.put_metadata(node, &meta)?; self.mount_node(node, memory)?; @@ -364,6 +419,13 @@ impl Storage for TransientStorage { fn write(&mut self, node: Node, offset: FileSize, buf: &[u8]) -> Result { let mut metadata = self.get_metadata(node)?; let end = offset + buf.len() as FileSize; + + if let Some(max_size) = metadata.maximum_size_allowed { + if end > max_size { + return Err(Error::MaxFileSizeExceeded); + } + } + let chunk_infos = get_chunk_infos(offset, end, FILE_CHUNK_SIZE_V1); let mut written_size = 0; for chunk in chunk_infos.into_iter() { @@ -378,7 +440,7 @@ impl Storage for TransientStorage { if end > metadata.size { metadata.size = end; - self.put_metadata(node, metadata) + self.put_metadata(node, &metadata)?; } Ok(written_size as FileSize) @@ -414,20 +476,22 @@ mod tests { fn read_and_write_filechunk() { let mut storage = TransientStorage::default(); let node = storage.new_node(); - storage.put_metadata( - node, - Metadata { + storage + .put_metadata( node, - file_type: FileType::RegularFile, - link_count: 1, - size: 10, - times: Times::default(), - first_dir_entry: None, - last_dir_entry: None, - chunk_type: Some(storage.chunk_type()), - maximum_size_allowed: None, - }, - ); + &Metadata { + node, + file_type: FileType::RegularFile, + link_count: 1, + size: 10, + times: Times::default(), + first_dir_entry: None, + last_dir_entry: None, + chunk_type: Some(storage.chunk_type()), + maximum_size_allowed: None, + }, + ) + .unwrap(); storage.write(node, 0, &[42; 10]).unwrap(); let mut buf = [0; 10]; storage.read(node, 0, &mut buf).unwrap(); diff --git a/src/storage/types.rs b/src/storage/types.rs index ae357e9..e0c32f0 100644 --- a/src/storage/types.rs +++ b/src/storage/types.rs @@ -28,7 +28,10 @@ pub type FileChunkIndex = u32; // The address in memory where the V2 chunk is stored. pub type FileChunkPtr = u64; -// A handle used for writing files in chunks +// An array filled with 0 used to fill memory with 0 via copy. +pub const ZEROES: [u8; MAX_FILE_CHUNK_SIZE_V2] = [0u8; MAX_FILE_CHUNK_SIZE_V2]; + +// A handle used for writing files in chunks. #[derive(Debug, PartialEq, Eq)] pub(crate) struct ChunkHandle { pub index: FileChunkIndex,