sparse file support

src/fs.rs

@@ -547,6 +547,7 @@ impl FileSystem {
 #[cfg(test)]
 mod tests {
 
+    use ic_stable_structures::memory_manager::{MemoryId, MemoryManager};
     use ic_stable_structures::{Memory, VectorMemory};
 
     use crate::{
@@ -556,7 +557,10 @@ mod tests {
             structure_helpers::find_node,
             types::{FdStat, OpenFlags},
         },
-        storage::types::{FileSize, FileType},
+        storage::{
+            stable::StableStorage,
+            types::{FileSize, FileType},
+        },
         test_utils::{
             new_vector_memory, read_text_file, test_fs, test_fs_setups, test_fs_transient,
             write_text_fd, write_text_file,
@@ -1353,6 +1357,32 @@ mod tests {
         println!("{:?}", buf);
     }
 
+    #[test]
+    fn reading_mounted_memory_after_upgrade() {
+        let memory_manager = MemoryManager::init(new_vector_memory());
+        let memory = memory_manager.get(MemoryId::new(1));
+
+        let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
+        let mut fs = FileSystem::new(Box::new(storage)).unwrap();
+        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
+            .unwrap();
+
+        let root_fd = fs.root_fd();
+        let content = "ABCDEFG123";
+        write_text_file(&mut fs, root_fd, "test.txt", content, 2).unwrap();
+
+        // imitate canister upgrade (we keep the memory manager but recreate the file system with the same virtual memories)
+        let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
+        let mut fs = FileSystem::new(Box::new(storage)).unwrap();
+        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
+            .unwrap();
+        let root_fd = fs.root_fd();
+
+        let content = read_text_file(&mut fs, root_fd, "test.txt", 0, 100);
+
+        assert_eq!(content, "ABCDEFG123ABCDEFG123");
+    }
+
     #[test]
     fn deleting_mounted_file_fails() {
         let memory: VectorMemory = new_vector_memory();
@@ -1367,7 +1397,7 @@ mod tests {
         let res = fs.remove_file(root_fd, "test.txt");
 
         assert!(
-            res.is_err(),
+            res == Err(Error::CannotRemoveMountedMemoryFile),
             "Deleting a mounted file should not be allowed!"
         );
 
@@ -1478,4 +1508,6 @@ mod tests {
             assert!(res.is_err());
         }
     }
+
+    // test sparse files
 }

src/runtime/structure_helpers.rs

@@ -118,6 +118,13 @@ pub fn create_dir_entry(
     }
 
     let node = storage.new_node();
+
+    let chunk_type = if entry_type == FileType::RegularFile {
+        Some(storage.chunk_type())
+    } else {
+        None
+    };
+
     storage.put_metadata(
         node,
         Metadata {
@@ -132,6 +139,7 @@ pub fn create_dir_entry(
             },
             first_dir_entry: None,
             last_dir_entry: None,
+            chunk_type,
         },
     );
 

src/runtime/types.rs

@@ -1,4 +1,5 @@
 use bitflags::bitflags;
+use serde::{Deserialize, Serialize};
 
 #[derive(Copy, Clone, Debug)]
 pub struct FdStat {
@@ -43,10 +44,10 @@ impl ChunkSize {
     ];
 }
 
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum ChunkType {
-    V1,
-    V2,
+    V1 = 1,
+    V2 = 2,
 }
 
 bitflags! {

src/storage.rs

@@ -9,6 +9,7 @@ use crate::{
 
 mod allocator;
 pub mod dummy;
+mod iterator;
 mod journal;
 pub mod stable;
 pub mod transient;

src/storage/dummy.rs

@@ -137,6 +137,7 @@ mod tests {
                 times: Times::default(),
                 first_dir_entry: Some(42),
                 last_dir_entry: Some(24),
+                chunk_type: None,
             },
         )
     }

src/storage/iterator.rs

@@ -0,0 +1,239 @@
+use crate::storage::types::FileChunkIndex;
+use crate::storage::types::FileChunkPtr;
+use crate::storage::FileSize;
+use crate::storage::Node;
+use ic_stable_structures;
+use ic_stable_structures::memory_manager::VirtualMemory;
+use ic_stable_structures::BTreeMap;
+use ic_stable_structures::Memory;
+use std::collections::HashMap;
+
+pub(crate) struct ChunkV2Iterator<'a, M: Memory> {
+    node: Node,
+    last_index_excluded: FileChunkIndex,
+    cur_index: FileChunkIndex,
+
+    is_prefetched: bool,
+    prefetched_pointers: HashMap<(Node, FileChunkIndex), FileChunkPtr>,
+
+    last_index: (Node, FileChunkIndex, FileChunkPtr),
+    v2_chunk_ptr: &'a mut BTreeMap<(Node, FileChunkIndex), FileChunkPtr, VirtualMemory<M>>,
+}
+
+impl<'a, M: Memory> ChunkV2Iterator<'a, M> {
+    pub fn new(
+        node: Node,
+        offset: FileSize,
+        file_size: FileSize,
+        chunk_size: FileSize,
+        last_index: (Node, FileChunkIndex, FileChunkPtr),
+        v2_chunk_ptr: &'a mut BTreeMap<(Node, FileChunkIndex), FileChunkPtr, VirtualMemory<M>>,
+    ) -> Self {
+        let cur_index = (offset / chunk_size) as FileChunkIndex;
+        let last_index_excluded = (file_size / chunk_size + 1) as FileChunkIndex;
+
+        Self {
+            node,
+            last_index_excluded,
+            cur_index,
+            is_prefetched: false,
+            prefetched_pointers: HashMap::new(),
+            last_index,
+            v2_chunk_ptr,
+        }
+    }
+}
+
+impl<'a, M: Memory> Iterator for ChunkV2Iterator<'a, M> {
+    type Item = ((Node, FileChunkIndex), Option<FileChunkPtr>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // we are at the end of the list, return None
+        if self.cur_index >= self.last_index_excluded {
+            return None;
+        }
+
+        // try get cached item first
+        let last = self.last_index;
+        if last.0 == self.node && last.1 == self.cur_index {
+            let res = Some(((self.node, self.cur_index), Some(last.2)));
+            self.cur_index += 1;
+            // return cached value
+            return res;
+        }
+
+        // cache failed, resort to reading the ranged values from the iterator
+        if !self.is_prefetched {
+            let range = (self.node, self.cur_index)..(self.node, self.last_index_excluded);
+            let items = self.v2_chunk_ptr.range(range);
+
+            for (k, v) in items {
+                self.prefetched_pointers.insert(k, v);
+            }
+
+            self.is_prefetched = true;
+        }
+
+        let found: Option<FileChunkPtr> = self
+            .prefetched_pointers
+            .get(&(self.node, self.cur_index))
+            .copied();
+
+        let res = Some(((self.node, self.cur_index), found));
+
+        self.cur_index += 1;
+
+        res
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::fs::FileSize;
+    use crate::storage::iterator::ChunkV2Iterator;
+    use crate::storage::stable::StableStorage;
+    use crate::storage::types::{FileType, Metadata, Node, Times};
+    use crate::storage::Storage;
+    use crate::test_utils::new_vector_memory;
+    use ic_stable_structures::Memory;
+
+    fn create_file_with_size<M: Memory>(size: FileSize, storage: &mut StableStorage<M>) -> Node {
+        let node = storage.new_node();
+
+        storage.put_metadata(
+            node,
+            Metadata {
+                node,
+                file_type: FileType::RegularFile,
+                link_count: 1,
+                size,
+                times: Times::default(),
+                first_dir_entry: Some(42),
+                last_dir_entry: Some(24),
+                chunk_type: Some(storage.chunk_type()),
+            },
+        );
+        node
+    }
+
+    #[test]
+    fn iterate_short_file() {
+        let mut storage = StableStorage::new(new_vector_memory());
+        let node = create_file_with_size(0, &mut storage);
+        let write_size = storage.chunk_size() * 3 - 100;
+
+        let buf = vec![142u8; write_size];
+
+        storage.write(node, 0, &*buf).unwrap();
+
+        let meta = storage.get_metadata(node).unwrap();
+        let file_size = meta.size;
+
+        let iterator = ChunkV2Iterator::new(
+            node,
+            30,
+            file_size,
+            storage.chunk_size() as FileSize,
+            storage.last_index,
+            &mut storage.v2_chunk_ptr,
+        );
+
+        let res_vec: Vec<_> = iterator.collect();
+
+        assert!(res_vec[0].1.is_some());
+        assert!(res_vec[1].1.is_some());
+        assert!(res_vec[2].1.is_some());
+
+        println!("{:?}", res_vec);
+    }
+
+    #[test]
+    fn iterate_file_with_size_and_no_stored_chunks() {
+        let mut storage = StableStorage::new(new_vector_memory());
+        let write_size = (storage.chunk_size() * 3 - 100) as FileSize;
+
+        let node = create_file_with_size(write_size, &mut storage);
+
+        let meta = storage.get_metadata(node).unwrap();
+        let file_size = meta.size;
+
+        let iterator = ChunkV2Iterator::new(
+            node,
+            30,
+            file_size,
+            storage.chunk_size() as FileSize,
+            storage.last_index,
+            &mut storage.v2_chunk_ptr,
+        );
+
+        let res_vec: Vec<_> = iterator.collect();
+
+        assert!(res_vec[0].1.is_none());
+        assert!(res_vec[1].1.is_none());
+        assert!(res_vec[2].1.is_none());
+
+        println!("{:?}", res_vec);
+    }
+
+    #[test]
+    fn iterate_file_missing_chunk_in_the_middle() {
+        let mut storage = StableStorage::new(new_vector_memory());
+        let node = create_file_with_size(0, &mut storage);
+
+        let write_size = (storage.chunk_size() * 3 - 200) as FileSize;
+
+        storage.write(node, 10, &[142u8; 100]).unwrap();
+        storage.write(node, write_size, &[142u8; 100]).unwrap();
+
+        let meta = storage.get_metadata(node).unwrap();
+        let file_size = meta.size;
+
+        let iterator = ChunkV2Iterator::new(
+            node,
+            30,
+            file_size,
+            storage.chunk_size() as FileSize,
+            storage.last_index,
+            &mut storage.v2_chunk_ptr,
+        );
+
+        let res_vec: Vec<_> = iterator.collect();
+
+        println!("{:?}", res_vec);
+
+        assert!(res_vec[0].1.is_some());
+        assert!(res_vec[1].1.is_none());
+        assert!(res_vec[2].1.is_some());
+    }
+
+    #[test]
+    fn iterate_file_only_middle_chunk_is_present() {
+        let mut storage = StableStorage::new(new_vector_memory());
+        let file_size = (storage.chunk_size() * 3 - 200) as FileSize;
+        let node = create_file_with_size(file_size, &mut storage);
+
+        let write_size = (storage.chunk_size() * 2 - 200) as FileSize;
+
+        storage.write(node, write_size, &[142u8; 102]).unwrap();
+
+        let meta = storage.get_metadata(node).unwrap();
+        let file_size = meta.size;
+
+        let iterator = ChunkV2Iterator::new(
+            node,
+            30,
+            file_size,
+            storage.chunk_size() as FileSize,
+            storage.last_index,
+            &mut storage.v2_chunk_ptr,
+        );
+
+        let res_vec: Vec<_> = iterator.collect();
+
+        println!("{:?}", res_vec);
+
+        assert!(res_vec[0].1.is_none());
+        assert!(res_vec[1].1.is_some());
+        assert!(res_vec[2].1.is_none());
+    }
+}

src/storage/journal.rs

@@ -132,6 +132,7 @@ mod tests {
             },
             first_dir_entry: None,
             last_dir_entry: Some(876),
+            chunk_type: None,
         };
 
         let mut node2 = 0;
@@ -224,6 +225,7 @@ mod tests {
             },
             first_dir_entry: None,
             last_dir_entry: Some(876),
+            chunk_type: None,
         };
 
         journal.write_mounted_meta(&123, &meta);