WIP: Support non-standard images

- Support non standard images. Read and Write!
- Add Kind for including endian and version information
- Add Kind: LE_V4_0 for linux kernel and upstream squashfs-tools support
- Add Kind: BE_V4_0 for custom vendor firmware
- Add Kind: AVM_BE_V4_0 for Fritz!OS firmware support. Added because
  it's interesting, as they kept some of it still BE.

- Change lookup table from u32 to u64. This was working in LE, but is
  very wrong for LE!

See #72

src/dir.rs

@@ -11,9 +11,11 @@ use std::path::PathBuf;
 use deku::prelude::*;
 
 use crate::inode::InodeId;
+use crate::kind::Kind;
 
 #[derive(Debug, DekuRead, DekuWrite, Clone, Default, PartialEq, Eq)]
-#[deku(endian = "little")]
+#[deku(ctx = "kind: Kind")]
+#[deku(endian = "kind.type_endian")]
 pub struct Dir {
     /// Number of entries following the header.
     pub(crate) count: u32,

src/entry.rs

@@ -11,7 +11,7 @@ use crate::inode::{
     InodeInner,
 };
 use crate::metadata::MetadataWriter;
-use crate::squashfs::SuperBlock;
+use crate::squashfs::{Kind, SuperBlock};
 use crate::{
     NodeHeader, SquashfsBlockDevice, SquashfsCharacterDevice, SquashfsDir, SquashfsSymlink,
 };
@@ -43,6 +43,7 @@ impl<'a> Entry<'a> {
         block_offset: u16,
         block_index: u32,
         superblock: &SuperBlock,
+        kind: Kind,
     ) -> Self {
         let dir_inode = Inode {
             id: InodeId::BasicDirectory,
@@ -59,10 +60,11 @@ impl<'a> Entry<'a> {
             }),
         };
 
-        dir_inode.to_bytes(name.as_bytes(), inode_writer, superblock)
+        dir_inode.to_bytes(name.as_bytes(), inode_writer, superblock, kind)
     }
 
     /// Write data and metadata for file node
+    #[allow(clippy::too_many_arguments)]
     pub fn file(
         node_path: &'a OsStr,
         header: &NodeHeader,
@@ -71,6 +73,7 @@ impl<'a> Entry<'a> {
         file_size: usize,
         added: &Added,
         superblock: &SuperBlock,
+        kind: Kind,
     ) -> Self {
         let basic_file = match added {
             Added::Data {
@@ -106,7 +109,7 @@ impl<'a> Entry<'a> {
             inner: InodeInner::BasicFile(basic_file),
         };
 
-        file_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock)
+        file_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock, kind)
     }
 
     /// Write data and metadata for symlink node
@@ -116,6 +119,7 @@ impl<'a> Entry<'a> {
         inode: u32,
         inode_writer: &mut MetadataWriter,
         superblock: &SuperBlock,
+        kind: Kind,
     ) -> Self {
         let link = symlink.link.as_os_str().as_bytes();
         let sym_inode = Inode {
@@ -131,7 +135,7 @@ impl<'a> Entry<'a> {
             }),
         };
 
-        sym_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock)
+        sym_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock, kind)
     }
 
     /// Write data and metadata for char device node
@@ -141,6 +145,7 @@ impl<'a> Entry<'a> {
         inode: u32,
         inode_writer: &mut MetadataWriter,
         superblock: &SuperBlock,
+        kind: Kind,
     ) -> Self {
         let char_inode = Inode {
             id: InodeId::BasicCharacterDevice,
@@ -154,7 +159,7 @@ impl<'a> Entry<'a> {
             }),
         };
 
-        char_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock)
+        char_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock, kind)
     }
 
     /// Write data and metadata for block device node
@@ -164,6 +169,7 @@ impl<'a> Entry<'a> {
         inode: u32,
         inode_writer: &mut MetadataWriter,
         superblock: &SuperBlock,
+        kind: Kind,
     ) -> Self {
         let block_inode = Inode {
             id: InodeId::BasicBlockDevice,
@@ -177,7 +183,7 @@ impl<'a> Entry<'a> {
             }),
         };
 
-        block_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock)
+        block_inode.to_bytes(node_path.as_bytes(), inode_writer, superblock, kind)
     }
 }
 
@@ -266,31 +272,7 @@ impl<'a> Entry<'a> {
 
 #[cfg(test)]
 mod tests {
-    use std::io::Write;
-
     use super::*;
-    use crate::compressor::Compressor;
-    use crate::metadata::{MetadataWriter, METADATA_MAXSIZE};
-
-    #[test]
-    #[cfg(feature = "xz")]
-    fn test_mwriter() {
-        let bytes = [0xffu8; METADATA_MAXSIZE - 3];
-
-        let mut mwriter = MetadataWriter::new(Compressor::Xz, None, 0x2000);
-
-        mwriter.write_all(&bytes).unwrap();
-        assert_eq!(0, mwriter.metadata_start);
-        assert_eq!(bytes, &*mwriter.uncompressed_bytes);
-        assert!(mwriter.compressed_bytes.is_empty());
-
-        let bytes = [0x11u8; 6];
-
-        mwriter.write_all(&bytes).unwrap();
-        assert_eq!(0x6e, mwriter.metadata_start);
-        assert_eq!(bytes[3..], mwriter.uncompressed_bytes);
-        assert_eq!(mwriter.compressed_bytes[0].len(), 0x6c);
-    }
 
     #[test]
     fn test_entry() {

src/filesystem/reader.rs

@@ -7,12 +7,13 @@ use crate::error::SquashfsError;
 use crate::fragment::Fragment;
 use crate::inode::BasicFile;
 use crate::reader::{ReadSeek, SquashfsReaderWithOffset};
-use crate::squashfs::{Cache, Id};
+use crate::squashfs::{Cache, Id, Kind};
 use crate::{Node, Squashfs, SquashfsDir, SquashfsFileReader};
 
 /// Representation of SquashFS filesystem after read from image
 #[derive(Debug)]
 pub struct FilesystemReader<R: ReadSeek> {
+    pub kind: Kind,
     /// See [`SuperBlock`].`block_size`
     pub block_size: u32,
     /// See [`SuperBlock`].`block_log`
@@ -39,6 +40,8 @@ pub struct FilesystemReader<R: ReadSeek> {
 
 impl<R: ReadSeek> FilesystemReader<R> {
     /// Call [`Squashfs::from_reader`], then [`Squashfs::into_filesystem_reader`]
+    ///
+    /// With default kind: [`crate::kind::LE_V4_0`] and offset `0`.
     pub fn from_reader(reader: R) -> Result<Self, SquashfsError> {
         let squashfs = Squashfs::from_reader(reader)?;
         squashfs.into_filesystem_reader()
@@ -51,6 +54,16 @@ impl<R: ReadSeek> FilesystemReader<SquashfsReaderWithOffset<R>> {
         let squashfs = Squashfs::from_reader_with_offset(reader, offset)?;
         squashfs.into_filesystem_reader()
     }
+
+    /// Same as [`Self::from_reader_with_offset`], but setting custom `kind`
+    pub fn from_reader_with_offset_and_kind(
+        reader: R,
+        offset: u64,
+        kind: Kind,
+    ) -> Result<Self, SquashfsError> {
+        let squashfs = Squashfs::from_reader_with_offset_and_kind(reader, offset, kind)?;
+        squashfs.into_filesystem_reader()
+    }
 }
 
 impl<R: ReadSeek> FilesystemReader<R> {

src/filesystem/writer.rs

@@ -2,7 +2,8 @@ use std::cell::RefCell;
 use std::io::{Read, Seek, SeekFrom, Write};
 use std::path::PathBuf;
 
-use deku::DekuContainerWrite;
+use deku::bitvec::BitVec;
+use deku::DekuWrite;
 use tracing::{info, instrument, trace};
 
 use crate::compressor::{CompressionOptions, Compressor};
@@ -11,6 +12,7 @@ use crate::error::SquashfsError;
 use crate::filesystem::dummy::DummyReadSeek;
 use crate::filesystem::node::SquashfsSymlink;
 use crate::fragment::Fragment;
+use crate::kind::Kind;
 use crate::metadata::{self, MetadataWriter};
 use crate::reader::{ReadSeek, WriteSeek};
 use crate::squashfs::{Id, SuperBlock};
@@ -23,6 +25,7 @@ use crate::{
 /// Representation of SquashFS filesystem to be written back to an image
 #[derive(Debug)]
 pub struct FilesystemWriter<'a, R: ReadSeek = DummyReadSeek> {
+    pub kind: Kind,
     /// See [`SuperBlock`].`block_size`
     pub block_size: u32,
     /// See [`SuperBlock`].`block_log`
@@ -68,6 +71,7 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
             })
             .collect::<Result<_, SquashfsError>>()?;
         Ok(Self {
+            kind: reader.kind,
             block_size: reader.block_size,
             block_log: reader.block_log,
             compressor: reader.compressor,
@@ -230,7 +234,7 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
     /// correct fields from `Filesystem`, and the data after that contains the nodes.
     #[instrument(skip_all)]
     pub fn write<W: Write + Seek>(&self, w: &mut W) -> Result<(), SquashfsError> {
-        let mut superblock = SuperBlock::new(self.compressor);
+        let mut superblock = SuperBlock::new(self.compressor, self.kind);
 
         trace!("{:#02x?}", self.nodes);
         info!("Creating Tree");
@@ -240,8 +244,9 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
         // Empty Squashfs Superblock
         w.write_all(&[0x00; 96])?;
         let mut data_writer = DataWriter::new(self.compressor, None, self.block_size);
-        let mut inode_writer = MetadataWriter::new(self.compressor, None, self.block_size);
-        let mut dir_writer = MetadataWriter::new(self.compressor, None, self.block_size);
+        let mut inode_writer =
+            MetadataWriter::new(self.compressor, None, self.block_size, self.kind);
+        let mut dir_writer = MetadataWriter::new(self.compressor, None, self.block_size, self.kind);
 
         info!("Creating Inodes and Dirs");
         //trace!("TREE: {:#02x?}", tree);
@@ -253,7 +258,7 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
 
         info!("Writing Other stuff");
         let (_, root_inode) =
-            tree.write_inode_dir(&mut inode_writer, &mut dir_writer, 0, superblock)?;
+            tree.write_inode_dir(&mut inode_writer, &mut dir_writer, 0, superblock, self.kind)?;
 
         superblock.root_inode = root_inode;
         superblock.inode_count = self.nodes.len() as u32 + 1; // + 1 for the "/"
@@ -270,20 +275,21 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
         dir_writer.finalize(w)?;
 
         info!("Writing Frag Lookup Table");
-        Self::write_frag_table(w, data_writer.fragment_table, &mut superblock)?;
+        self.write_frag_table(w, data_writer.fragment_table, &mut superblock)?;
 
         info!("Writing Id Lookup Table");
-        Self::write_id_table(w, &self.id_table, &mut superblock)?;
+        self.write_id_table(w, &self.id_table, &mut superblock)?;
 
         info!("Finalize Superblock and End Bytes");
-        Self::finalize(w, &mut superblock)?;
+        self.finalize(w, &mut superblock)?;
 
         info!("Superblock: {:#02x?}", superblock);
         info!("Success");
         Ok(())
     }
 
     fn finalize<W: Write + Seek>(
+        &self,
         w: &mut W,
         superblock: &mut SuperBlock,
     ) -> Result<(), SquashfsError> {
@@ -299,14 +305,17 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
         info!("Writing Superblock");
         trace!("{:#02x?}", superblock);
         w.rewind()?;
-        w.write_all(&superblock.to_bytes()?)?;
+        let mut bv = BitVec::new();
+        superblock.write(&mut bv, self.kind)?;
+        w.write_all(bv.as_raw_slice())?;
 
         info!("Writing Finished");
 
         Ok(())
     }
 
     fn write_id_table<W: Write + Seek>(
+        &self,
         w: &mut W,
         id_table: &Option<Vec<Id>>,
         write_superblock: &mut SuperBlock,
@@ -315,37 +324,53 @@ impl<'a, R: ReadSeek> FilesystemWriter<'a, R> {
             let id_table_dat = w.stream_position()?;
             let mut id_bytes = Vec::with_capacity(id.len() * ((u32::BITS / 8) as usize));
             for i in id {
-                let bytes = i.to_bytes()?;
-                id_bytes.write_all(&bytes)?;
+                let mut bv = BitVec::new();
+                i.write(&mut bv, self.kind)?;
+                id_bytes.write_all(bv.as_raw_slice())?;
             }
-            let metadata_len = metadata::set_if_uncompressed(id_bytes.len() as u16).to_le_bytes();
-            w.write_all(&metadata_len)?;
+            // write metdata_length
+            let mut bv = BitVec::new();
+            metadata::set_if_uncompressed(id_bytes.len() as u16)
+                .write(&mut bv, self.kind.data_endian)?;
+            w.write_all(bv.as_raw_slice())?;
             w.write_all(&id_bytes)?;
             write_superblock.id_table = w.stream_position()?;
             write_superblock.id_count = id.len() as u16;
-            w.write_all(&id_table_dat.to_le_bytes())?;
+
+            let mut bv = BitVec::new();
+            id_table_dat.write(&mut bv, self.kind.type_endian)?;
+            w.write_all(bv.as_raw_slice())?;
         }
 
         Ok(())
     }
 
     fn write_frag_table<W: Write + Seek>(
+        &self,
         w: &mut W,
         frag_table: Vec<Fragment>,
         write_superblock: &mut SuperBlock,
     ) -> Result<(), SquashfsError> {
         let frag_table_dat = w.stream_position()?;
         let mut frag_bytes = Vec::with_capacity(frag_table.len() * fragment::SIZE);
         for f in &frag_table {
-            let bytes = f.to_bytes()?;
-            frag_bytes.write_all(&bytes)?;
+            let mut bv = BitVec::new();
+            f.write(&mut bv, self.kind)?;
+            frag_bytes.write_all(bv.as_raw_slice())?;
         }
-        let metadata_len = metadata::set_if_uncompressed(frag_bytes.len() as u16).to_le_bytes();
-        w.write_all(&metadata_len)?;
+        // write metdata_length
+        let mut bv = BitVec::new();
+        metadata::set_if_uncompressed(frag_bytes.len() as u16)
+            .write(&mut bv, self.kind.data_endian)?;
+        w.write_all(bv.as_raw_slice())?;
+
         w.write_all(&frag_bytes)?;
         write_superblock.frag_table = w.stream_position()?;
         write_superblock.frag_count = frag_table.len() as u32;
-        w.write_all(&frag_table_dat.to_le_bytes())?;
+
+        let mut bv = BitVec::new();
+        frag_table_dat.write(&mut bv, self.kind.type_endian)?;
+        w.write_all(bv.as_raw_slice())?;
 
         Ok(())
     }

src/fragment.rs

@@ -3,12 +3,13 @@
 use deku::prelude::*;
 
 use crate::data::DataSize;
+use crate::kind::Kind;
 
 pub(crate) const SIZE: usize =
     std::mem::size_of::<u64>() + std::mem::size_of::<u32>() + std::mem::size_of::<u32>();
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, DekuRead, DekuWrite)]
-#[deku(endian = "little")]
+#[deku(endian = "kind.type_endian", ctx = "kind: Kind")]
 pub struct Fragment {
     pub start: u64,
     pub size: DataSize,