Merging of any number of logs

Cargo.toml

@@ -4,18 +4,21 @@ description = "IRC log converter/collector/cruncher"
 license = "Apache-2.0"
 name = "ilc"
 repository = "https://github.com/tilpner/ilc"
-version = "0.2.2"
+version = "0.3.0"
 
 [[bin]]
 name = "ilc"
 doc = false
 
 [dependencies]
-ilc-cli = "0.1.0"
-ilc-base = "0.1.0"
-ilc-ops = "0.1.0"
-ilc-format-weechat = "0.1.0"
-ilc-format-energymech = "0.1.0"
+ilc-cli = "~0.1"
+ilc-base = "~0.2"
+ilc-ops = "~0.1"
+ilc-format-weechat = "~0.2"
+ilc-format-energymech = "~0.2"
+
+[dev-dependencies]
+flate2 = "~0.2"
 
 [profile.release]
 debug = false

base/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ilc-base"
-version = "0.1.2"
+version = "0.2.0"
 description = "IRC log converter/collector/cruncher"
 homepage = "https://github.com/tilpner/ilc"
 license = "Apache-2.0"

base/src/context.rs

@@ -7,3 +7,13 @@ pub struct Context {
     pub override_date: Option<NaiveDate>,
     pub channel: Option<String>,
 }
+
+impl Default for Context {
+    fn default() -> Context {
+        Context {
+            timezone: FixedOffset::west(0),
+            override_date: None,
+            channel: None,
+        }
+    }
+}

base/src/dummy.rs

@@ -18,10 +18,11 @@ use std::io::BufRead;
 use event::Event;
 use context::Context;
 
+#[derive(Copy, Clone)]
 pub struct Dummy;
 
 impl ::Decode for Dummy {
-    fn decode<'a>(&'a mut self,
+    fn decode<'a>(&'a self,
                   _context: &'a Context,
                   _input: &'a mut BufRead)
                   -> Box<Iterator<Item = ::Result<Event<'a>>> + 'a> {

base/src/lib.rs

@@ -26,7 +26,7 @@ pub mod dummy;
 use std::io::{BufRead, Write};
 
 pub use context::Context;
-pub use event::Event;
+pub use event::{Event, Time};
 pub use error::*;
 
 pub trait Encode {
@@ -37,8 +37,8 @@ pub trait Encode {
                   -> error::Result<()>;
 }
 
-pub trait Decode  {
-    fn decode<'a>(&'a mut self,
+pub trait Decode {
+    fn decode<'a>(&'a self,
                   context: &'a Context,
                   input: &'a mut BufRead)
                   -> Box<Iterator<Item = error::Result<Event<'a>>> + 'a>;

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ilc-cli"
-version = "0.1.1"
+version = "0.1.2"
 description = "IRC log converter/collector/cruncher"
 homepage = "https://github.com/tilpner/ilc"
 license = "Apache-2.0"
@@ -16,7 +16,7 @@ clap = "2.1.2"
 chrono = "0.2.19"
 env_logger = "0.3.2"
 glob = "0.2.10"
-ilc-base = "0.1.0"
-ilc-ops = "0.1.0"
-ilc-format-weechat = { optional = true, version = "0.1.0" }
-ilc-format-energymech = { optional = true, version = "0.1.0" }
+ilc-base = "~0.2"
+ilc-ops = "~0.1"
+ilc-format-weechat = { optional = true, version = "~0.2" }
+ilc-format-energymech = { optional = true, version = "~0.2" }

cli/src/lib.rs

@@ -47,6 +47,10 @@ mod chain;
 
 pub fn main() {
     env_logger::init().unwrap();
+    if option_env!("FUSE").is_some() {
+        info!("Compiled with FUSEs")
+    }
+
     let args = App::new("ilc")
                    .version(crate_version!())
                    .setting(AppSettings::GlobalVersion)
@@ -181,21 +185,22 @@ pub fn main() {
                          &*e.encoder())
         }
         ("merge", Some(args)) => {
-            // TODO: avoid (de-)serialization to weechat
             let e = Environment(&args);
-            let (ctx, i, d, o, e) = (&e.context(),
-                                     &mut e.input(),
-                                     &mut *e.decoder(),
-                                     &mut *e.output(),
-                                     &*e.encoder());
-            let mut buffer = Vec::new();
-            match sort::sort(ctx, i, d, &mut buffer, &Weechat) {
-                Err(e) => error(Box::new(e)),
-                _ => (),
-            }
-            let mut read = io::Cursor::new(&buffer);
-            dedup::dedup(ctx, &mut read, &mut Weechat, o, e)
 
+            let mut inputs = e.inputs();
+            // let mut decoders = e.decoders();
+
+            let borrowed_inputs = inputs.iter_mut()
+                                        .map(|a| a as &mut BufRead)
+                                        .collect();
+            // let borrowed_decoders = decoders.iter_mut()
+            //                                .map(|a| &mut **a as &mut Decode)
+            //                                .collect();
+            merge::merge(&e.context(),
+                         borrowed_inputs,
+                         &mut *e.decoder(),
+                         &mut *e.output(),
+                         &*e.encoder())
         }
         (sc, _) if !sc.is_empty() => panic!("Unimplemented subcommand `{}`, this is a bug", sc),
         _ => die("No command specified"),
@@ -272,15 +277,44 @@ impl<'a> Environment<'a> {
     pub fn context(&self) -> Context {
         build_context(self.0)
     }
+
     pub fn input(&self) -> Box<BufRead> {
         open_files(gather_input(self.0))
     }
+
+    pub fn inputs(&self) -> Vec<Box<BufRead>> {
+        gather_input(self.0)
+            .iter()
+            .map(|path| {
+                Box::new(BufReader::new(File::open(path)
+                                            .unwrap_or_else(|e| {
+                                                error(Box::new(e))
+                                            }))) as Box<BufRead>
+            })
+            .collect()
+    }
+
     pub fn output(&self) -> Box<Write> {
         open_output(self.0)
     }
+
     pub fn decoder(&self) -> Box<Decode> {
         force_decoder(self.0.value_of("format").or(self.0.value_of("input_format")))
     }
+
+    /* pub fn decoders(&self) -> Vec<Box<Decode>> {
+     * self.0
+     * .value_of("format")
+     * .into_iter()
+     * .chain(self.0
+     * .values_of("input_formats")
+     * .map(|i| Box::new(i) as Box<Iterator<Item = _>>)
+     * .unwrap_or(Box::new(iter::empty()) as Box<Iterator<Item = _>>))
+     * .map(Option::Some)
+     * .map(force_decoder)
+     * .collect()
+     * } */
+
     pub fn encoder(&self) -> Box<Encode> {
         force_encoder(self.0.value_of("format").or(self.0.value_of("output_format")))
     }

formats/energymech/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ilc-format-energymech"
-version = "0.1.1"
+version = "0.2.0"
 description = "IRC log converter/collector/cruncher"
 homepage = "https://github.com/tilpner/ilc"
 license = "Apache-2.0"
@@ -10,4 +10,4 @@ authors = ["Till Höppner <[email protected]>"]
 [dependencies]
 log = "0.3.5"
 chrono = "0.2.19"
-ilc-base = "0.1.0"
+ilc-base = "~0.2"

formats/energymech/src/lib.rs

@@ -29,6 +29,7 @@ use log::LogLevel::Info;
 
 use chrono::*;
 
+#[derive(Copy, Clone)]
 pub struct Energymech;
 
 static TIME_FORMAT: &'static str = "%H:%M:%S";
@@ -219,12 +220,15 @@ impl<'a> Iterator for Iter<'a> {
                     channel: self.context.channel.clone().map(Into::into),
                 }));
             }
+            if option_env!("FUSE").is_some() {
+                panic!("Shouldn't reach here, this is a bug!")
+            }
         }
     }
 }
 
 impl Decode for Energymech {
-    fn decode<'a>(&'a mut self,
+    fn decode<'a>(&'a self,
                   context: &'a Context,
                   input: &'a mut BufRead)
                   -> Box<Iterator<Item = ilc_base::Result<Event<'a>>> + 'a> {
@@ -250,6 +254,13 @@ impl Encode for Energymech {
                               from,
                               content))
             }
+            &Event { ty: Type::Notice { ref from, ref content }, ref time, .. } => {
+                try!(writeln!(&mut output,
+                              "[{}] -{}- {}",
+                              time.with_format(&context.timezone, TIME_FORMAT),
+                              from,
+                              content))
+            }
             &Event { ty: Type::Action { ref from, ref content }, ref time, .. } => {
                 try!(writeln!(&mut output,
                               "[{}] * {} {}",
@@ -302,7 +313,12 @@ impl Encode for Energymech {
                               nick.as_ref().expect("Nick not present, but required."),
                               new_topic))
             }
-            _ => (),
+            _ => {
+                if option_env!("FUSE").is_some() {
+                    panic!("Shouldn't reach here, this is a bug!")
+                }
+                ()
+            }
         }
         Ok(())
     }

formats/weechat/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ilc-format-weechat"
-version = "0.1.1"
+version = "0.2.0"
 description = "IRC log converter/collector/cruncher"
 homepage = "https://github.com/tilpner/ilc"
 license = "Apache-2.0"
@@ -10,4 +10,4 @@ authors = ["Till Höppner <[email protected]>"]
 [dependencies]
 log = "0.3.5"
 chrono = "0.2.19"
-ilc-base = "0.1.0"
+ilc-base = "~0.2"

formats/weechat/src/lib.rs

@@ -26,6 +26,7 @@ use ilc_base::format::{rejoin, strip_one};
 
 use log::LogLevel::Info;
 
+#[derive(Copy, Clone)]
 pub struct Weechat;
 
 static TIME_DATE_FORMAT: &'static str = "%Y-%m-%d %H:%M:%S";
@@ -76,44 +77,40 @@ impl<'a> Iterator for Iter<'a> {
 
             let len = tokens.len();
 
-            if len >= 6 && tokens[5] == "has" {
-                // 2016-02-25 01:15:05 --> Foo ([email protected]) has joined #example
-                if len >= 8 && tokens[6] == "joined" {
-                    return Some(Ok(Event {
-                        ty: Type::Join {
-                            nick: tokens[3].to_owned().into(),
-                            mask: Some(strip_one(tokens[4]).into()),
-                        },
-                        channel: Some(tokens[7].to_owned().into()),
-                        time: parse_time(&self.context, tokens[0], tokens[1]),
-                    }));
-                }
-                // 2016-02-25 01:36:13 <-- Foo ([email protected]) has left #channel (Some reason)
-                else if len >= 9 && tokens[6] == "left" {
-                    return Some(Ok(Event {
-                        ty: Type::Part {
-                            nick: tokens[3].to_owned().into(),
-                            mask: Some(strip_one(&tokens[4]).into()),
-                            reason: Some(strip_one(&rejoin(&tokens[8..], &split_tokens[8..]))
-                                             .into()),
-                        },
-                        channel: Some(tokens[7].to_owned().into()),
-                        time: parse_time(&self.context, tokens[0], tokens[1]),
-                    }));
-                }
-                // 2016-02-25 01:38:55 <-- Foo ([email protected]) has quit (Some reason)
-                else if len >= 8 && tokens[6] == "quit" {
-                    return Some(Ok(Event {
-                        ty: Type::Quit {
-                            nick: tokens[3].to_owned().into(),
-                            mask: Some(strip_one(tokens[4]).into()),
-                            reason: Some(strip_one(&rejoin(&tokens[7..], &split_tokens[7..]))
-                                             .into()),
-                        },
-                        time: parse_time(&self.context, tokens[0], tokens[1]),
-                        channel: self.context.channel.clone().map(Into::into),
-                    }));
-                }
+            // 2016-02-25 01:15:05 --> Foo ([email protected]) has joined #example
+            if len >= 8 && tokens[5] == "has" && tokens[6] == "joined" {
+                return Some(Ok(Event {
+                    ty: Type::Join {
+                        nick: tokens[3].to_owned().into(),
+                        mask: Some(strip_one(tokens[4]).into()),
+                    },
+                    channel: Some(tokens[7].to_owned().into()),
+                    time: parse_time(&self.context, tokens[0], tokens[1]),
+                }));
+            }
+            // 2016-02-25 01:36:13 <-- Foo ([email protected]) has left #channel (Some reason)
+            else if len >= 9 && tokens[5] == "has" && tokens[6] == "left" {
+                return Some(Ok(Event {
+                    ty: Type::Part {
+                        nick: tokens[3].to_owned().into(),
+                        mask: Some(strip_one(&tokens[4]).into()),
+                        reason: Some(strip_one(&rejoin(&tokens[8..], &split_tokens[8..])).into()),
+                    },
+                    channel: Some(tokens[7].to_owned().into()),
+                    time: parse_time(&self.context, tokens[0], tokens[1]),
+                }));
+            }
+            // 2016-02-25 01:38:55 <-- Foo ([email protected]) has quit (Some reason)
+            else if len >= 8 && tokens[5] == "has" && tokens[6] == "quit" {
+                return Some(Ok(Event {
+                    ty: Type::Quit {
+                        nick: tokens[3].to_owned().into(),
+                        mask: Some(strip_one(tokens[4]).into()),
+                        reason: Some(strip_one(&rejoin(&tokens[7..], &split_tokens[7..])).into()),
+                    },
+                    time: parse_time(&self.context, tokens[0], tokens[1]),
+                    channel: self.context.channel.clone().map(Into::into),
+                }));
             } else if len >= 3 && tokens[2] == "--" {
                 // 2016-02-25 04:32:15	--	Notice(playbot-veno): ""
                 if len >= 5 && tokens[3].starts_with("Notice(") {
@@ -176,7 +173,7 @@ impl<'a> Iterator for Iter<'a> {
 }
 
 impl Decode for Weechat {
-    fn decode<'a>(&'a mut self,
+    fn decode<'a>(&'a self,
                   context: &'a Context,
                   input: &'a mut BufRead)
                   -> Box<Iterator<Item = ilc_base::Result<Event<'a>>> + 'a> {
@@ -252,7 +249,12 @@ impl Encode for Weechat {
                               from,
                               content))
             }
-            _ => (),
+            _ => {
+                if option_env!("FUSE").is_some() {
+                    panic!("Shouldn't reach here, this is a bug!")
+                }
+                ()
+            }
         }
         Ok(())
     }