Refactoring

ray-tracing-pbrt-scene/src/error.rs

@@ -0,0 +1,116 @@
+use std::{io::Read, path::PathBuf};
+
+use miette::{SourceCode, SourceSpan, SpanContents};
+
+#[derive(Debug)]
+pub struct SourceFile {
+    pub(crate) path: PathBuf,
+}
+
+impl SourceCode for SourceFile {
+    fn read_span<'a>(
+        &'a self,
+        span: &miette::SourceSpan,
+        context_lines_before: usize,
+        context_lines_after: usize,
+    ) -> std::result::Result<Box<dyn miette::SpanContents<'a> + 'a>, miette::MietteError> {
+        let mut file = std::fs::File::open(&self.path)?;
+
+        // read keeping the last
+
+        let mut data = Vec::new();
+
+        file.read_to_end(&mut data)?;
+
+        let data = data.leak();
+
+        let line = data.iter().filter(|&&x| x == b'\n').count();
+
+        let mut start_new_lines = 0;
+
+        let start_pos = data[0..span.offset()]
+            .iter()
+            .rposition(|&x| {
+                if x == b'\n' {
+                    if start_new_lines > context_lines_before {
+                        true
+                    } else {
+                        start_new_lines += 1;
+                        false
+                    }
+                } else {
+                    false
+                }
+            })
+            .map(|x| x + 1)
+            .unwrap_or(0);
+
+        let mut end_new_lines = 0;
+
+        let end_pos = data[span.offset()..]
+            .iter()
+            .position(|&x| {
+                if x == b'\n' {
+                    if end_new_lines >= context_lines_after {
+                        true
+                    } else {
+                        end_new_lines += 1;
+                        false
+                    }
+                } else {
+                    false
+                }
+            })
+            .map(|x| x + span.offset())
+            .unwrap_or(data[span.offset()..].len());
+
+        let compact_data = &data[start_pos..end_pos];
+
+        Ok(Box::new(SourceFileSpanContents {
+            span: SourceSpan::new(start_pos.into(), end_pos - start_pos),
+            start_line_number: line - start_new_lines,
+            line_count: start_new_lines + 1 + end_new_lines,
+            path: self.path.clone(),
+            data: compact_data,
+        }))
+    }
+}
+
+#[derive(Debug)]
+struct SourceFileSpanContents {
+    data: &'static [u8],
+    span: SourceSpan,
+    start_line_number: usize,
+    line_count: usize,
+    path: PathBuf,
+}
+
+impl<'a> SpanContents<'a> for SourceFileSpanContents {
+    fn data(&'_ self) -> &'a [u8] {
+        self.data
+    }
+
+    fn span(&self) -> &SourceSpan {
+        &self.span
+    }
+
+    fn line(&self) -> usize {
+        self.start_line_number
+    }
+
+    fn column(&self) -> usize {
+        0
+    }
+
+    fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    fn name(&self) -> Option<&str> {
+        self.path.to_str()
+    }
+
+    fn language(&self) -> Option<&str> {
+        None
+    }
+}

ray-tracing-pbrt-scene/src/lib.rs

@@ -1,134 +1,20 @@
-use std::{
+use crate::tokenizer::Tokenizer;
-    fs::File,
+use error::SourceFile;
-    io::{BufRead, BufReader, Bytes, Read, Seek, Write},
+use miette::{Diagnostic, IntoDiagnostic, Result, SourceSpan, bail, miette};
-    iter::Peekable,
-    os::unix::fs::FileExt,
-    path::{Path, PathBuf},
-};
-
-use miette::{
-    Diagnostic, Error, IntoDiagnostic, Result, SourceCode, SourceSpan, SpanContents, bail, miette,
-};
-use nom::IResult;
 use ray_tracing_core::{
     affine_transform::AffineTransform,
     math::{Dir3, Pos3},
     prelude::Float,
 };
+use std::{
+    iter::Peekable,
+    path::{Path, PathBuf},
+};
 use thiserror::Error;
 
-#[derive(Debug)]
+mod tokenizer;
-struct SourceFile {
-    path: PathBuf,
-}
 
-impl SourceCode for SourceFile {
+mod error;
-    fn read_span<'a>(
-        &'a self,
-        span: &miette::SourceSpan,
-        context_lines_before: usize,
-        context_lines_after: usize,
-    ) -> std::result::Result<Box<dyn miette::SpanContents<'a> + 'a>, miette::MietteError> {
-        let mut file = std::fs::File::open(&self.path)?;
-
-        // read keeping the last
-
-        let mut data = Vec::new();
-
-        file.read_to_end(&mut data)?;
-
-        let data = data.leak();
-
-        let line = data.iter().filter(|&&x| x == b'\n').count();
-
-        let mut start_new_lines = 0;
-
-        let start_pos = data[0..span.offset()]
-            .iter()
-            .rposition(|&x| {
-                if x == b'\n' {
-                    if start_new_lines > context_lines_before {
-                        true
-                    } else {
-                        start_new_lines += 1;
-                        false
-                    }
-                } else {
-                    false
-                }
-            })
-            .map(|x| x + 1)
-            .unwrap_or(0);
-
-        let mut end_new_lines = 0;
-
-        let end_pos = data[span.offset()..]
-            .iter()
-            .position(|&x| {
-                if x == b'\n' {
-                    if end_new_lines >= context_lines_after {
-                        true
-                    } else {
-                        end_new_lines += 1;
-                        false
-                    }
-                } else {
-                    false
-                }
-            })
-            .map(|x| x + span.offset())
-            .unwrap_or(data[span.offset()..].len());
-
-        let compact_data = &data[start_pos..end_pos];
-
-        Ok(Box::new(SourceFileSpanContents {
-            span: SourceSpan::new(start_pos.into(), end_pos - start_pos),
-            start_line_number: line - start_new_lines,
-            line_count: start_new_lines + 1 + end_new_lines,
-            path: self.path.clone(),
-            data: compact_data,
-        }))
-    }
-}
-
-#[derive(Debug)]
-struct SourceFileSpanContents {
-    data: &'static [u8],
-    span: SourceSpan,
-    start_line_number: usize,
-    line_count: usize,
-    path: PathBuf,
-}
-
-impl<'a> SpanContents<'a> for SourceFileSpanContents {
-    fn data(&'_ self) -> &'a [u8] {
-        self.data
-    }
-
-    fn span(&self) -> &SourceSpan {
-        &self.span
-    }
-
-    fn line(&self) -> usize {
-        self.start_line_number
-    }
-
-    fn column(&self) -> usize {
-        0
-    }
-
-    fn line_count(&self) -> usize {
-        self.line_count
-    }
-
-    fn name(&self) -> Option<&str> {
-        self.path.to_str()
-    }
-
-    fn language(&self) -> Option<&str> {
-        None
-    }
-}
 
 #[derive(Error, Debug, Diagnostic)]
 #[error("oops!")]
@@ -141,105 +27,15 @@ struct MyBad {
     bad_bit: SourceSpan,
 }
 
-struct Tokenizer<I: Iterator<Item = Result<(usize, char)>>> {
+struct Lexer {
-    input_iterator: Peekable<I>,
+    input: Tokenizer,
 }
 
-impl<I: Iterator<Item = Result<(usize, char)>>> Tokenizer<I> {
+impl Lexer {
-    fn new(input_iterator: I) -> Self {
+    fn new(path: impl AsRef<Path>) -> Result<Self> {
-        Self {
+        Ok(Self {
-            input_iterator: input_iterator.peekable(),
+            input: Tokenizer::new(path)?,
-        }
+        })
-    }
-}
-
-impl<I> Iterator for Tokenizer<I>
-where
-    I: Iterator<Item = Result<(usize, char)>>,
-{
-    type Item = Result<String>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        while self.input_iterator.peek().is_some_and(|c| {
-            c.as_ref()
-                .is_ok_and(|&(_, c)| c.is_whitespace() || c == '#')
-        }) {
-            if self
-                .input_iterator
-                .peek()
-                .is_some_and(|c| c.as_ref().is_ok_and(|&(_, c)| c == '#'))
-            {
-                while self
-                    .input_iterator
-                    .next()
-                    .is_some_and(|c| c.is_ok_and(|(_, c)| c != '\n'))
-                {}
-            } else {
-                match self.input_iterator.next()? {
-                    Ok(_) => (),
-                    Err(e) => return Some(Err(e)),
-                }
-            }
-        }
-
-        match self.input_iterator.next() {
-            Some(Ok((_, '['))) => Some(Ok(String::from('['))),
-            Some(Ok((_, ']'))) => Some(Ok(String::from(']'))),
-            Some(Ok((_, '"'))) => {
-                let mut r = String::from('"');
-                while let Some(p) = self
-                    .input_iterator
-                    .next_if(|c| c.as_ref().is_ok_and(|&(_, c)| c != '"'))
-                {
-                    match p {
-                        Ok((_, c)) => r.push(c),
-                        Err(e) => return Some(Err(e)),
-                    }
-                }
-                if self
-                    .input_iterator
-                    .next()
-                    .is_none_or(|c| !c.is_ok_and(|(_, c)| c == '"'))
-                {
-                    return Some(Err(From::from(MyBad {
-                        src: SourceFile {
-                            path: PathBuf::from("ray-tracing-pbrt-scene/example.pbrt"),
-                        },
-                        bad_bit: SourceSpan::new(40.into(), 4),
-                    })));
-                };
-                r.push('"');
-                Some(Ok(r))
-            }
-            Some(Ok((_, c))) => {
-                let mut r = String::new();
-                r.push(c);
-                while let Some(p) = self.input_iterator.next_if(|c| {
-                    c.as_ref()
-                        .is_ok_and(|&(_, c)| c != '#' && c != '[' && c != ']' && !c.is_whitespace())
-                }) {
-                    match p {
-                        Ok((_, c)) => r.push(c),
-                        Err(e) => return Some(Err(e)),
-                    }
-                }
-                Some(Ok(r))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}
-
-struct Lexer<I: Iterator<Item = Result<(usize, char)>>> {
-    input: Peekable<Tokenizer<I>>,
-}
-
-impl<I: Iterator<Item = Result<(usize, char)>>> Lexer<I> {
-    fn new(iter: I) -> Self {
-        Self {
-            input: Tokenizer::new(iter).peekable(),
-        }
     }
 }
 
@@ -254,7 +50,7 @@ enum Statement {
     Transform(AffineTransform),
 }
 
-fn parse_look_at(iter: &mut impl Iterator<Item = Result<String>>) -> Result<Statement> {
+fn parse_look_at(iter: &mut Tokenizer) -> Result<Statement> {
     let eye = Pos3::new(
         iter.next()
             .ok_or(miette!("missing argument"))??
@@ -304,9 +100,8 @@ fn parse_look_at(iter: &mut impl Iterator<Item = Result<String>>) -> Result<Stat
     ))
 }
 
-fn parse_parameter<I, T>(iter: &mut Peekable<Tokenizer<I>>) -> Result<T>
+fn parse_parameter<T>(iter: &mut Tokenizer) -> Result<T>
 where
-    I: Iterator<Item = Result<(usize, char)>>,
     T: std::str::FromStr,
     <T as std::str::FromStr>::Err:
         std::marker::Send + std::marker::Sync + std::error::Error + 'static,
@@ -332,9 +127,8 @@ where
     }
 }
 
-fn parse_list<I, T>(iter: &mut Peekable<Tokenizer<I>>, data: &mut Vec<T>) -> Result<()>
+fn parse_list<T>(iter: &mut Tokenizer, data: &mut Vec<T>) -> Result<()>
 where
-    I: Iterator<Item = Result<(usize, char)>>,
     T: std::str::FromStr,
     <T as std::str::FromStr>::Err:
         std::marker::Send + std::marker::Sync + std::error::Error + 'static,
@@ -363,13 +157,8 @@ where
     Ok(())
 }
 
-fn parse_list_2<I, T, P, F>(
+fn parse_list_2<T, P, F>(iter: &mut Tokenizer, data: &mut Vec<P>, f: F) -> Result<()>
-    iter: &mut Peekable<Tokenizer<I>>,
-    data: &mut Vec<P>,
-    f: F,
-) -> Result<()>
 where
-    I: Iterator<Item = Result<(usize, char)>>,
     T: std::str::FromStr,
     <T as std::str::FromStr>::Err:
         std::marker::Send + std::marker::Sync + std::error::Error + 'static,
@@ -409,13 +198,8 @@ where
     Ok(())
 }
 
-fn parse_list_3<I, T, P, F>(
+fn parse_list_3<T, P, F>(iter: &mut Tokenizer, data: &mut Vec<P>, f: F) -> Result<()>
-    iter: &mut Peekable<Tokenizer<I>>,
-    data: &mut Vec<P>,
-    f: F,
-) -> Result<()>
 where
-    I: Iterator<Item = Result<(usize, char)>>,
     T: std::str::FromStr,
     <T as std::str::FromStr>::Err:
         std::marker::Send + std::marker::Sync + std::error::Error + 'static,
@@ -463,9 +247,7 @@ where
     Ok(())
 }
 
-fn parse_shape<I: Iterator<Item = Result<(usize, char)>>>(
+fn parse_shape(iter: &mut Tokenizer) -> Result<Statement> {
-    iter: &mut Peekable<Tokenizer<I>>,
-) -> std::result::Result<Statement, Error> {
     let shape_type = iter.next().ok_or(miette!("unable to get shape type"))??;
 
     match shape_type.as_str() {
@@ -690,7 +472,7 @@ fn parse_shape<I: Iterator<Item = Result<(usize, char)>>>(
     }
 }
 
-impl<I: Iterator<Item = Result<(usize, char)>>> Lexer<I> {
+impl Lexer {
     fn next(&mut self, ctm: AffineTransform) -> Option<Result<Statement>> {
         match self.input.next() {
             Some(Ok(s)) => match s.as_str() {
@@ -745,9 +527,7 @@ impl<I: Iterator<Item = Result<(usize, char)>>> Lexer<I> {
     }
 }
 
-fn parse_transform<I: Iterator<Item = Result<(usize, char)>>>(
+fn parse_transform(input: &mut Tokenizer) -> Result<AffineTransform> {
-    input: &mut Peekable<Tokenizer<I>>,
-) -> Result<AffineTransform> {
     if !input
         .next()
         .is_none_or(|p| p.is_ok_and(|p| p.as_str() == "["))
@@ -776,17 +556,15 @@ fn parse_transform<I: Iterator<Item = Result<(usize, char)>>>(
         bail!("invalid transform entry")
     }
 
-    Ok(AffineTransform::new([
+    AffineTransform::new([
         [v[0], v[4], v[8], v[12]],
         [v[1], v[5], v[9], v[13]],
         [v[2], v[6], v[10], v[14]],
     ])
-    .ok_or(miette!("Unable to invert transformation"))?)
+    .ok_or(miette!("Unable to invert transformation"))
 }
 
-fn parse_translate<I: Iterator<Item = Result<(usize, char)>>>(
+fn parse_translate(iter: &mut Tokenizer) -> Result<Statement> {
-    iter: &mut Peekable<Tokenizer<I>>,
-) -> Result<Statement> {
     let pos = Pos3::new(
         iter.next()
             .ok_or(miette!("missing argument"))??
@@ -807,9 +585,7 @@ fn parse_translate<I: Iterator<Item = Result<(usize, char)>>>(
     )))
 }
 
-fn parse_scale<I: Iterator<Item = Result<(usize, char)>>>(
+fn parse_scale(iter: &mut Tokenizer) -> Result<Statement> {
-    iter: &mut Peekable<Tokenizer<I>>,
-) -> Result<Statement> {
     Ok(Statement::ConcatTransform(AffineTransform::scale(
         iter.next()
             .ok_or(miette!("missing argument"))??
@@ -826,9 +602,7 @@ fn parse_scale<I: Iterator<Item = Result<(usize, char)>>>(
     )))
 }
 
-fn parse_rotate<I: Iterator<Item = Result<(usize, char)>>>(
+fn parse_rotate(iter: &mut Tokenizer) -> Result<Statement> {
-    iter: &mut Peekable<Tokenizer<I>>,
-) -> Result<Statement> {
     let angle = iter
         .next()
         .ok_or(miette!("missing argument"))??
@@ -885,17 +659,16 @@ impl<I: Iterator<Item = Result<u8, std::io::Error>>> Iterator for BytesToChar<I>
 struct Parser<P> {
     path: P,
     inner: Option<Box<Parser<PathBuf>>>,
-    iter: Lexer<BytesToChar<Bytes<BufReader<File>>>>,
+    iter: Lexer,
 }
 
 impl<P: AsRef<Path> + std::fmt::Debug> Parser<P> {
     fn new(path: P) -> Result<Self> {
         dbg!(&path);
-        let bufreader = BufReader::new(std::fs::File::open(&path).into_diagnostic()?);
         Ok(Self {
+            iter: Lexer::new(path.as_ref())?,
             path,
             inner: None,
-            iter: Lexer::new(BytesToChar::new(bufreader.bytes())),
         })
     }
 }

ray-tracing-pbrt-scene/src/tokenizer.rs

@@ -0,0 +1,136 @@
+use miette::{IntoDiagnostic, Result, SourceSpan};
+use std::{
+    fs::File,
+    io::{BufReader, Bytes, Read},
+    iter::Peekable,
+    path::{Path, PathBuf},
+};
+
+use crate::{BytesToChar, MyBad, SourceFile};
+
+pub struct Tokenizer {
+    inner: InnerTokenizer,
+    peeked: Option<Option<Result<String>>>,
+    last_span: SourceSpan,
+}
+
+impl Tokenizer {
+    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
+        Ok(Self {
+            peeked: None,
+            inner: InnerTokenizer::new(path)?,
+            last_span: SourceSpan::new(0.into(), 0),
+        })
+    }
+}
+
+struct InnerTokenizer {
+    input_iterator: Peekable<BytesToChar<Bytes<BufReader<File>>>>,
+}
+
+impl InnerTokenizer {
+    fn new(path: impl AsRef<Path>) -> Result<Self> {
+        Ok(Self {
+            input_iterator: BytesToChar::new(
+                BufReader::new(File::open(path).into_diagnostic()?).bytes(),
+            )
+            .peekable(),
+        })
+    }
+}
+
+impl InnerTokenizer {
+    fn next(&mut self) -> Option<Result<String>> {
+        while self.input_iterator.peek().is_some_and(|c| {
+            c.as_ref()
+                .is_ok_and(|&(_, c)| c.is_whitespace() || c == '#')
+        }) {
+            if self
+                .input_iterator
+                .peek()
+                .is_some_and(|c| c.as_ref().is_ok_and(|&(_, c)| c == '#'))
+            {
+                while self
+                    .input_iterator
+                    .next()
+                    .is_some_and(|c| c.is_ok_and(|(_, c)| c != '\n'))
+                {}
+            } else {
+                match self.input_iterator.next()? {
+                    Ok(_) => (),
+                    Err(e) => return Some(Err(e)),
+                }
+            }
+        }
+
+        match self.input_iterator.next() {
+            Some(Ok((_, '['))) => Some(Ok(String::from('['))),
+            Some(Ok((_, ']'))) => Some(Ok(String::from(']'))),
+            Some(Ok((_, '"'))) => {
+                let mut r = String::from('"');
+                while let Some(p) = self
+                    .input_iterator
+                    .next_if(|c| c.as_ref().is_ok_and(|&(_, c)| c != '"'))
+                {
+                    match p {
+                        Ok((_, c)) => r.push(c),
+                        Err(e) => return Some(Err(e)),
+                    }
+                }
+                if self
+                    .input_iterator
+                    .next()
+                    .is_none_or(|c| !c.is_ok_and(|(_, c)| c == '"'))
+                {
+                    return Some(Err(From::from(MyBad {
+                        src: SourceFile {
+                            path: PathBuf::from("ray-tracing-pbrt-scene/example.pbrt"),
+                        },
+                        bad_bit: SourceSpan::new(40.into(), 4),
+                    })));
+                };
+                r.push('"');
+                Some(Ok(r))
+            }
+            Some(Ok((_, c))) => {
+                let mut r = String::new();
+                r.push(c);
+                while let Some(p) = self.input_iterator.next_if(|c| {
+                    c.as_ref()
+                        .is_ok_and(|&(_, c)| c != '#' && c != '[' && c != ']' && !c.is_whitespace())
+                }) {
+                    match p {
+                        Ok((_, c)) => r.push(c),
+                        Err(e) => return Some(Err(e)),
+                    }
+                }
+                Some(Ok(r))
+            }
+            Some(Err(e)) => Some(Err(e)),
+            None => None,
+        }
+    }
+}
+
+impl Tokenizer {
+    pub fn next(&mut self) -> Option<Result<String>> {
+        match self.peeked.take() {
+            Some(v) => v,
+            None => self.inner.next(),
+        }
+    }
+
+    pub fn next_if(
+        &mut self,
+        func: impl FnOnce(&Result<String>) -> bool,
+    ) -> Option<Result<String>> {
+        match self.next() {
+            Some(matched) if func(&matched) => Some(matched),
+            other => {
+                assert!(self.peeked.is_none());
+                self.peeked = Some(other);
+                None
+            }
+        }
+    }
+}