Textures working a little bit

This commit is contained in:
hal8174 2025-08-16 01:35:52 +02:00
parent 1196cb7758
commit 2476d2d49a
Signed by: hal8174
SSH key fingerprint: SHA256:JwuqS+eVfISfKr+DkDQ6NWAbGd1jFAHkPpCM1yCnlTs
7 changed files with 407 additions and 84 deletions

51
Cargo.lock generated
View file

@ -1207,9 +1207,9 @@ dependencies = [
[[package]]
name = "image"
version = "0.25.5"
version = "0.25.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b"
checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a"
dependencies = [
"bytemuck",
"byteorder-lite",
@ -1359,10 +1359,11 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.72"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9"
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
dependencies = [
"once_cell",
"wasm-bindgen",
]
@ -2258,7 +2259,7 @@ dependencies = [
name = "ray-tracing-image"
version = "0.1.0"
dependencies = [
"image 0.25.5",
"image 0.25.6",
"rand",
"ray-tracing-core",
"ray-tracing-renderer",
@ -2290,6 +2291,7 @@ name = "ray-tracing-pbrt-scene"
version = "0.1.0"
dependencies = [
"clap",
"image 0.25.6",
"miette",
"nom 8.0.0",
"ray-tracing-core",
@ -2462,6 +2464,12 @@ dependencies = [
"windows-sys 0.60.2",
]
[[package]]
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.18"
@ -2530,9 +2538,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.133"
version = "1.0.142"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
dependencies = [
"itoa",
"memchr",
@ -3077,24 +3085,24 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.95"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e"
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.95"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358"
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.89",
@ -3103,9 +3111,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.95"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56"
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -3113,9 +3121,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.95"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
@ -3126,9 +3134,12 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.95"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
dependencies = [
"unicode-ident",
]
[[package]]
name = "wayland-client"
@ -3205,9 +3216,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.72"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112"
checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
dependencies = [
"js-sys",
"wasm-bindgen",

View file

@ -9,3 +9,4 @@ clap = { version = "4.5.41", features = ["derive"] }
miette = { version = "7.6.0", features = ["fancy"] }
nom = "8.0.0"
thiserror = "2.0.12"
image = "0.25.6"

View file

@ -25,9 +25,9 @@ struct Lexer {
}
impl Lexer {
fn new(path: impl AsRef<Path>) -> Result<Self> {
fn new(path: PathBuf, base_path: PathBuf) -> Result<Self> {
Ok(Self {
input: Tokenizer::new(path)?,
input: Tokenizer::new(path, base_path)?,
})
}
}
@ -396,7 +396,7 @@ impl Lexer {
"Rotate" => Some(parse_rotate(&mut self.input)),
"Transform" => Some(parse_transform(&mut self.input).map(Statement::Transform)),
"Texture" => Some(
texture::parse_texture(&mut self.input)
texture::parse_texture(&mut self.input, textures)
.map(|(name, texture)| Statement::Texture(name, texture)),
),
"ConcatTransform" => {
@ -545,23 +545,25 @@ impl<I: Iterator<Item = Result<u8, std::io::Error>>> Iterator for BytesToChar<I>
}
}
struct Parser<P> {
path: P,
inner: Option<Box<Parser<PathBuf>>>,
struct Parser {
path: PathBuf,
base_path: PathBuf,
inner: Option<Box<Parser>>,
iter: Lexer,
}
impl<P: AsRef<Path> + std::fmt::Debug> Parser<P> {
fn new(path: P) -> Result<Self> {
impl Parser {
fn new(path: PathBuf, base_path: PathBuf) -> Result<Self> {
Ok(Self {
iter: Lexer::new(path.as_ref())?,
iter: Lexer::new(path.clone(), base_path.clone())?,
base_path,
path,
inner: None,
})
}
}
impl<P: AsRef<Path>> Parser<P> {
impl Parser {
fn next(
&mut self,
textures: &HashMap<String, Arc<dyn PbrtTexture>>,
@ -575,8 +577,8 @@ impl<P: AsRef<Path>> Parser<P> {
match self.iter.next(textures) {
Some(Ok(Statement::Include(s))) => {
let path = self.path.as_ref().parent().unwrap().join(s);
self.inner = Some(Box::new(Parser::new(path).unwrap()));
let path = self.path.parent().unwrap().join(s);
self.inner = Some(Box::new(Parser::new(path, self.base_path.clone()).unwrap()));
self.next(textures)
}
@ -670,7 +672,13 @@ fn inner_parse_pbrt(path: impl AsRef<Path> + std::fmt::Debug) -> Result<Pbrt> {
// unwrap on context.last() ok because context is never empty
let mut context_ctm = vec![AffineTransform::identity()];
let mut parser = Parser::new(path)?;
let mut parser = Parser::new(
path.as_ref().to_path_buf(),
path.as_ref()
.parent()
.ok_or_else(|| miette!("parent from file not found"))?
.to_path_buf(),
)?;
// parse global settings
@ -761,10 +769,15 @@ fn inner_parse_pbrt(path: impl AsRef<Path> + std::fmt::Debug) -> Result<Pbrt> {
Statement::Unknown(s, _items) => {
eprintln!("Unknown statement: {s}")
}
Statement::Texture(name, texture) => {
textures.insert(name, texture);
}
s => bail!("unexpected statemnet in world settings: {s:?}"),
}
}
dbg!(textures);
Ok(pbrt)
}

View file

@ -1,19 +1,23 @@
use imagemap::{ImageMapEncoding, ImageMapWrap, SpectrumImageMapTexture};
use miette::{Result, miette};
use ray_tracing_core::{color::Color, prelude::Float};
use std::sync::Arc;
use ray_tracing_core::{affine_transform::AffineTransform, color::Color, prelude::Float};
use std::{collections::HashMap, ops::Deref, sync::Arc};
use crate::tokenizer::Tokenizer;
pub trait PbrtTexture: std::fmt::Debug {
mod checkerboard;
mod imagemap;
pub trait PbrtTexture: std::fmt::Debug + Send + Sync {
fn get_2d_float_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_float_texture>>;
fn get_2d_spectrum_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_spectrum_texture>>;
}
pub trait Pbrt_2d_float_texture: std::fmt::Debug {
pub trait Pbrt_2d_float_texture: std::fmt::Debug + Sync + Send {
fn get(&self, u: Float, v: Float) -> Float;
}
pub trait Pbrt_2d_spectrum_texture: std::fmt::Debug {
pub trait Pbrt_2d_spectrum_texture: std::fmt::Debug + Sync + Send {
fn get(&self, u: Float, v: Float) -> Color;
}
@ -33,12 +37,20 @@ impl TextureMapping {
}
}
pub fn parse_texture(input: &mut Tokenizer) -> Result<(String, Arc<dyn PbrtTexture>)> {
let texture_name: String = input.parse_next()?;
let texture_type: String = input.parse_next()?;
fn parse_spectrum(input: &mut Tokenizer) -> Result<Color> {
todo!()
}
fn parse_rgb(input: &mut Tokenizer) -> Result<Color> {
let t = input.parse_list()?;
Ok(Color::new(t[0], t[1], t[2]))
}
fn parse_spectrum_texture(
input: &mut Tokenizer,
textures: &HashMap<String, Arc<dyn PbrtTexture>>,
) -> Result<Arc<dyn PbrtTexture>> {
let texture_class: String = input.parse_next()?;
match texture_class.as_str() {
"\"checkerboard\"" => {
let t = parse_dict!(input =>
@ -47,54 +59,115 @@ pub fn parse_texture(input: &mut Tokenizer) -> Result<(String, Arc<dyn PbrtTextu
vscale, Float, 1.0;
udelta, Float, 0.0;
vdelta, Float, 0.0;
dimension, u8, 2
dimension, u8, 2;
tex1, ValueTexture<Color, String>, ValueTexture::Value(Color::white());
tex2, ValueTexture<Color, String>, ValueTexture::Value(Color::black())
=>
mapping, "\"mapping\"", TextureMapping::new(input.parse_parameter()?)?;
uscale, "\"uscale\"", input.parse_parameter()?;
vscale, "\"vscale\"", input.parse_parameter()?;
udelta, "\"udelta\"", input.parse_parameter()?;
vdelta, "\"vdelta\"", input.parse_parameter()?;
dimension, "\"dimension\"", input.parse_parameter()?
mapping, "\"string mapping\"", TextureMapping::new(input.parse_parameter()?)?;
uscale, "\"float uscale\"", input.parse_parameter()?;
vscale, "\"float vscale\"", input.parse_parameter()?;
udelta, "\"float udelta\"", input.parse_parameter()?;
vdelta, "\"float vdelta\"", input.parse_parameter()?;
dimension, "\"integer dimension\"", input.parse_parameter()?;
tex1, "\"rgb tex1\"", ValueTexture::Value(parse_rgb(input)?);
tex1, "\"spectrum tex1\"", ValueTexture::Value(parse_spectrum(input)?);
tex1, "\"texture tex1\"", ValueTexture::Texture(input.parse_parameter()?);
tex2, "\"rgb tex2\"", ValueTexture::Value(parse_rgb(input)?);
tex2, "\"spectrum tex2\"", ValueTexture::Value(parse_spectrum(input)?);
tex2, "\"texture tex2\"", ValueTexture::Texture(input.parse_parameter()?)
);
dbg!(t);
todo!()
let mapping = UVMapping::new(t.mapping, t.uscale, t.vscale, t.udelta, t.vdelta);
match t.dimension {
2 => Ok(Arc::new(checkerboard::SpectrumCheckerboardTexture2d {
mapping,
tex: [
match t.tex1 {
ValueTexture::Value(v) => ValueTexture::Value(v),
ValueTexture::Texture(name) => ValueTexture::Texture(
Arc::clone(
textures
.get(&name)
.ok_or_else(|| miette!("unable to find texture"))?,
)
.get_2d_spectrum_texture()?,
),
},
match t.tex2 {
ValueTexture::Value(v) => ValueTexture::Value(v),
ValueTexture::Texture(name) => ValueTexture::Texture(
Arc::clone(
textures
.get(&name)
.ok_or_else(|| miette!("unable to find texture"))?,
)
.get_2d_spectrum_texture()?,
),
},
],
})),
_ => Err(miette!("Only dimension 2 and 3 are supported")),
}
}
"\"checkerboard\"" => {
"\"imagemap\"" => {
let t = parse_dict!(input =>
mapping, TextureMapping, TextureMapping::UV;
uscale, Float, 1.0;
vscale, Float, 1.0;
udelta, Float, 0.0;
vdelta, Float, 0.0
vdelta, Float, 0.0;
filename, String, String::from("");
wrap, ImageMapWrap, ImageMapWrap::Repeat;
encoding, ImageMapEncoding, ImageMapEncoding::SRGB;
scale, Float, 1.0
=>
mapping, "\"mapping\"", TextureMapping::new(input.parse_parameter()?)?;
uscale, "\"uscale\"", input.parse_parameter()?;
vscale, "\"vscale\"", input.parse_parameter()?;
udelta, "\"udelta\"", input.parse_parameter()?;
vdelta, "\"vdelta\"", input.parse_parameter()?
mapping, "\"string mapping\"", TextureMapping::new(input.parse_parameter()?)?;
uscale, "\"float uscale\"", input.parse_parameter()?;
vscale, "\"float vscale\"", input.parse_parameter()?;
udelta, "\"float udelta\"", input.parse_parameter()?;
vdelta, "\"float vdelta\"", input.parse_parameter()?;
filename, "\"string filename\"", input.parse_parameter()?;
wrap, "\"string wrap\"", ImageMapWrap::new(input.parse_parameter()?)?;
encoding, "\"string encoding\"", ImageMapEncoding::new(input.parse_parameter()?)?;
scale, "\"float scale\"", input.parse_parameter()?
);
todo!()
let mapping = UVMapping::new(t.mapping, t.uscale, t.vscale, t.udelta, t.vdelta);
let path = dbg!(input.get_base_path()).join(t.filename.trim_matches('\"'));
Ok(Arc::new(SpectrumImageMapTexture::new(
mapping,
t.scale,
t.wrap,
t.encoding,
false,
dbg!(path),
)?))
}
_ => Err(miette!("unknown error")),
_ => Err(miette!("unknown error {texture_class}")),
}
}
pub fn parse_texture(
input: &mut Tokenizer,
textures: &HashMap<String, Arc<dyn PbrtTexture>>,
) -> Result<(String, Arc<dyn PbrtTexture>)> {
let texture_name: String = input.parse_next()?;
let texture_type: String = input.parse_next()?;
match texture_type.as_str() {
"\"spectrum\"" => parse_spectrum_texture(input, textures).map(|t| (texture_name, t)),
_ => Err(miette!("Texture type has to be spectrum or float")),
}
}
#[derive(Debug)]
enum FloatTexture2d {
Value(Float),
Texture {
float: Option<Arc<dyn Pbrt_2d_float_texture>>,
spectrum: Option<Arc<dyn Pbrt_2d_spectrum_texture>>,
},
}
#[derive(Debug)]
struct TextureD2<T> {
inner: T,
struct UVMapping {
uscale: Float,
vscale: Float,
udelta: Float,
@ -102,19 +175,32 @@ struct TextureD2<T> {
mapping: TextureMapping,
}
impl<T: PbrtTexture> PbrtTexture for TextureD2<T> {
fn get_2d_float_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_float_texture>> {
todo!()
impl UVMapping {
fn new(
mapping: TextureMapping,
uscale: Float,
vscale: Float,
udelta: Float,
vdelta: Float,
) -> Self {
Self {
uscale,
vscale,
udelta,
vdelta,
mapping,
}
}
fn get_2d_spectrum_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_spectrum_texture>> {
todo!()
fn map(&self, u: Float, v: Float) -> (Float, Float) {
match self.mapping {
TextureMapping::UV => (u * self.uscale + self.udelta, v * self.vscale + self.vdelta),
TextureMapping::Spherical => todo!(),
}
}
}
#[derive(Debug)]
struct CheckerboardTexture {
dimension: u8,
tex1: FloatTexture2d,
tex2: FloatTexture2d,
enum ValueTexture<V, T> {
Value(V),
Texture(T),
}

View file

@ -0,0 +1,74 @@
use super::*;
#[derive(Debug)]
pub(super) struct FloatCheckerboardTexture2d {
pub(super) mapping: UVMapping,
pub(super) tex: [ValueTexture<Float, Arc<dyn Pbrt_2d_float_texture>>; 2],
}
impl Pbrt_2d_float_texture for FloatCheckerboardTexture2d {
fn get(&self, u: Float, v: Float) -> Float {
let (u, v) = self.mapping.map(u, v);
let even = if u >= 0.0 {
u.fract() >= 0.5
} else {
u.fract() >= -0.5
} ^ if v >= 0.0 {
v.fract() >= 0.5
} else {
v.fract() >= -0.5
};
match &self.tex[even as usize] {
ValueTexture::Value(v) => *v,
ValueTexture::Texture(t) => t.get(u, v),
}
}
}
impl PbrtTexture for FloatCheckerboardTexture2d {
fn get_2d_float_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_float_texture>> {
Ok(self)
}
fn get_2d_spectrum_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_spectrum_texture>> {
Err(miette!("Unable to use this texture as a spectrum texture"))
}
}
#[derive(Debug)]
pub(super) struct SpectrumCheckerboardTexture2d {
pub(super) mapping: UVMapping,
pub(super) tex: [ValueTexture<Color, Arc<dyn Pbrt_2d_spectrum_texture>>; 2],
}
impl Pbrt_2d_spectrum_texture for SpectrumCheckerboardTexture2d {
fn get(&self, u: Float, v: Float) -> Color {
let (u, v) = self.mapping.map(u, v);
let even = if u >= 0.0 {
u.fract() >= 0.5
} else {
u.fract() >= -0.5
} ^ if v >= 0.0 {
v.fract() >= 0.5
} else {
v.fract() >= -0.5
};
match &self.tex[even as usize] {
ValueTexture::Value(v) => *v,
ValueTexture::Texture(t) => t.get(u, v),
}
}
}
impl PbrtTexture for SpectrumCheckerboardTexture2d {
fn get_2d_float_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_float_texture>> {
Err(miette!("Unable to use this texture as a float texture"))
}
fn get_2d_spectrum_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_spectrum_texture>> {
Ok(self)
}
}

View file

@ -0,0 +1,128 @@
use std::path::Path;
use miette::IntoDiagnostic;
use super::*;
#[derive(Debug)]
pub(super) enum ImageMapWrap {
Repeat,
Black,
Clamp,
}
impl ImageMapWrap {
pub(super) fn new(x: String) -> Result<Self> {
match x.as_str() {
"\"repeat\"" => Ok(Self::Repeat),
"\"black\"" => Ok(Self::Black),
"\"clamp\"" => Ok(Self::Clamp),
_ => Err(miette!("error image map wrap")),
}
}
}
#[derive(Debug)]
pub(super) enum ImageMapEncoding {
SRGB,
Linear,
Gamma(Float),
}
impl ImageMapEncoding {
pub(super) fn new(x: String) -> Result<Self> {
match x.as_str() {
"\"sRGB\"" => Ok(Self::SRGB),
"\"linear\"" => Ok(Self::Linear),
s if s.starts_with("\"gamma ") => Ok(Self::Gamma(
s.split_at(7)
.1
.trim_matches('\"')
.parse()
.into_diagnostic()?,
)),
_ => Err(miette!("error image map encoding")),
}
}
}
#[derive(Debug)]
pub(super) struct SpectrumImageMapTexture {
mapping: UVMapping,
scale: Float,
wrap: ImageMapWrap,
invert: bool,
image: image::Rgb32FImage,
}
fn srgb_nonlinear(x: f32) -> f32 {
if x <= 0.04045 {
x / 12.92
} else {
((x + 0.055) / 1.055).powf(2.4)
}
}
impl SpectrumImageMapTexture {
pub(super) fn new(
mapping: UVMapping,
scale: Float,
wrap: ImageMapWrap,
encoding: ImageMapEncoding,
invert: bool,
path: impl AsRef<Path>,
) -> Result<Self> {
let image = image::open(path).unwrap();
let mut image = image.into_rgb32f();
match encoding {
ImageMapEncoding::SRGB => {
for pixel in image.pixels_mut() {
pixel.0 = [
srgb_nonlinear(pixel.0[0]),
srgb_nonlinear(pixel.0[1]),
srgb_nonlinear(pixel.0[2]),
];
}
}
ImageMapEncoding::Linear => (),
ImageMapEncoding::Gamma(gamma) => {
for pixel in image.pixels_mut() {
pixel.0 = [
pixel.0[0].powf(gamma),
pixel.0[1].powf(gamma),
pixel.0[2].powf(gamma),
];
}
}
}
Ok(Self {
mapping,
scale,
invert,
wrap,
image,
})
}
}
impl Pbrt_2d_spectrum_texture for SpectrumImageMapTexture {
fn get(&self, u: Float, v: Float) -> Color {
let (u, v) = self.mapping.map(u, v);
let (w, h) = self.image.dimensions();
todo!()
}
}
impl PbrtTexture for SpectrumImageMapTexture {
fn get_2d_float_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_float_texture>> {
Err(miette!("Unable to use this texture as a float texture"))
}
fn get_2d_spectrum_texture(self: Arc<Self>) -> Result<Arc<dyn Pbrt_2d_spectrum_texture>> {
Ok(self)
}
}

View file

@ -12,19 +12,29 @@ use crate::{BytesToChar, SourceFile};
pub struct Tokenizer {
inner: InnerTokenizer,
path: PathBuf,
base_path: PathBuf,
peeked: Option<Option<Result<(String, SourceSpan)>>>,
last_span: SourceSpan,
}
impl Tokenizer {
pub fn new(path: impl AsRef<Path>) -> Result<Self> {
pub fn new(path: PathBuf, base_path: PathBuf) -> Result<Self> {
Ok(Self {
peeked: None,
path: PathBuf::from(path.as_ref()),
inner: InnerTokenizer::new(path)?,
inner: InnerTokenizer::new(&path)?,
path,
base_path,
last_span: SourceSpan::new(0.into(), 0),
})
}
pub fn get_path(&self) -> &Path {
&self.path
}
pub fn get_base_path(&self) -> &Path {
&self.base_path
}
}
struct InnerTokenizer {