fix multi-level includes in different directories

The code to handle includes lost the information about the base path after the first level of include.
Consider the following situation:
    somedir/main.a2l -> /include "foo/first.a2l" (already OK)
    somedir/foo/first.a2l -> /include "bar/second.a2l" (fixed here)

Previously the code would look for $pwd/foo/bar/second.a2l instead of somedir/foo/bar/second.a2l

a2lfile/src/a2ml.rs

@@ -1,5 +1,8 @@
-use super::writer::{TaggedItemInfo, Writer};
-use super::{loader, tokenizer};
+use crate::{
+    loader, tokenizer,
+    writer::{TaggedItemInfo, Writer},
+    Filename,
+};
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::path::Path;
@@ -121,7 +124,7 @@ pub enum GenericIfData {
 
 // tokenize()
 // Tokenize the text of the a2ml section
-fn tokenize_a2ml(filename: &str, input: &str) -> Result<(Vec<TokenType>, String), String> {
+fn tokenize_a2ml(filename: &Filename, input: &str) -> Result<(Vec<TokenType>, String), String> {
     let mut amltokens = Vec::<TokenType>::new();
     let input_bytes = input.as_bytes();
     let datalen = input_bytes.len();
@@ -247,7 +250,7 @@ fn tokenize_tag(input: &str, bytepos: &mut usize) -> Result<TokenType, String> {
 }
 
 fn tokenize_include(
-    filename: &str,
+    filename: &Filename,
     input: &str,
     bytepos: &mut usize,
 ) -> Result<(Vec<TokenType>, String), String> {
@@ -301,13 +304,13 @@ fn tokenize_include(
     }
 
     let incname = &input[fname_idx_start..fname_idx_end];
-    let incfilename = loader::make_include_filename(incname, filename);
+    let incfilename = loader::make_include_filename(incname, &filename.full);
 
     // check if incname is an accessible file
     let incpathref = Path::new(&incfilename);
     let loadresult = loader::load(incpathref);
     if let Ok(incfiledata) = loadresult {
-        tokenize_a2ml(incpathref.to_string_lossy().as_ref(), &incfiledata)
+        tokenize_a2ml(&Filename::from(incpathref), &incfiledata)
     } else {
         Err(format!("failed reading {}", incpathref.display()))
     }
@@ -389,7 +392,10 @@ fn make_errtxt(pos: usize, input_bytes: &[u8]) -> Cow<str> {
 // parse an a2ml fragment in an a2l file
 // The target data structure is the parsing definition used by the a2l parser, so that the
 // a2ml can control the parsing of IF_DATA blocks
-pub(crate) fn parse_a2ml(filename: &str, input: &str) -> Result<(A2mlTypeSpec, String), String> {
+pub(crate) fn parse_a2ml(
+    filename: &Filename,
+    input: &str,
+) -> Result<(A2mlTypeSpec, String), String> {
     let (tok_result, complete_string) = tokenize_a2ml(filename, input)?;
     let mut tok_iter = tok_result.iter().peekable();
 
@@ -1335,42 +1341,43 @@ impl PartialEq for GenericIfDataTaggedItem {
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::io::Write;
     use tempfile::tempdir;
 
     #[test]
     fn tokenize() {
-        let (tokenvec, _) = tokenize_a2ml("test", "       ").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "       ").unwrap();
         assert!(tokenvec.is_empty());
 
-        let (tokenvec, _) = tokenize_a2ml("test", "/* // */").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "/* // */").unwrap();
         assert!(tokenvec.is_empty());
-        let (tokenvec, _) = tokenize_a2ml("test", "/*/*/").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "/*/*/").unwrap();
         assert!(tokenvec.is_empty());
-        let (tokenvec, _) = tokenize_a2ml("test", "/***/").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "/***/").unwrap();
         assert!(tokenvec.is_empty());
-        let tokenvec_err = tokenize_a2ml("test", "/* ");
+        let tokenvec_err = tokenize_a2ml(&Filename::from("test"), "/* ");
         assert!(tokenvec_err.is_err());
-        let (tokenvec, _) = tokenize_a2ml("test", "//*/").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "//*/").unwrap();
         assert!(tokenvec.is_empty());
 
-        let (tokenvec, _) = tokenize_a2ml("test", r#""TAG""#).unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), r#""TAG""#).unwrap();
         assert_eq!(tokenvec.len(), 1);
         let _tag = TokenType::Tag("TAG".to_string());
         assert!(matches!(&tokenvec[0], _tag));
 
-        let (tokenvec, _) = tokenize_a2ml("test", ";").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), ";").unwrap();
         assert_eq!(tokenvec.len(), 1);
         assert!(matches!(tokenvec[0], TokenType::Semicolon));
 
-        let (tokenvec, _) = tokenize_a2ml("test", "0").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "0").unwrap();
         assert_eq!(tokenvec.len(), 1);
         assert!(matches!(tokenvec[0], TokenType::Constant(0)));
 
-        let (tokenvec, _) = tokenize_a2ml("test", "0x03").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "0x03").unwrap();
         assert_eq!(tokenvec.len(), 1);
         assert!(matches!(tokenvec[0], TokenType::Constant(3)));
 
-        let (tokenvec, _) = tokenize_a2ml("test", "123456").unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), "123456").unwrap();
         assert_eq!(tokenvec.len(), 1);
         assert!(matches!(tokenvec[0], TokenType::Constant(123456)));
 
@@ -1381,19 +1388,24 @@ mod test {
         // create the empty "testfile" so that it can be included
         std::fs::File::create_new("testfile").unwrap();
 
-        let (tokenvec, _) = tokenize_a2ml("test", r#"/include "testfile""#).unwrap();
+        let (tokenvec, _) =
+            tokenize_a2ml(&Filename::from("test"), r#"/include "testfile""#).unwrap();
         assert_eq!(tokenvec.len(), 0);
 
-        let (tokenvec, _) = tokenize_a2ml("test", r#"/include"testfile""#).unwrap();
+        let (tokenvec, _) =
+            tokenize_a2ml(&Filename::from("test"), r#"/include"testfile""#).unwrap();
         assert_eq!(tokenvec.len(), 0);
 
-        let (tokenvec, _) = tokenize_a2ml("test", r#"/include testfile"#).unwrap();
+        let (tokenvec, _) = tokenize_a2ml(&Filename::from("test"), r#"/include testfile"#).unwrap();
         assert_eq!(tokenvec.len(), 0);
 
-        let err_result = tokenize_a2ml("test", r#"/include "testfile_unclosed_quote"#);
+        let err_result = tokenize_a2ml(
+            &Filename::from("test"),
+            r#"/include "testfile_unclosed_quote"#,
+        );
         assert!(err_result.is_err());
 
-        let err_result = tokenize_a2ml("test", r#" "unclosed "#);
+        let err_result = tokenize_a2ml(&Filename::from("test"), r#" "unclosed "#);
         assert!(err_result.is_err());
     }
 
@@ -1512,10 +1524,39 @@ mod test {
             A2mlTypeSpec::TaggedStruct(taggedstruct_hashmap),
         ]);
 
-        let parse_result = parse_a2ml("test", TEST_INPUT);
+        let parse_result = parse_a2ml(&Filename::from("test"), TEST_INPUT);
         assert!(parse_result.is_ok());
         let (a2ml_spec, _complete_string) = parse_result.unwrap();
         println!("{:?}", a2ml_spec);
         assert_eq!(a2ml_spec, expected_parse_result);
     }
+
+    #[test]
+    fn included_files() {
+        let dir = tempdir().unwrap();
+
+        // base file at <tempdir>/base
+        let base_filename = dir.path().join("base");
+        let mut basefile = std::fs::File::create_new(&base_filename).unwrap();
+        basefile.write(br#"/include "abc/include1""#).unwrap();
+
+        // include file 1 at <tempdir>/abc/include1
+        let subdir = dir.path().join("abc");
+        let inc1name = subdir.join("include1");
+        std::fs::create_dir(&subdir).unwrap();
+        let mut incfile1 = std::fs::File::create_new(&inc1name).unwrap();
+        incfile1.write(br#"/include "def/include2""#).unwrap();
+
+        // include file 2 at <tempdir>/abc/def/include2
+        let subdir2 = subdir.join("def");
+        std::fs::create_dir(&subdir2).unwrap();
+        let _incfile2 = std::fs::File::create_new(subdir2.join("include2")).unwrap();
+
+        // run the a2ml tokenizer. It should not return an error from the includes
+        let filetext = loader::load(&base_filename).unwrap();
+        let (tokens, fulltext) =
+            tokenize_a2ml(&Filename::from(base_filename.as_path()), &filetext).unwrap();
+        assert_eq!(tokens.len(), 0);
+        assert!(fulltext.trim().is_empty());
+    }
 }

a2lfile/src/ifdata.rs

@@ -250,7 +250,7 @@ fn parse_ifdata_taggeditem(
                 let endtag = parser.get_token_text(endident);
                 if endtag != tag {
                     return Err(ParserError::IncorrectEndTag {
-                        filename: parser.filenames[context.fileid].clone(),
+                        filename: parser.filenames[context.fileid].to_string(),
                         error_line: parser.last_token_position,
                         tag: endtag.to_owned(),
                         block: newcontext.element.clone(),
@@ -437,7 +437,7 @@ fn parse_unknown_taggedstruct(
             let endtag = parser.get_token_text(endident);
             if endtag != tag {
                 return Err(ParserError::IncorrectEndTag {
-                    filename: parser.filenames[newcontext.fileid].clone(),
+                    filename: parser.filenames[newcontext.fileid].to_string(),
                     error_line: parser.last_token_position,
                     tag: endtag.to_owned(),
                     block: newcontext.element.clone(),
@@ -470,7 +470,7 @@ fn parse_unknown_taggedstruct(
     }) = parser.peek_token()
     {
         return Err(ParserError::InvalidBegin {
-            filename: parser.filenames[context.fileid].clone(),
+            filename: parser.filenames[context.fileid].to_string(),
             error_line: parser.last_token_position,
             block: context.element.clone(),
         });
@@ -539,7 +539,8 @@ fn remove_unknown_ifdata_from_list(ifdata_list: &mut Vec<IfData>) {
 
 #[cfg(test)]
 mod ifdata_test {
-    use crate::{self as a2lfile, IfData};
+    use super::*;
+    use crate::{self as a2lfile, Filename, IfData};
 
     crate::a2ml_specification! {
         <A2mlTest>
@@ -739,21 +740,24 @@ mod ifdata_test {
         assert!(decoded_ifdata.none.is_some());
     }
 
-    fn parse_helper(
-        ifdata: &str,
-    ) -> Result<(Option<super::GenericIfData>, bool), super::ParserError> {
-        let token_result = a2lfile::tokenizer::tokenize("".to_string(), 0, ifdata).unwrap();
+    fn parse_helper(ifdata: &str) -> Result<(Option<GenericIfData>, bool), ParserError> {
+        let token_result =
+            a2lfile::tokenizer::tokenize(&Filename::from("test"), 0, ifdata).unwrap();
         let mut log_msgs = Vec::new();
         let ifdatas = [ifdata.to_string()];
-        let filenames = ["".to_string()];
-        let mut parser = super::ParserState::new_internal(
+        let filenames = [Filename::from("test")];
+        let mut parser = ParserState::new_internal(
             &token_result.tokens,
             &ifdatas,
             &filenames,
             &mut log_msgs,
             false,
         );
-        parser.builtin_a2mlspec = Some(a2lfile::a2ml::parse_a2ml("test", A2MLTEST_TEXT).unwrap().0);
+        parser.builtin_a2mlspec = Some(
+            a2lfile::a2ml::parse_a2ml(&Filename::from("test"), A2MLTEST_TEXT)
+                .unwrap()
+                .0,
+        );
         super::parse_ifdata(
             &mut parser,
             &a2lfile::ParseContext {
@@ -765,9 +769,7 @@ mod ifdata_test {
         )
     }
 
-    fn check_and_decode(
-        result: Result<(Option<super::GenericIfData>, bool), super::ParserError>,
-    ) -> A2mlTest {
+    fn check_and_decode(result: Result<(Option<GenericIfData>, bool), ParserError>) -> A2mlTest {
         let (data, valid) = result.unwrap();
         assert!(data.is_some());
         assert_eq!(valid, true);

a2lfile/src/lib.rs

@@ -18,6 +18,8 @@ mod writer;
 pub use namemap::{ModuleNameMap, NameMapCompuTab, NameMapObject, NameMapTypedef};
 pub use parser::ParserError;
 use std::convert::AsRef;
+use std::ffi::OsString;
+use std::fmt::Display;
 use std::path::Path;
 use std::path::PathBuf;
 use thiserror::Error;
@@ -191,7 +193,7 @@ fn load_impl(
     a2ml_spec: Option<String>,
 ) -> Result<A2lFile, A2lError> {
     // tokenize the input data
-    let tokenresult = tokenizer::tokenize(path.to_string_lossy().to_string(), 0, filedata)
+    let tokenresult = tokenizer::tokenize(&Filename::from(path), 0, filedata)
         .map_err(|tokenizer_error| A2lError::TokenizerError { tokenizer_error })?;
 
     if tokenresult.tokens.is_empty() {
@@ -206,7 +208,7 @@ fn load_impl(
     // if a built-in A2ml specification was passed as a string, then it is parsed here
     if let Some(spec) = a2ml_spec {
         parser.builtin_a2mlspec = Some(
-            a2ml::parse_a2ml(path.to_string_lossy().as_ref(), &spec)
+            a2ml::parse_a2ml(&Filename::from(path), &spec)
                 .map_err(|parse_err| A2lError::InvalidBuiltinA2mlSpec { parse_err })?
                 .0,
         );
@@ -231,7 +233,7 @@ fn load_impl(
 pub fn load_fragment(a2ldata: &str) -> Result<Module, A2lError> {
     let fixed_a2ldata = format!(r#"fragment "" {a2ldata} /end MODULE"#);
     // tokenize the input data
-    let tokenresult = tokenizer::tokenize("(fragment)".to_string(), 0, &fixed_a2ldata)
+    let tokenresult = tokenizer::tokenize(&Filename::from("(fragment)"), 0, &fixed_a2ldata)
         .map_err(|tokenizer_error| A2lError::TokenizerError { tokenizer_error })?;
     let firstline = tokenresult.tokens.first().map_or(1, |tok| tok.line);
     let context = ParseContext {
@@ -369,6 +371,56 @@ impl Module {
     }
 }
 
+#[derive(Debug, Clone)]
+struct Filename {
+    // the full filename, which has been extended with a base path relative to the working directory
+    full: OsString,
+    // the "display" name, i.e. the name that appears in an /include directive or an error message
+    display: String,
+}
+
+impl Filename {
+    pub(crate) fn new(full: OsString, display: &str) -> Self {
+        Self {
+            full,
+            display: display.to_string(),
+        }
+    }
+}
+
+impl From<&str> for Filename {
+    fn from(value: &str) -> Self {
+        Self {
+            full: OsString::from(value),
+            display: String::from(value),
+        }
+    }
+}
+
+impl From<&Path> for Filename {
+    fn from(value: &Path) -> Self {
+        Self {
+            display: value.to_string_lossy().to_string(),
+            full: OsString::from(value),
+        }
+    }
+}
+
+impl From<OsString> for Filename {
+    fn from(value: OsString) -> Self {
+        Self {
+            display: value.to_string_lossy().to_string(),
+            full: value,
+        }
+    }
+}
+
+impl Display for Filename {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.display)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;