BFD-3331: Represents Data Dictionary in BFD Model along FHIR Resources & pulls in descriptions from codebook

.gitignore

@@ -10,6 +10,7 @@
 
 # Ignore everything within top-level dist directory
 /dist/**/*
+/dist-temp/**/*
 
 # OSX system files
 .DS_Store

apps/bfd-model/bfd-model-dsl-codegen/bfd-model-dsl-codegen-plugin/pom.xml

@@ -108,6 +108,45 @@
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+        </dependency>
+        <dependency>
+            <!-- At least one "structures" JAR must also be included -->
+            <groupId>ca.uhn.hapi.fhir</groupId>
+            <artifactId>hapi-fhir-structures-r4</artifactId>
+            <version>${hapi-fhir.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>junit</groupId>
+                    <artifactId>junit</artifactId>
+                </exclusion>
+                <!-- Need the newer but compatible version from com.newrelic.telemetry -->
+                <exclusion>
+                    <groupId>com.squareup.okhttp3</groupId>
+                    <artifactId>okhttp</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <!-- This dependency includes the server HAPI-FHIR classes -->
+            <groupId>ca.uhn.hapi.fhir</groupId>
+            <artifactId>hapi-fhir-server</artifactId>
+            <version>${hapi-fhir.version}</version>
+        </dependency>
+        <dependency>
+            <!-- This dependency includes the client HAPI-FHIR classes -->
+            <groupId>ca.uhn.hapi.fhir</groupId>
+            <artifactId>hapi-fhir-client</artifactId>
+            <version>${hapi-fhir.version}</version>
+        </dependency>
+        <dependency>
+            <!-- This dependency includes the base HAPI-FHIR classes -->
+            <groupId>ca.uhn.hapi.fhir</groupId>
+            <artifactId>hapi-fhir-base</artifactId>
+            <version>${hapi-fhir.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

apps/bfd-model/bfd-model-dsl-codegen/bfd-model-dsl-codegen-plugin/src/main/java/gov/cms/model/dsl/codegen/plugin/GenerateDataDictionaryFromDslMojo.java

@@ -0,0 +1,215 @@
+package gov.cms.model.dsl.codegen.plugin;
+
+import gov.cms.model.dsl.codegen.plugin.mappers.CsvToExcel;
+import gov.cms.model.dsl.codegen.plugin.mappers.FhirElementToCsv;
+import gov.cms.model.dsl.codegen.plugin.mappers.FhirElementToJson;
+import gov.cms.model.dsl.codegen.plugin.model.FhirElement;
+import gov.cms.model.dsl.codegen.plugin.model.FhirElementBean;
+import gov.cms.model.dsl.codegen.plugin.model.MappingBean;
+import gov.cms.model.dsl.codegen.plugin.model.ModelUtil;
+import gov.cms.model.dsl.codegen.plugin.model.RootBean;
+import gov.cms.model.dsl.codegen.plugin.util.Version;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.project.MavenProject;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+/**
+ * A Maven Mojo that generates a data dictionary from yaml mappings. The generated transformer class
+ * consists of the following parts:
+ */
+@Mojo(name = "dataDictionary", defaultPhase = LifecyclePhase.GENERATE_SOURCES)
+public class GenerateDataDictionaryFromDslMojo extends AbstractMojo {
+
+  /** Path to a single mapping file or a directory containing one or more mapping files. */
+  @Parameter(property = "mappingPath")
+  private String mappingPath;
+
+  /** Path to a single mapping file or a directory containing one or more mapping files. */
+  @Parameter(property = "fhirMappingPath")
+  private String fhirMappingPath;
+
+  /** Path to a single mapping file or a directory containing one or more mapping files. */
+  @Parameter(property = "codeBookPath")
+  private String codeBookPath;
+
+  /** Project version. */
+  @Parameter(property = "projectVersion")
+  private String projectVersion;
+
+  /** Path to directory to contain generated data dictionary. */
+  @Parameter(property = "destinationDirectory")
+  private String destinationDirectory;
+
+  /** V1 CSV template file path. */
+  @Parameter(property = "v1TemplateFilePath")
+  private String v1TemplateFilePath;
+
+  /** V2 CSV template file path. */
+  @Parameter(property = "v2TemplateFilePath")
+  private String v2TemplateFilePath;
+
+  /**
+   * Instance of {@link MavenProject} used to call {@link
+   * MavenProject#addCompileSourceRoot(String)}.
+   */
+  @Parameter(property = "project", readonly = true)
+  private MavenProject project;
+
+  /** Path to directory to contain generated code. */
+  @Parameter(
+      property = "fhirEntitiesDirectory",
+      defaultValue = "${project.build.directory}/generated-sources/fhirEntities")
+  private String fhirEntitiesDirectory;
+
+  /** Parameterless constructor used by Maven to instantiate the plugin. */
+  public GenerateDataDictionaryFromDslMojo() {}
+
+  /**
+   * Executed by maven to execute the mojo. Reads all mapping files and generates a data dictionary
+   * using fhir elements from every {@link MappingBean}
+   *
+   * @throws MojoExecutionException if the process fails due to some error
+   */
+  public void execute() throws MojoExecutionException {
+    final File fhirMappingDir = MojoUtil.initializeOutputDirectory(fhirEntitiesDirectory);
+    var templateFileMap = Map.of(Version.V1, v1TemplateFilePath, Version.V2, v2TemplateFilePath);
+    var xlsxFilename =
+        String.format("%s/data-dictionary-%s.xlsx", destinationDirectory, projectVersion);
+
+    // Open Excel output here since both API versions are written to the same workbook
+    try (var xlsxOutputStream = new FileOutputStream(xlsxFilename);
+        var workbook = new XSSFWorkbook()) {
+
+      // Process each data dictionary resource directory in turn
+      for (Version version : List.of(Version.V2)) {
+        var resourceDirPath = String.format("%s/%s", fhirMappingPath, version.name());
+        var templatePath = templateFileMap.get(version);
+        final RootBean root =
+            ModelUtil.appendFhirElementsToYamlFiles(
+                mappingPath, fhirMappingDir, resourceDirPath, codeBookPath);
+        MojoUtil.validateModel(root); // modify so fits for our use case
+
+        if (project != null) {
+          project.addCompileSourceRoot(fhirEntitiesDirectory);
+        }
+
+        processYamlFiles(
+            root, destinationDirectory, projectVersion, templatePath, workbook, version);
+      }
+
+      // save Excel workbook file
+      workbook.write(xlsxOutputStream);
+
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Function to encapsulate the processing of a single resource directory.
+   *
+   * @param root root bean containing mappings of fhir elements
+   * @param destinationDirectory the output file path
+   * @param projectVersion the project version
+   * @param csvTemplatePath the path within the module resources to the CSV template file
+   * @param workbook the Excel workbook (shared across data dictionary versions)
+   * @param version the BFD API version, e.g. V1, V2
+   */
+  public static void processYamlFiles(
+      RootBean root,
+      String destinationDirectory,
+      String projectVersion,
+      String csvTemplatePath,
+      XSSFWorkbook workbook,
+      Version version) {
+
+    // name output files
+    var basePath = getOutputFileBaseName(destinationDirectory, projectVersion, version);
+    var jsonPath = basePath + ".json";
+    var csvPath = basePath + ".csv";
+
+    List<FhirElement> allFhirElements = new ArrayList<>();
+
+    try {
+      // create transformers/writers and stream
+      var elementToJson = FhirElementToJson.createInstance(new FileWriter(jsonPath));
+      var elementToCsv = FhirElementToCsv.createInstance(new FileWriter(csvPath), csvTemplatePath);
+      var csvToExcel = CsvToExcel.createInstance(workbook, version);
+      List<FhirElementBean> fhirElements = Collections.emptyList();
+      for (MappingBean mapping : root.getMappings()) {
+        switch (version) {
+          case Version.V2:
+            fhirElements = mapping.getR4FhirElements();
+            break;
+          case Version.V1:
+            System.out.println("V1 not yet supported");
+            break;
+        }
+        List<FhirElement> fhirElementList =
+            fhirElements.stream()
+                .map(
+                    bean ->
+                        FhirElement.builder()
+                            .id(bean.getId())
+                            .name(bean.getName())
+                            .description(bean.getDescription())
+                            .appliesTo(bean.getAppliesTo())
+                            .suppliedIn(bean.getSuppliedIn())
+                            .bfdTableType(bean.getBfdTableType())
+                            .bfdColumnName(bean.getBfdColumnName())
+                            .bfdDbType(bean.getBfdDbType())
+                            .bfdDbSize(bean.getBfdDbSize())
+                            .bfdJavaFieldName(bean.getBfdJavaFieldName())
+                            .ccwMapping(bean.getCcwMapping())
+                            .cclfMapping(bean.getCclfMapping())
+                            .fhirMapping(bean.getFhirMapping())
+                            .build())
+                .toList();
+        allFhirElements.addAll(fhirElementList);
+      }
+      allFhirElements.sort(
+          new Comparator<FhirElement>() {
+            @Override
+            public int compare(final FhirElement fhirElement1, FhirElement fhirElement2) {
+              return Integer.compare(fhirElement1.getId(), fhirElement2.getId());
+            }
+          });
+      allFhirElements.stream()
+          .map(elementToJson)
+          .map(elementToCsv)
+          .flatMap(Collection::stream)
+          .forEach(csvToExcel);
+      csvToExcel.close();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Forms the base file name for output files.
+   *
+   * @param destinationDirectory output file destination directory path
+   * @param projectVersion the project version
+   * @param version the BFD API version
+   * @return a String representing the base output file name
+   */
+  private static String getOutputFileBaseName(
+      String destinationDirectory, String projectVersion, Version version) {
+    return String.format(
+        "%s/%s-data-dictionary-%s", destinationDirectory, version.name(), projectVersion);
+  }
+}