Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BFD-3706: Refactor NPI Enrichment #2471

Draft
wants to merge 20 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ jobs:
cache: 'maven'

- name: 'Run Maven Build'
run: mvn --threads 1C --quiet --batch-mode -Dmaven.build.cache.enabled=false -Dapidocgen.skip=false -DbfdOps.skip=false verify
run: mvn --threads 1C --batch-mode -Dmaven.build.cache.enabled=false -Dapidocgen.skip=false -DbfdOps.skip=false verify
working-directory: ./apps

# TODO: Conformance testing is currently missing from mvn-verify. BFD-3245 will re-examine conformance regression testing in BFD.
8 changes: 8 additions & 0 deletions apps/bfd-data-npi/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,14 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>
<build>
<finalName>bfd-data-npi</finalName>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package gov.cms.bfd.data.npi.dto;

import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;

/** DTO used to supply the server with NPI enrichment information. */
@Builder
@Getter
@AllArgsConstructor
@NoArgsConstructor
@JsonInclude(NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class NPIData {
/** Provider or Org npi. */
String npi;

/** Entity Type. Will be 1 or 2. */
String entityTypeCode;

/** Organization name. */
String providerOrganizationName;

/** Taxonomy code. */
String taxonomyCode;

/** Taxonomy display. */
String taxonomyDisplay;

/** Provider name prefix. */
String providerNamePrefix;

/** Provider first name. */
String providerFirstName;

/** Provider middle name. */
String providerMiddleName;

/** Provider last name. */
String providerLastName;

/** Provider suffix. */
String providerNameSuffix;

/** Provider credential. */
String providerCredential;
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
package gov.cms.bfd.data.npi.lookup;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import gov.cms.bfd.data.npi.dto.NPIData;
import gov.cms.bfd.data.npi.utility.App;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.zip.InflaterInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -21,6 +27,9 @@ public class NPIOrgLookup {
/** A field to return the production org lookup. */
private static NPIOrgLookup npiOrgLookupForProduction;

/** Zlib compression header. */
private static final byte[] COMPRESSED_HEADER = {120, -100};

/**
* Factory method for creating a {@link NPIOrgLookup } for production that does not include the
* fake org name.
Expand Down Expand Up @@ -53,7 +62,7 @@ public NPIOrgLookup(Map<String, String> npiOrgMap) {
* @throws IOException if there is an issue reading file
*/
public NPIOrgLookup(InputStream npiDataStream) throws IOException {
npiOrgHashMap = readNPIOrgDataStream(npiDataStream);
npiOrgHashMap = readNPIOrgDataStream(new BufferedInputStream(npiDataStream));
}

/**
Expand All @@ -63,7 +72,7 @@ public NPIOrgLookup(InputStream npiDataStream) throws IOException {
* @param npiNumber - npiNumber value in claim records
* @return the npi org data display string
*/
public Optional<String> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
public Optional<NPIData> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
/*
* Handle bad data (e.g. our random test data) if npiNumber is empty
*/
Expand All @@ -72,8 +81,14 @@ public Optional<String> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
}

if (npiOrgHashMap.containsKey(npiNumber.get())) {
String npiDisplay = npiOrgHashMap.get(npiNumber.get());
return Optional.of(npiDisplay);
String json = npiOrgHashMap.get(npiNumber.get());
ObjectMapper mapper = new ObjectMapper();
try {
NPIData npiData = mapper.readValue(json, NPIData.class);
return Optional.of(npiData);
} catch (JsonProcessingException e) {
return Optional.empty();
}
}

return Optional.empty();
Expand All @@ -88,24 +103,44 @@ public Optional<String> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
* @return the hashmapped for npis and the npi org names
*/
protected Map<String, String> readNPIOrgDataStream(InputStream inputStream) throws IOException {
Map<String, String> npiProcessedData = new HashMap<String, String>();

try (final InputStream npiStream = inputStream;
final BufferedReader npiReader = new BufferedReader(new InputStreamReader(npiStream))) {

String line = "";
while ((line = npiReader.readLine()) != null) {
String npiDataColumns[] = line.split("\t");
if (npiDataColumns.length == 2) {
npiProcessedData.put(
npiDataColumns[0].replace("\"", ""), npiDataColumns[1].replace("\"", ""));
Map<String, String> npiProcessedData = new HashMap<>();
// if the stream is compressed, we will have to use InflaterInputStream to read it.
boolean isCompressedStream = isStreamDeflated(inputStream);
String line;
try (final InputStream npiStream =
isCompressedStream ? new InflaterInputStream(inputStream) : inputStream;
final BufferedReader reader = new BufferedReader(new InputStreamReader(npiStream))) {
while ((line = reader.readLine()) != null) {
// the first part of the line will be the NPI, and the second part is the json.
String[] tsv = line.split("\t");
if (tsv.length == 2) {
npiProcessedData.put(tsv[0], tsv[1]);
}
}
}

return npiProcessedData;
}

/**
* Checks if a stream is deflated. We will read the first two bytes of the stream to compare
* against the Zlib header (used by DeflaterOutputStream), then reset the stream back to the
* beginning.
*
* @param inputStream The stream to check
* @return true if the stream is deflated
* @throws IOException on read error.
*/
public boolean isStreamDeflated(InputStream inputStream) throws IOException {
// Mark the current position in the stream.
inputStream.mark(2);
// Read the first two bytes
byte[] bytes = new byte[2];
int bytesRead = inputStream.read(bytes);
// Reset the stream to the marked position
inputStream.reset();
return (bytesRead == 2 && Arrays.equals(bytes, COMPRESSED_HEADER));
}

/**
* Returns a inputStream from file name passed in.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ public static void main(String[] args) throws IOException {
if (args.length > 1) {
throw new IllegalArgumentException("Invalid arguments supplied for NPI download.");
}

String outputDir = null;
Optional<String> downloadUrl = Optional.empty();

Expand All @@ -56,7 +55,6 @@ public static void main(String[] args) throws IOException {
if (!Strings.isNullOrEmpty(downloadUrlProperty)) {
downloadUrl = Optional.of(downloadUrlProperty);
}

DataUtilityCommons.getNPIOrgNames(outputDir, downloadUrl, NPI_RESOURCE);
}
}
Loading
Loading