condition agnostic bulk upload: config and initial validation logic (#…

…6698)

* config and part 1

* code smellse

* add a datetime util test

* cover the partial name cases in the tests

* code review feedback

* remove feature flag in one more place

backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConditionAgnosticConvertToDiagnosticReportProps.java

@@ -0,0 +1,15 @@
+package gov.cdc.usds.simplereport.api.converter;
+
+import lombok.Builder;
+import lombok.Getter;
+import org.hl7.fhir.r4.model.Observation;
+import org.hl7.fhir.r4.model.Patient;
+
+@Builder
+@Getter
+public class ConditionAgnosticConvertToDiagnosticReportProps {
+  private String testPerformedCode;
+  private Patient patient;
+  private Observation observation;
+  private String testEffectiveDate;
+}

backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConditionAgnosticConvertToObservationProps.java

@@ -0,0 +1,15 @@
+package gov.cdc.usds.simplereport.api.converter;
+
+import gov.cdc.usds.simplereport.db.model.auxiliary.TestCorrectionStatus;
+import lombok.Builder;
+import lombok.Getter;
+import org.hl7.fhir.r4.model.Patient;
+
+@Builder
+@Getter
+public class ConditionAgnosticConvertToObservationProps {
+  private TestCorrectionStatus correctionStatus;
+  private String testPerformedCode;
+  private Patient patient;
+  private String resultValue;
+}

backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConditionAgnosticConvertToPatientProps.java

@@ -0,0 +1,14 @@
+package gov.cdc.usds.simplereport.api.converter;
+
+import lombok.Builder;
+import lombok.Getter;
+
+@Builder
+@Getter
+public class ConditionAgnosticConvertToPatientProps {
+  private String id;
+  private String firstName;
+  private String lastName;
+  private String nameAbsentReason;
+  private String gender;
+}

backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConditionAgnosticCreateFhirBundleProps.java

@@ -0,0 +1,19 @@
+package gov.cdc.usds.simplereport.api.converter;
+
+import java.util.List;
+import lombok.Builder;
+import lombok.Getter;
+import org.hl7.fhir.r4.model.DiagnosticReport;
+import org.hl7.fhir.r4.model.Observation;
+import org.hl7.fhir.r4.model.Patient;
+import org.springframework.boot.info.GitProperties;
+
+@Builder
+@Getter
+public class ConditionAgnosticCreateFhirBundleProps {
+  private Patient patient;
+  private List<Observation> resultObservations;
+  private DiagnosticReport diagnosticReport;
+  private GitProperties gitProperties;
+  private String processingId;
+}

backend/src/main/java/gov/cdc/usds/simplereport/api/converter/FhirConstants.java

@@ -88,4 +88,16 @@ private FhirConstants() {
 
   public static final String ORDER_EFFECTIVE_DATE_EXTENSION_URL =
       "https://reportstream.cdc.gov/fhir/StructureDefinition/order-effective-date";
+
+  public static final String OBSERVATION_CATEGORY_CODE_SYSTEM =
+      "https://terminology.hl7.org/5.2.0/CodeSystem-observation-category.html";
+  public static final String DATA_ABSENT_REASON_EXTENSION_URL =
+      "http://hl7.org/fhir/StructureDefinition/data-absent-reason";
+
+  public static final String DIAGNOSTIC_CODE_SYSTEM =
+      "https://terminology.hl7.org/5.2.0/CodeSystem-v2-0074.html";
+
+  //  Hardcoded category values for our initial condition agnostic spec
+  public static final String LAB_STRING_LITERAL = "LAB";
+  public static final String LABORATORY_STRING_LITERAL = "laboratory";
 }

backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/ConditionAgnosticResultRow.java

@@ -0,0 +1,117 @@
+package gov.cdc.usds.simplereport.api.model.filerow;
+
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.getValue;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateBiologicalSex;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateDataAbsentReason;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateDateTime;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateTestPerformedCode;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateTestResult;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateTestResultStatus;
+
+import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage;
+import gov.cdc.usds.simplereport.validators.CsvValidatorUtils.ValueOrError;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+
+@Getter
+public class ConditionAgnosticResultRow implements FileRow {
+  final ValueOrError testResultStatus;
+  final ValueOrError testResultEffectiveDate;
+  final ValueOrError testPerformedCode;
+  final ValueOrError testResultValue;
+  final ValueOrError patientId;
+  final ValueOrError patientFirstName;
+  final ValueOrError patientLastName;
+  final ValueOrError patientNameAbsentReason;
+  final ValueOrError patientAdminGender;
+
+  public static final String TEST_RESULT_STATUS = "test_result_status";
+  public static final String TEST_RESULT_EFFECTIVE_DATE = "test_result_effective_date";
+  public static final String TEST_PERFORMED_CODE = "test_performed_code";
+  public static final String TEST_RESULT_VALUE = "test_result_value";
+  public static final String PATIENT_ID = "patient_id";
+  public static final String PATIENT_LAST_NAME = "patient_last_name";
+  public static final String PATIENT_FIRST_NAME = "patient_first_name";
+  public static final String PATIENT_NAME_ABSENT_REASON = "patient_name_absent_reason";
+  public static final String PATIENT_ADMIN_GENDER = "patient_admin_gender";
+
+  private static List<String> initialRequiredFields =
+      List.of(
+          TEST_RESULT_STATUS,
+          TEST_RESULT_EFFECTIVE_DATE,
+          TEST_PERFORMED_CODE,
+          TEST_RESULT_VALUE,
+          PATIENT_ID,
+          PATIENT_ADMIN_GENDER);
+
+  private List<String> requiredFields;
+
+  public ConditionAgnosticResultRow(Map<String, String> rawRow) {
+    this.requiredFields = generateRequiredFields(rawRow);
+    testResultStatus = getValue(rawRow, TEST_RESULT_STATUS, isRequired(TEST_RESULT_STATUS));
+    testResultEffectiveDate =
+        getValue(rawRow, TEST_RESULT_EFFECTIVE_DATE, isRequired(TEST_RESULT_EFFECTIVE_DATE));
+    testPerformedCode = getValue(rawRow, TEST_PERFORMED_CODE, isRequired(TEST_PERFORMED_CODE));
+    testResultValue = getValue(rawRow, TEST_RESULT_VALUE, isRequired(TEST_RESULT_VALUE));
+    patientId = getValue(rawRow, PATIENT_ID, isRequired(PATIENT_ID));
+    patientNameAbsentReason =
+        getValue(rawRow, PATIENT_NAME_ABSENT_REASON, isRequired(PATIENT_NAME_ABSENT_REASON));
+    patientFirstName = getValue(rawRow, PATIENT_FIRST_NAME, isRequired(PATIENT_FIRST_NAME));
+    patientLastName = getValue(rawRow, PATIENT_LAST_NAME, isRequired(PATIENT_LAST_NAME));
+    patientAdminGender = getValue(rawRow, PATIENT_ADMIN_GENDER, isRequired(PATIENT_ADMIN_GENDER));
+  }
+
+  // The schema expects that (first_name || last_name) XOR name_absent_reason be present, so add the
+  // name-related fields to the required fields list accordingly
+  private static List<String> generateRequiredFields(Map<String, String> rawRow) {
+    String firstNameVal = getValue(rawRow, PATIENT_FIRST_NAME, false).getValue();
+    String lastNameVal = getValue(rawRow, PATIENT_LAST_NAME, false).getValue();
+
+    boolean firstNameAbsent = StringUtils.isBlank(firstNameVal);
+    boolean lastNameAbsent = StringUtils.isBlank(lastNameVal);
+
+    List<String> requiredFields = new ArrayList<>(initialRequiredFields);
+
+    if (firstNameAbsent ^ lastNameAbsent) {
+      if (firstNameAbsent) requiredFields.add(PATIENT_LAST_NAME);
+      else requiredFields.add(PATIENT_FIRST_NAME);
+    } else if (firstNameAbsent && lastNameAbsent) {
+      requiredFields.add(PATIENT_NAME_ABSENT_REASON);
+    } else {
+      requiredFields.add(PATIENT_LAST_NAME);
+      requiredFields.add(PATIENT_FIRST_NAME);
+    }
+
+    return requiredFields;
+  }
+
+  @Override
+  public Boolean isRequired(String rowName) {
+    return requiredFields.contains(rowName);
+  }
+
+  @Override
+  public List<FeedbackMessage> validateRequiredFields() {
+    return getPossibleErrorsFromFields();
+  }
+
+  @Override
+  public List<String> getRequiredFields() {
+    return requiredFields;
+  }
+
+  @Override
+  public List<FeedbackMessage> validateIndividualValues() {
+    var errors = new ArrayList<FeedbackMessage>();
+    errors.addAll(validateBiologicalSex(patientAdminGender));
+    errors.addAll(validateDataAbsentReason(patientNameAbsentReason));
+    errors.addAll(validateTestResultStatus(testResultStatus));
+    errors.addAll(validateDateTime(testResultEffectiveDate));
+    errors.addAll(validateTestPerformedCode(testPerformedCode));
+    errors.addAll(validateTestResult(testResultValue));
+    return errors;
+  }
+}

backend/src/main/java/gov/cdc/usds/simplereport/config/FeatureFlagsConfig.java

@@ -28,6 +28,7 @@ public class FeatureFlagsConfig {
   private boolean rsvEnabled;
   private boolean singleEntryRsvEnabled;
   private boolean agnosticEnabled;
+  private boolean agnosticBulkUploadEnabled;
   private boolean testCardRefactorEnabled;
 
   @Scheduled(fixedRateString = "60000") // 1 min
@@ -42,6 +43,7 @@ private void flagMapping(String flagName, Boolean flagValue) {
       case "rsvEnabled" -> setRsvEnabled(flagValue);
       case "singleEntryRsvEnabled" -> setSingleEntryRsvEnabled(flagValue);
       case "agnosticEnabled" -> setAgnosticEnabled(flagValue);
+      case "agnosticBulkUploadEnabled" -> setAgnosticBulkUploadEnabled(flagValue);
       case "testCardRefactorEnabled" -> setTestCardRefactorEnabled(flagValue);
       default -> log.info("no mapping for " + flagName);
     }

backend/src/main/java/gov/cdc/usds/simplereport/config/FileValidatorConfiguration.java

@@ -1,5 +1,6 @@
 package gov.cdc.usds.simplereport.config;
 
+import gov.cdc.usds.simplereport.api.model.filerow.ConditionAgnosticResultRow;
 import gov.cdc.usds.simplereport.api.model.filerow.PatientUploadRow;
 import gov.cdc.usds.simplereport.api.model.filerow.TestResultRow;
 import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService;
@@ -17,6 +18,11 @@ public FileValidator<TestResultRow> testResultRowFileValidator(
         row -> new TestResultRow(row, resultsUploaderCachingService, featureFlagsConfig));
   }
 
+  @Bean
+  public FileValidator<ConditionAgnosticResultRow> conditionAgnosticResultRowFileValidator() {
+    return new FileValidator<>(row -> new ConditionAgnosticResultRow(row));
+  }
+
   @Bean
   public FileValidator<PatientUploadRow> patientUploadRowFileValidator() {
     return new FileValidator<>(PatientUploadRow::new);

backend/src/main/java/gov/cdc/usds/simplereport/config/WebConfiguration.java

@@ -26,7 +26,10 @@ public class WebConfiguration implements WebMvcConfigurer {
   public static final String FEATURE_FLAGS = "/feature-flags";
   public static final String PATIENT_UPLOAD = "/upload/patients";
   public static final String RESULT_UPLOAD = "/upload/results";
+
   public static final String HIV_RESULT_UPLOAD = "/upload/hiv-results";
+
+  public static final String CONDITION_AGNOSTIC_RESULT_UPLOAD = "/upload/condition-agnostic";
   public static final String GRAPH_QL = "/graphql";
 
   @Autowired private RestLoggingInterceptor _loggingInterceptor;

backend/src/main/java/gov/cdc/usds/simplereport/utils/DateTimeUtils.java

@@ -75,12 +75,7 @@ public static ZonedDateTime convertToZonedDateTime(
 
     // If user provided timezone code in datetime field
     if (hasTimezoneSubstring(dateString)) {
-      var timezoneCode = dateString.substring(dateString.lastIndexOf(" ")).trim();
-      try {
-        zoneId = parseZoneId(timezoneCode);
-      } catch (DateTimeException e) {
-        zoneId = FALLBACK_TIMEZONE_ID;
-      }
+      zoneId = parseDateStringZoneId(dateString);
     } else { // Otherwise try to get timezone by address
       zoneId = resultsUploaderCachingService.getZoneIdByAddress(addressForTimezone);
       // If that fails, use fallback
@@ -93,6 +88,24 @@ public static ZonedDateTime convertToZonedDateTime(
     return ZonedDateTime.of(localDateTime, zoneId);
   }
 
+  public static ZonedDateTime convertToZonedDateTime(String dateString) {
+    ZoneId zoneId = parseDateStringZoneId(dateString);
+    LocalDateTime localDateTime = parseLocalDateTime(dateString, DATE_TIME_FORMATTER);
+    return ZonedDateTime.of(localDateTime, zoneId);
+  }
+
+  private static ZoneId parseDateStringZoneId(String dateString) {
+    ZoneId zoneId;
+
+    var timezoneCode = dateString.substring(dateString.lastIndexOf(" ")).trim();
+    try {
+      zoneId = parseZoneId(timezoneCode);
+    } catch (DateTimeException e) {
+      zoneId = FALLBACK_TIMEZONE_ID;
+    }
+    return zoneId;
+  }
+
   public static boolean hasTimezoneSubstring(String value) {
     return value.matches(TIMEZONE_SUFFIX_REGEX);
   }

backend/src/main/java/gov/cdc/usds/simplereport/utils/ResultUtils.java

@@ -0,0 +1,19 @@
+package gov.cdc.usds.simplereport.utils;
+
+import gov.cdc.usds.simplereport.db.model.auxiliary.TestCorrectionStatus;
+
+public class ResultUtils {
+  private ResultUtils() {
+    throw new IllegalStateException("ResultUtils is a utility class");
+  }
+
+  public static TestCorrectionStatus mapTestResultStatusToSRValue(String input) {
+    switch (input) {
+      case "C":
+        return TestCorrectionStatus.CORRECTED;
+      case "F":
+      default:
+        return TestCorrectionStatus.ORIGINAL;
+    }
+  }
+}

backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java

@@ -93,6 +93,7 @@ public class CsvValidatorUtils {
   private static final String DATE_TIME_REGEX =
       "^(0{0,1}[1-9]|1[0-2])\\/(0{0,1}[1-9]|1\\d|2\\d|3[01])\\/\\d{4}( ([0-1]?[0-9]|2[0-3]):[0-5][0-9]( \\S+)?)?$";
 
+  private static final String LOINC_CODE_REGEX = "([0-9]{5})-[0-9]";
   private static final String EMAIL_REGEX = "^[\\w-\\.]+@([\\w-]+\\.)+[\\w-]{2,4}$";
   private static final String SNOMED_REGEX = "(^[0-9]{9}$)|(^[0-9]{15}$)";
   private static final String CLIA_REGEX = "^[A-Za-z0-9]{2}[Dd][A-Za-z0-9]{7}$";
@@ -198,6 +199,25 @@ public class CsvValidatorUtils {
   private static final Set<String> TEST_RESULT_STATUS_VALUES = Set.of("f", "c");
   public static final String ITEM_SCOPE = "item";
 
+  //  http://hl7.org/fhir/R4/codesystem-data-absent-reason.html#data-absent-reason-unknown
+  private static final Set<String> DATA_ABSENT_REASONS =
+      Set.of(
+          UNKNOWN_LITERAL,
+          "asked-unknown",
+          "temp-unknown",
+          "not-asked",
+          "asked-declined",
+          "masked",
+          "not-applicable",
+          "unsupported",
+          "as-text",
+          "error",
+          "not-a-number",
+          "negative-infinity",
+          "positive-infinity",
+          "not-performed",
+          "not-permitted");
+
   private CsvValidatorUtils() {
     throw new IllegalStateException("CsvValidatorUtils is a utility class");
   }
@@ -214,6 +234,10 @@ public static List<FeedbackMessage> validateTestResult(ValueOrError input) {
     return validateSpecificValueOrSNOMED(input, TEST_RESULT_VALUES);
   }
 
+  public static List<FeedbackMessage> validateTestPerformedCode(ValueOrError input) {
+    return validateRegex(input, LOINC_CODE_REGEX);
+  }
+
   public static List<FeedbackMessage> validateSpecimenType(
       ValueOrError input, Map<String, String> specimenNameSNOMEDMap) {
     List<FeedbackMessage> errors = new ArrayList<>();
@@ -377,6 +401,10 @@ public static Map<String, String> getNextRow(MappingIterator<Map<String, String>
     }
   }
 
+  public static List<FeedbackMessage> validateDataAbsentReason(ValueOrError input) {
+    return validateInSet(input, DATA_ABSENT_REASONS);
+  }
+
   public static ValueOrError getValue(Map<String, String> row, String name, boolean isRequired) {
     String value = row.get(name);
     if (value != null && !value.isBlank()) {

backend/src/main/resources/application-azure-prod.yaml

@@ -28,3 +28,4 @@ features:
   singleEntryRsvEnabled: false
   agnosticEnabled: false
   testCardRefactorEnabled: false
+  agnosticBulkUploadEnabled: false

backend/src/main/resources/application.yaml

@@ -154,6 +154,7 @@ features:
   singleEntryRsvEnabled: true
   agnosticEnabled: true
   testCardRefactorEnabled: true
+  agnosticBulkUploadEnabled: true
 slack:
   hook:
     token: ${SLACK_HOOK_TOKEN:foo}

backend/src/test/java/gov/cdc/usds/simplereport/utils/DateTimeUtilsTest.java

@@ -56,6 +56,15 @@ void testConvertToZonedDateTime_withFallback() {
     testConvertToZonedDateTime("6/28/2023 14:00", ZoneId.of("US/Eastern"));
   }
 
+  @Test
+  void testConvertToZonedDateTime_withFallbackWithJustString() {
+    String dateString = "6/28/2023 14:00";
+
+    var actualZonedDateTime = convertToZonedDateTime(dateString);
+    var expectedZonedDateTime = ZonedDateTime.of(2023, 6, 28, 14, 0, 0, 0, ZoneId.of("US/Eastern"));
+    assertThat(actualZonedDateTime).hasToString(expectedZonedDateTime.toString());
+  }
+
   @Test
   void testConvertToZonedDateTime_withICANNTzIdentifier() {
     testConvertToZonedDateTime("6/28/2023 14:00 US/Samoa", ZoneId.of("US/Samoa"));