Merge pull request cfpb#1225 from jmarin/check-digit

Check digit

Documents/public-api.md

@@ -84,6 +84,151 @@ This documenatation describes de public HMDA Platform HTTP API
 For a definition of these fields, please consult the [HMDA Filing Instructions Guide](http://www.consumerfinance.gov/data-research/hmda/static/for-filers/2017/2017-HMDA-FIG.pdf).
 Please note that the Modified LAR does not include the fields `Loan Application Number`, `Date Application Received` or `Date of Action` described in HMDA Filing Instructions Guide.
 
+## Check Digit
 
+### Check digit generation
+
+* `/uli/checkDigit`
+
+   * `POST` - Calculates check digit and full ULI from a loan id.
+
+Example payload, in `JSON` format:
+
+```json
+{
+  "loanId": "10Bx939c5543TqA1144M999143X"
+}
+```
+
+Example response:
+
+```json
+{
+    "loanId": "10Cx939c5543TqA1144M999143X",
+    "checkDigit": 10,
+    "uli": "10Cx939c5543TqA1144M999143X10"
+}
+```
+
+A file with a list of Loan Ids can also be uploaded to this endpoint for batch check digit generation.
+
+Example file contents:
+
+```
+10Cx939c5543TqA1144M999143X
+10Bx939c5543TqA1144M999143X
+```
+
+Example response in `JSON` format:
+
+```json
+{
+    "loanIds": [
+        {
+            "loanId": "10Bx939c5543TqA1144M999143X",
+            "checkDigit": 38,
+            "uli": "10Bx939c5543TqA1144M999143X38"
+        },
+        {
+            "loanId": "10Cx939c5543TqA1144M999143X",
+            "checkDigit": 10,
+            "uli": "10Cx939c5543TqA1144M999143X10"
+        }
+    ]
+}
+```
+
+* `/uli/checkDigit/csv`
+
+   * `POST` - calculates check digits for loan ids submitted as a file
+
+Example file contents:
+
+```
+10Cx939c5543TqA1144M999143X
+10Bx939c5543TqA1144M999143X
+```
+
+Example response in `CSV` format:
+
+```csv
+loanId,checkDigit,uli
+10Bx939c5543TqA1144M999143X,38,10Bx939c5543TqA1144M999143X38
+10Cx939c5543TqA1144M999143X,10,10Cx939c5543TqA1144M999143X10
+```
+
+### ULI Validation
+
+* `/uli/validate`
+
+   * `POST` - Validates a ULI (correct check digit)
+
+Example payload, in `JSON` format:
+
+```json
+{
+	"uli": "10Bx939c5543TqA1144M999143X38"
+}
+```
+
+Example response:
+
+```json
+{
+    "isValid": true
+}
+```
+
+A file with a list of ULIs can also be uploaded to this endpoint for batch ULI validation.
+
+Example file contents:
+
+```
+10Cx939c5543TqA1144M999143X10
+10Bx939c5543TqA1144M999143X38
+10Bx939c5543TqA1144M999133X38
+```
+
+Example response in `JSON` format:
+
+```json
+{
+    "ulis": [
+        {
+            "uli": "10Cx939c5543TqA1144M999143X10",
+            "isValid": true
+        },
+        {
+            "uli": "10Bx939c5543TqA1144M999143X38",
+            "isValid": true
+        },
+        {
+            "uli": "10Bx939c5543TqA1144M999133X38",
+            "isValid": false
+        }
+    ]
+}
+```
+
+* `/uli/validate/csv`
+
+   * `POST` - Batch validation of ULIs
+
+Example file contents:
+
+```
+10Cx939c5543TqA1144M999143X10
+10Bx939c5543TqA1144M999143X38
+10Bx939c5543TqA1144M999133X38
+```
+
+Example response in `CSV` format:
+
+```csv
+uli,isValid
+10Cx939c5543TqA1144M999143X10,true
+10Bx939c5543TqA1144M999143X38,true
+10Bx939c5543TqA1144M999133X38,false
+```
 
 

api-model/src/main/scala/hmda/api/model/public/ULIModel.scala

@@ -0,0 +1,16 @@
+package hmda.api.model.public
+
+object ULIModel {
+
+  case class Loan(loanId: String)
+  case class ULI(loanId: String, checkDigit: Int, uli: String) {
+    def toCSV: String = s"$loanId,$checkDigit,$uli"
+  }
+  case class LoanCheckDigitResponse(loanIds: Seq[ULI])
+  case class ULICheck(uli: String)
+  case class ULIValidated(isValid: Boolean)
+  case class ULIBatchValidated(uli: String, isValid: Boolean) {
+    def toCSV: String = s"$uli,$isValid"
+  }
+  case class ULIBatchValidatedResponse(ulis: Seq[ULIBatchValidated])
+}

api-model/src/main/scala/hmda/api/protocol/public/ULIProtocol.scala

@@ -0,0 +1,16 @@
+package hmda.api.protocol.public
+
+import hmda.api.model.public.ULIModel._
+import spray.json.DefaultJsonProtocol
+
+trait ULIProtocol extends DefaultJsonProtocol {
+
+  implicit val loanFormat = jsonFormat1(Loan.apply)
+  implicit val uliFormat = jsonFormat3(ULI.apply)
+  implicit val loanCheckDigitResponse = jsonFormat1(LoanCheckDigitResponse.apply)
+  implicit val uliCheckFormat = jsonFormat1(ULICheck.apply)
+  implicit val uliValidatedFormat = jsonFormat1(ULIValidated.apply)
+  implicit val uliBatchValidatedFormat = jsonFormat2(ULIBatchValidated.apply)
+  implicit val uliBatchValidatedResponseFormat = jsonFormat1(ULIBatchValidatedResponse.apply)
+
+}

api/src/main/scala/hmda/api/http/public/PublicHttpApi.scala

@@ -3,13 +3,24 @@ package hmda.api.http.public
 import akka.actor.ActorSystem
 import akka.event.LoggingAdapter
 import akka.stream.ActorMaterializer
-import akka.util.Timeout
+import akka.util.{ ByteString, Timeout }
 import hmda.api.http.HmdaCustomDirectives
 import akka.http.scaladsl.server.Directives._
+import hmda.api.model.public.ULIModel._
+import hmda.validation.engine.lar.ULI._
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import akka.http.scaladsl.marshalling.ToResponseMarshallable
+import akka.http.scaladsl.model.{ HttpCharsets, HttpEntity, StatusCodes }
+import akka.http.scaladsl.model.MediaTypes.`text/csv`
+import akka.stream.scaladsl.{ Sink, Source }
+import hmda.api.protocol.processing.ApiErrorProtocol
+import hmda.api.protocol.public.ULIProtocol
+import hmda.api.util.FlowUtils
 
 import scala.concurrent.ExecutionContext
+import scala.util.{ Failure, Success }
 
-trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives {
+trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives with ApiErrorProtocol with ULIProtocol with FlowUtils {
   implicit val system: ActorSystem
   implicit val materializer: ActorMaterializer
   implicit val timeout: Timeout
@@ -22,7 +33,108 @@ trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives {
       encodeResponse {
         pathPrefix("institutions" / Segment) { instId =>
           modifiedLar(instId)
-        }
+        } ~
+          pathPrefix("uli") {
+            path("checkDigit") {
+              timedPost { _ =>
+                entity(as[Loan]) { loan =>
+                  val loanId = loan.loanId
+                  val check = checkDigit(loanId)
+                  val uli = ULI(loanId, check.toInt, loanId + check)
+                  complete(ToResponseMarshallable(uli))
+                } ~
+                  fileUpload("file") {
+                    case (_, byteSource) =>
+                      val checkDigitF = processLoanIdFile(byteSource).runWith(Sink.seq)
+                      onComplete(checkDigitF) {
+                        case Success(checkDigits) => {
+                          complete(ToResponseMarshallable(LoanCheckDigitResponse(checkDigits)))
+                        }
+                        case Failure(error) =>
+                          log.error(error.getLocalizedMessage)
+                          complete(ToResponseMarshallable(StatusCodes.InternalServerError))
+                      }
+                    case _ =>
+                      complete(ToResponseMarshallable(StatusCodes.BadRequest))
+                  }
+              }
+            } ~
+              path("checkDigit" / "csv") {
+                timedPost { _ =>
+                  fileUpload("file") {
+                    case (_, byteSource) =>
+                      val headerSource = Source.fromIterator(() => List("loanId,checkDigit,uli\n").toIterator)
+                      val checkDigit = processLoanIdFile(byteSource)
+                        .map(l => l.toCSV)
+                        .map(l => l + "\n")
+                        .map(s => ByteString(s))
+
+                      val csv = headerSource.map(s => ByteString(s)).concat(checkDigit)
+                      complete(HttpEntity.Chunked.fromData(`text/csv`.toContentType(HttpCharsets.`UTF-8`), csv))
+
+                    case _ =>
+                      complete(ToResponseMarshallable(StatusCodes.BadRequest))
+                  }
+                }
+              } ~
+              path("validate") {
+                timedPost { _ =>
+                  entity(as[ULICheck]) { uc =>
+                    val uli = uc.uli
+                    val isValid = validateULI(uli)
+                    val validated = ULIValidated(isValid)
+                    complete(ToResponseMarshallable(validated))
+                  } ~
+                    fileUpload("file") {
+                      case (_, byteSource) =>
+                        val validatedF = processUliFile(byteSource).runWith(Sink.seq)
+                        onComplete(validatedF) {
+                          case Success(validated) =>
+                            complete(ToResponseMarshallable(ULIBatchValidatedResponse(validated)))
+                          case Failure(error) =>
+                            log.error(error.getLocalizedMessage)
+                            complete(ToResponseMarshallable(StatusCodes.InternalServerError))
+                        }
+
+                      case _ =>
+                        complete(ToResponseMarshallable(StatusCodes.BadRequest))
+                    }
+                }
+              } ~
+              path("validate" / "csv") {
+                timedPost { _ =>
+                  fileUpload("file") {
+                    case (_, byteSource) =>
+                      val headerSource = Source.fromIterator(() => List("uli,isValid\n").toIterator)
+                      val validated = processUliFile(byteSource)
+                        .map(u => u.toCSV)
+                        .map(l => l + "\n")
+                        .map(s => ByteString(s))
+
+                      val csv = headerSource.map(s => ByteString(s)).concat(validated)
+                      complete(HttpEntity.Chunked.fromData(`text/csv`.toContentType(HttpCharsets.`UTF-8`), csv))
+
+                    case _ =>
+                      complete(ToResponseMarshallable(StatusCodes.BadRequest))
+                  }
+                }
+              }
+          }
       }
     }
+
+  private def processLoanIdFile(byteSource: Source[ByteString, Any]) = {
+    byteSource
+      .via(framing)
+      .map(_.utf8String)
+      .map(loanId => ULI(loanId, checkDigit(loanId).toInt, loanId + checkDigit(loanId)))
+  }
+
+  private def processUliFile(byteSource: Source[ByteString, Any]) = {
+    byteSource
+      .via(framing)
+      .map(_.utf8String)
+      .map(uli => (uli, validateULI(uli)))
+      .map(validated => ULIBatchValidated(validated._1, validated._2))
+  }
 }

api/src/test/scala/hmda/api/http/FileUploadUtils.scala

@@ -0,0 +1,12 @@
+package hmda.api.http
+
+import akka.http.scaladsl.model.{ ContentTypes, HttpEntity, Multipart }
+
+trait FileUploadUtils {
+  def multiPartFile(contents: String, fileName: String) =
+    Multipart.FormData(Multipart.FormData.BodyPart.Strict(
+      "file",
+      HttpEntity(ContentTypes.`text/plain(UTF-8)`, contents),
+      Map("filename" -> fileName)
+    ))
+}

api/src/test/scala/hmda/api/http/InstitutionHttpSpec.scala

@@ -23,7 +23,7 @@ import scala.concurrent.duration._
 import akka.pattern.ask
 import scala.concurrent._
 
-trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with RequestHeaderUtils with InstitutionsHttpApi with ScalatestRouteTest { suite: Suite =>
+trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with RequestHeaderUtils with InstitutionsHttpApi with FileUploadUtils with ScalatestRouteTest { suite: Suite =>
   val configuration: Config = ConfigFactory.load()
 
   val validationStats = ValidationStats.createValidationStats(system)
@@ -55,11 +55,4 @@ trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with Reque
     FileUtils.deleteRecursively(snapshotStore)
   }
 
-  def multiPartFile(contents: String, fileName: String) =
-    Multipart.FormData(Multipart.FormData.BodyPart.Strict(
-      "file",
-      HttpEntity(ContentTypes.`text/plain(UTF-8)`, contents),
-      Map("filename" -> fileName)
-    ))
-
 }