From 76253b24aa0af4fd7175799a5824e973d986d171 Mon Sep 17 00:00:00 2001 From: Guillian DROUIN Date: Wed, 2 Aug 2023 14:54:27 +0200 Subject: [PATCH] Gestion de l'ocr sans widget --- example/lib/main.dart | 15 +- example/pubspec.lock | 20 +- lib/ocr_scan/helper/pdf_helper.dart | 2 +- lib/ocr_scan/model/shape/trapezoid.dart | 8 +- lib/ocr_scan/services/ocr_scan_service.dart | 231 ++++++++++++++++++++ lib/ocr_scan/widget/live_scan_widget.dart | 6 +- lib/ocr_scan/widget/scan_widget.dart | 98 ++------- lib/ocr_scan/widget/static_scan_widget.dart | 73 +------ lib/ocr_scan_text.dart | 1 + pubspec.yaml | 1 + 10 files changed, 293 insertions(+), 162 deletions(-) create mode 100644 lib/ocr_scan/services/ocr_scan_service.dart diff --git a/example/lib/main.dart b/example/lib/main.dart index 98fb026..1ff1f73 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,5 +1,6 @@ import 'package:flutter/material.dart'; import 'package:ocr_scan_text/ocr_scan_text.dart'; +import 'package:ocr_scan_text_example/scan_all_module.dart'; void main() { runApp(const MyApp()); @@ -31,13 +32,15 @@ class _MyAppState extends State { Widget _buildLiveScan() { return LiveScanWidget( - matchedResult: (ScanModule module, List scanResult) { - for (var element in scanResult) { - print("matchedResult : ${element.cleanedText}"); - // module.stop(); - } + ocrTextResult: (ocrTextResult) { + ocrTextResult.mapResult.forEach((module, result) { + for (var element in result) { + print("matchedResult : ${element.cleanedText}"); + // module.stop(); + } + }); }, - scanModules: [], + scanModules: [ScanAllModule()], ); } } diff --git a/example/pubspec.lock b/example/pubspec.lock index 31301a8..21ee9a3 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -133,10 +133,18 @@ packages: dependency: transitive description: name: ffi - sha256: ed5337a5660c506388a9f012be0288fb38b49020ce2b45fe1f8b8323fe429f99 + sha256: "13a6ccf6a459a125b3fcdb6ec73bd5ff90822e071207c663bfd1f70062d51d18" url: "https://pub.dev" source: hosted - version: "2.0.2" + version: "1.2.1" + file_picker: + dependency: transitive + description: + name: file_picker + sha256: "704259669b5e9cb24e15c11cfcf02caf5f20d30901b3916d60b6d1c2d647035f" + url: "https://pub.dev" + source: hosted + version: "4.6.1" flutter: dependency: "direct main" description: flutter @@ -291,10 +299,10 @@ packages: dependency: transitive description: name: path_provider_windows - sha256: "1cb68ba4cd3a795033de62ba1b7b4564dace301f952de6bfb3cd91b202b6ee96" + sha256: a34ecd7fb548f8e57321fd8e50d865d266941b54e6c3b7758cf8f37c24116905 url: "https://pub.dev" source: hosted - version: "2.1.7" + version: "2.0.7" pdf_render: dependency: transitive description: @@ -424,10 +432,10 @@ packages: dependency: transitive description: name: win32 - sha256: a6f0236dbda0f63aa9a25ad1ff9a9d8a4eaaa5012da0dc59d21afdb1dc361ca4 + sha256: c0e3a4f7be7dae51d8f152230b86627e3397c1ba8c3fa58e63d44a9f3edc9cef url: "https://pub.dev" source: hosted - version: "3.1.4" + version: "2.6.1" xdg_directories: dependency: transitive description: diff --git a/lib/ocr_scan/helper/pdf_helper.dart b/lib/ocr_scan/helper/pdf_helper.dart index 67bfa5d..54449f6 100644 --- a/lib/ocr_scan/helper/pdf_helper.dart +++ b/lib/ocr_scan/helper/pdf_helper.dart @@ -33,7 +33,7 @@ class PDFHelper { /// On prend que les 2 premiers page max, sinon c'est le bordel for (int i = 1; i <= min(2, document.pageCount); i++) { final page = await document.getPage(i); - int scaleUp = 5; // 4 is an arbitrary number, we enlarge the image to improve text detection + int scaleUp = 5; // 5 is an arbitrary number, we enlarge the image to improve text detection final pageImage = await page.render( width: page.width.toInt() * scaleUp, height: page.height.toInt() * scaleUp, diff --git a/lib/ocr_scan/model/shape/trapezoid.dart b/lib/ocr_scan/model/shape/trapezoid.dart index c441955..4631136 100644 --- a/lib/ocr_scan/model/shape/trapezoid.dart +++ b/lib/ocr_scan/model/shape/trapezoid.dart @@ -3,7 +3,7 @@ import 'dart:math'; import 'dart:ui'; import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; -import 'package:ocr_scan_text/ocr_scan/widget/scan_widget.dart'; +import 'package:ocr_scan_text/ocr_scan/services/ocr_scan_service.dart'; import '../recognizer_text/text_element.dart'; @@ -28,7 +28,7 @@ class Trapezoid { /// The cornersPoints, with Android, have positions that differ from the main axes /// X and Y are inverted and the 0 of the inverted axis is at imageSize.height /// Just with camera - if (Platform.isAndroid && ScanWidget.actualMode == Mode.camera) { + if (Platform.isAndroid && OcrScanService.actualMode == Mode.camera) { return Offset( imageSize.height - point.y.toDouble(), point.x.toDouble(), @@ -186,7 +186,7 @@ class Trapezoid { Size absoluteImageSize, double adjustTranslate, ) { - double denominator = Platform.isIOS || (Platform.isAndroid && ScanWidget.actualMode == Mode.static) + double denominator = Platform.isIOS || (Platform.isAndroid && OcrScanService.actualMode == Mode.static) ? absoluteImageSize.width : absoluteImageSize.height; switch (rotation) { @@ -206,7 +206,7 @@ class Trapezoid { Size absoluteImageSize, double adjustTranslate, ) { - double denominator = Platform.isIOS || (Platform.isAndroid && ScanWidget.actualMode == Mode.static) + double denominator = Platform.isIOS || (Platform.isAndroid && OcrScanService.actualMode == Mode.static) ? absoluteImageSize.height : absoluteImageSize.width; switch (rotation) { diff --git a/lib/ocr_scan/services/ocr_scan_service.dart b/lib/ocr_scan/services/ocr_scan_service.dart new file mode 100644 index 0000000..7d9e661 --- /dev/null +++ b/lib/ocr_scan/services/ocr_scan_service.dart @@ -0,0 +1,231 @@ +import 'dart:io'; +import 'dart:ui' as ui show Image; + +import 'package:file_picker/file_picker.dart'; +import 'package:flutter/material.dart'; +import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; +import 'package:image/image.dart' as img; +import 'package:path/path.dart' as path; +import 'package:pdf_render/pdf_render.dart'; + +import '../../ocr_scan_text.dart'; +import '../helper/pdf_helper.dart'; +import '../render/scan_renderer.dart'; + +enum Mode { + camera, + static, +} + +class OcrScanService { + static Mode _actualMode = Mode.camera; + static Mode get actualMode => _actualMode; + List scanModules; + + /// MLKit text detection object + final TextRecognizer textRecognizer = TextRecognizer(); + + OcrScanService( + this.scanModules, + ); + + Future startScanWithPhoto() async { + return _startStaticScanProcess( + await FilePicker.platform.pickFiles( + type: FileType.image, + allowCompression: false, + allowMultiple: false, + ), + ); + } + + Future startScanWithOpenFile() async { + return _startStaticScanProcess( + await FilePicker.platform.pickFiles( + type: FileType.custom, + allowCompression: false, + allowMultiple: false, + allowedExtensions: [ + 'png', + 'jpg', + 'pdf', + ], + ), + ); + } + + Future _startStaticScanProcess(FilePickerResult? result) async { + if (result == null) { + return null; + } + String? path = result.files.first.path; + if (path == null) { + return null; + } + return await startScanProcess( + File(path), + ); + } + + Future startScanProcess( + File file, + ) async { + String extension = path.extension(file.path).toLowerCase(); + + assert(extension == '.pdf' || extension == '.png' || extension == '.jpg' || extension == '.jpeg'); + if (extension == '.pdf') { + final PdfDocument document = await PdfDocument.openFile( + file.path, + ); + return await _processStaticPDF( + document, + scanModules, + ); + } else if (extension == '.png' || extension == '.jpg' || extension == '.jpeg') { + return await _processStaticImage( + file, + scanModules, + ); + } + return null; + } + +// Process image from camera stream + Future _processStaticPDF( + PdfDocument pdfDocument, + List scanModules, + ) async { + ImagePDF? imagePDF = await PDFHelper.convertToPDFImage(pdfDocument); + if (imagePDF == null) { + return null; + } + + ui.Image background = await decodeImageFromList(await imagePDF.file.readAsBytes()); + + return await processImage( + InputImage.fromFilePath(imagePDF.file.path), + Size( + background.width.toDouble(), + background.height.toDouble(), + ), + background, + Mode.static, + scanModules, + null, + ); + } + + Future _processStaticImage( + File file, + List scanModules, + ) async { + final cmd = img.Command()..decodeImageFile(file.path); + await cmd.executeThread(); + img.Image? image = cmd.outputImage; + if (image == null) { + return null; + } + ui.Image background = await decodeImageFromList(await file.readAsBytes()); + + return await processImage( + InputImage.fromFilePath(file.path), + Size( + image.width.toDouble(), + image.height.toDouble(), + ), + background, + Mode.static, + scanModules, + null, + ); + } + + /// Launch the search for results from the image for all the modules started + Future processImage( + InputImage inputImage, + Size imageSize, + ui.Image? background, + Mode mode, + List scanModules, + TextRecognizer? recognizer, + ) async { + _actualMode = mode; + TextRecognizer textRecognizer = recognizer ?? TextRecognizer(); + + /// Ask MLKit to return the list of TextBlocks in the image + final recognizedText = await textRecognizer.processImage(inputImage); + + /// create a global String corresponding to the texts found by MLKIt + String scannedText = ''; + List textBlocks = []; + for (final textBlock in recognizedText.blocks) { + for (final element in textBlock.lines) { + for (final textBlock in element.elements) { + textBlocks.add(textBlock); + scannedText += " ${textBlock.text}"; + } + } + } + + /// Start the text search for each module + Map> mapModule = >{}; + for (var scanModule in scanModules) { + if (!scanModule.started) { + continue; + } + + /// Generate the results of each module + List scanLines = await scanModule.generateResult( + recognizedText.blocks, + scannedText, + imageSize, + ); + + mapModule.putIfAbsent( + scanModule, + () => scanLines, + ); + } + + /// Create a ScanRenderer to display the visual rendering of the results found + var painter = ScanRenderer( + mapScanModules: mapModule, + imageRotation: inputImage.metadata?.rotation ?? InputImageRotation.rotation90deg, + imageSize: imageSize, + background: background, + ); + + Map> mapResult = >{}; + mapModule.forEach((key, matchCounterList) { + List list = matchCounterList + .where( + (matchCounter) => matchCounter.validated == true, + ) + .map((e) => e.scanResult) + .toList(); + + if (list.isNotEmpty) { + mapResult.putIfAbsent(key, () => list); + } + }); + + await textRecognizer.close(); + if (mapResult.isEmpty) { + return null; + } + + return OcrTextRecognizerResult( + CustomPaint( + painter: painter, + ), + mapResult, + ); + } +} + +class OcrTextRecognizerResult { + CustomPaint customPaint; + Map> mapResult; + + OcrTextRecognizerResult(this.customPaint, this.mapResult); +} diff --git a/lib/ocr_scan/widget/live_scan_widget.dart b/lib/ocr_scan/widget/live_scan_widget.dart index 920dfb3..0ce3089 100644 --- a/lib/ocr_scan/widget/live_scan_widget.dart +++ b/lib/ocr_scan/widget/live_scan_widget.dart @@ -9,12 +9,12 @@ import 'package:ocr_scan_text/ocr_scan/widget/scan_widget.dart'; /// Widget allowing "live" scanning using the camera class LiveScanWidget extends ScanWidget { - LiveScanWidget({ + const LiveScanWidget({ super.key, required super.scanModules, - required super.matchedResult, + required super.ocrTextResult, super.respectRatio = false, - }) : super(mode: Mode.camera); + }); @override LiveScanWidgetState createState() => LiveScanWidgetState(); diff --git a/lib/ocr_scan/widget/scan_widget.dart b/lib/ocr_scan/widget/scan_widget.dart index 05e3b5c..2dc2a2a 100644 --- a/lib/ocr_scan/widget/scan_widget.dart +++ b/lib/ocr_scan/widget/scan_widget.dart @@ -4,21 +4,10 @@ import 'dart:ui' as ui show Image; import 'package:flutter/material.dart'; import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; -import '../model/matched_counter.dart'; -import '../model/scan_result.dart'; import '../module/scan_module.dart'; -import '../render/scan_renderer.dart'; - -enum Mode { - camera, - static, -} +import '../services/ocr_scan_service.dart'; class ScanWidget extends StatefulWidget { - static Mode _actualMode = Mode.camera; - - static Mode get actualMode => _actualMode; - /// Respect the ratio of the camera for the display of the preview final bool respectRatio; @@ -26,17 +15,14 @@ class ScanWidget extends StatefulWidget { final List scanModules; /// Callback method returning the results found and validated - final Function(ScanModule module, List textBlockResult) matchedResult; + final Function(OcrTextRecognizerResult ocrTextResult) ocrTextResult; - ScanWidget({ + const ScanWidget({ Key? key, required this.scanModules, - required this.matchedResult, + required this.ocrTextResult, required this.respectRatio, - required Mode mode, - }) : super(key: key) { - ScanWidget._actualMode = mode; - } + }) : super(key: key); @override ScanWidgetState createState() => ScanWidgetState(); @@ -52,9 +38,12 @@ class ScanWidgetState extends State { /// Overlay on the image of the different areas of the results coming from the modules CustomPaint? customPaint; + late OcrScanService _ocrScanService; + @override void initState() { super.initState(); + _ocrScanService = OcrScanService(widget.scanModules); } @override @@ -72,68 +61,19 @@ class ScanWidgetState extends State { if (_isBusy) return; _isBusy = true; - /// Ask MLKit to return the list of TextBlocks in the image - final recognizedText = await _textRecognizer.processImage(inputImage); - - /// create a global String corresponding to the texts found by MLKIt - String scannedText = ''; - List textBlocks = []; - for (final textBlock in recognizedText.blocks) { - for (final element in textBlock.lines) { - for (final textBlock in element.elements) { - textBlocks.add(textBlock); - scannedText += " ${textBlock.text}"; - } - } - } - - /// Start the text search for each module - Map> mapModule = >{}; - for (var scanModule in widget.scanModules) { - if (!scanModule.started) { - continue; - } - - /// Generate the results of each module - List scanLines = await scanModule.generateResult( - recognizedText.blocks, - scannedText, - imageSize, - ); - - mapModule.putIfAbsent( - scanModule, - () => scanLines, - ); - } - - /// Create a ScanRenderer to display the visual rendering of the results found - var painter = ScanRenderer( - mapScanModules: mapModule, - imageRotation: inputImage.metadata?.rotation ?? InputImageRotation.rotation90deg, - imageSize: imageSize, - background: background, + OcrTextRecognizerResult? result = await _ocrScanService.processImage( + inputImage, + imageSize, + background, + Mode.camera, + widget.scanModules, + _textRecognizer, ); - /// Update the customPaint with the ScanRenderer - customPaint = CustomPaint(painter: painter); - - mapModule.forEach((key, matchCounterList) { - List list = matchCounterList - .where( - (matchCounter) => matchCounter.validated == true, - ) - .map((e) => e.scanResult) - .toList(); - - if (list.isNotEmpty) { - /// Return the list of validated results with CallBack method - widget.matchedResult( - key, - list, - ); - } - }); + if (result != null && result.mapResult.isNotEmpty) { + widget.ocrTextResult(result); + customPaint = result.customPaint; + } _isBusy = false; await _textRecognizer.close(); diff --git a/lib/ocr_scan/widget/static_scan_widget.dart b/lib/ocr_scan/widget/static_scan_widget.dart index fb8bb16..0b4eab5 100644 --- a/lib/ocr_scan/widget/static_scan_widget.dart +++ b/lib/ocr_scan/widget/static_scan_widget.dart @@ -1,23 +1,18 @@ import 'dart:io'; -import 'dart:ui' as ui show Image; import 'package:flutter/material.dart'; -import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; -import 'package:image/image.dart' as img; -import 'package:ocr_scan_text/ocr_scan/helper/pdf_helper.dart'; +import 'package:ocr_scan_text/ocr_scan/services/ocr_scan_service.dart'; import 'package:ocr_scan_text/ocr_scan/widget/scan_widget.dart'; -import 'package:path/path.dart' as path; -import 'package:pdf_render/pdf_render.dart'; class StaticScanWidget extends ScanWidget { final File file; - StaticScanWidget({ + const StaticScanWidget({ super.key, required super.scanModules, - required super.matchedResult, + required super.ocrTextResult, required this.file, super.respectRatio = false, - }) : super(mode: Mode.static); + }); @override StaticScanWidgetState createState() => StaticScanWidgetState(); @@ -26,24 +21,18 @@ class StaticScanWidget extends ScanWidget { class StaticScanWidgetState extends ScanWidgetState { @override void initState() { - // TODO: implement initState super.initState(); _startProcess(); } Future _startProcess() async { - String extension = path.extension(widget.file.path).toLowerCase(); - - assert(extension == '.pdf' || extension == '.png' || extension == '.jpg'); - - if (extension == '.pdf') { - final PdfDocument document = await PdfDocument.openFile( - widget.file.path, - ); - await _processStaticPDF(document); - } else if (extension == '.png' || extension == '.jpg') { - await _processStaticImage(widget.file); + OcrTextRecognizerResult? result = await OcrScanService(widget.scanModules).startScanProcess(widget.file); + if (result == null) { + return; } + customPaint = result.customPaint; + widget.ocrTextResult(result); + setState(() {}); } @override @@ -61,46 +50,4 @@ class StaticScanWidgetState extends ScanWidgetState { child: customPaint, ); } - - // Process image from camera stream - Future _processStaticPDF( - PdfDocument pdfDocument, - ) async { - ImagePDF? imagePDF = await PDFHelper.convertToPDFImage(pdfDocument); - if (imagePDF == null) { - return; - } - - ui.Image background = await decodeImageFromList(await imagePDF.file.readAsBytes()); - - await processImage( - InputImage.fromFilePath(imagePDF.file.path), - Size( - background.width.toDouble() ?? 0, - background.height.toDouble() ?? 0, - ), - background, - ); - setState(() {}); - } - - Future _processStaticImage(File file) async { - final cmd = img.Command()..decodeImageFile(file.path); - await cmd.executeThread(); - img.Image? image = cmd.outputImage; - if (image == null) { - return; - } - ui.Image background = await decodeImageFromList(await file.readAsBytes()); - - await processImage( - InputImage.fromFilePath(file.path), - Size( - image.width.toDouble(), - image.height.toDouble(), - ), - background, - ); - setState(() {}); - } } diff --git a/lib/ocr_scan_text.dart b/lib/ocr_scan_text.dart index b6e8931..ce0a923 100644 --- a/lib/ocr_scan_text.dart +++ b/lib/ocr_scan_text.dart @@ -6,4 +6,5 @@ export 'package:ocr_scan_text/ocr_scan/model/recognizer_text/text_line.dart'; export 'package:ocr_scan_text/ocr_scan/model/scan_result.dart'; export 'package:ocr_scan_text/ocr_scan/model/shape/trapezoid.dart'; export 'package:ocr_scan_text/ocr_scan/module/scan_module.dart'; +export 'package:ocr_scan_text/ocr_scan/services/ocr_scan_service.dart'; export 'package:ocr_scan_text/ocr_scan/widget/live_scan_widget.dart'; diff --git a/pubspec.yaml b/pubspec.yaml index aaac174..0e30353 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -15,6 +15,7 @@ dependencies: image: ^4.0.17 pdf_render: ^1.4.0 path_provider: ^2.0.15 + file_picker: 4.6.1 #dependency_overrides: # This is due to the fact that the package native_pdf_view use a version of the package