import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; import 'package:async/async.dart'; import 'package:camera/camera.dart'; import 'package:ffi/ffi.dart'; import 'package:flutter/material.dart'; import 'package:native_vision/native/dynamic_lib_singleton.dart'; import 'package:native_vision/edge_detection.dart'; import 'package:native_vision/native_vision_helper.dart'; import 'package:path_provider/path_provider.dart'; class NativeROI extends Struct { @Int32() external int left; @Int32() external int top; @Int32() external int width; @Int32() external int height; } class NativeCameraImage extends Struct { external Pointer buf; @Int32() external int height; @Int32() external int width; external Pointer format; @Int32() external int bytesPerRow; external Pointer roiPtr; static Pointer New(imgSizeInByte, String format) { final nativeCameraImgPtr = malloc.allocate(sizeOf()); final ref = nativeCameraImgPtr.ref; ref.buf = malloc.allocate(imgSizeInByte); ref.roiPtr = malloc.allocate(sizeOf()); ref.format = format.toNativeUtf8(); return nativeCameraImgPtr; } } extension CameraImageExtention on CameraImage { void fillNativeCameraImage(int left, int top, int cropW, int cropH, Pointer cameraImgPtr) { final imgSize = planes[0].bytes.length; cameraImgPtr.ref.height = height; cameraImgPtr.ref.width = width; final buf = cameraImgPtr.ref.buf; buf.asTypedList(imgSize).setRange(0, imgSize, planes[0].bytes); cameraImgPtr.ref.bytesPerRow = planes[0].bytesPerRow; final roiPtr = cameraImgPtr.ref.roiPtr; roiPtr.ref.left = left; roiPtr.ref.top = top; roiPtr.ref.width = cropW; roiPtr.ref.height = cropH; } } extension NativeCameraImageExtention on Pointer { void release() { malloc.free(ref.buf); malloc.free(ref.format); malloc.free(ref.roiPtr); malloc.free(this); } } class DocScanner { static final DynamicLibrary _docScan = DynamicLibSingleton().dl; static final int Function( Pointer docModelPath, Pointer cornerModelPath, double) _nativeDocFinder = _docScan .lookup< NativeFunction< IntPtr Function( Pointer, Pointer, Float)>>('initDocFinder') .asFunction(); static final void Function(int docFinderPtr) _nativeDeleteDocFinder = _docScan .lookup>('deleteDocFinder') .asFunction(); static final bool Function(Pointer, Pointer, int docFinderPtr, Pointer tmpDirPtr) _findDocWithCameraImg = _docScan .lookup< NativeFunction< Bool Function( Pointer coordsPtr, Pointer, IntPtr, Pointer)>>('findDocWithCameraImg') .asFunction(); static final bool Function(Pointer imgPathPtr, Pointer outPathPtr, int docFinderPtr, Pointer, Pointer) _findDocWithImgPath = _docScan .lookup< NativeFunction< Bool Function( Pointer, Pointer, IntPtr, Pointer, Pointer)>>("findDocWithImgPath") .asFunction(); SendPort? _sendport; StreamQueue? _events; bool _release = true; Pointer? _cameraImgPtr; Future init(double precise) async { final docModelPath = await NativeVisionHelper.getModelPath("doc_model_20220802.tflite"); final cornerModelPath = await NativeVisionHelper.getModelPath("corner_model_20220802.tflite"); final tmpDir = (await getTemporaryDirectory()).path; final p = ReceivePort(); Isolate.spawn(isolateHandler, { "sendPort": p.sendPort, "docModelPath": docModelPath, "cornerModelPath": cornerModelPath, "tmpDir": tmpDir, "precise": precise, }); _events = StreamQueue(p); _sendport = await _events?.next; _release = false; } Future?> scanWithCameraImage(CameraImage image, int left, int top, int cropW, int cropH, int width, int height) async { try { if (_release) throw "The DocScanner has release!"; _cameraImgPtr ??= NativeCameraImage.New( image.planes[0].bytes.length, image.format.group.name); image.fillNativeCameraImage(left, top, cropW, cropH, _cameraImgPtr!); _sendport!.send({ "data_source": "CameraImage", "CameraImagePtr": _cameraImgPtr!.address }); final msg = await _events?.next; if (_release) throw "The DocScanner has release!"; if (msg["succ"] == true) { Pointer coordArr = Pointer.fromAddress(msg["coordsPtr"]); final coords = List.generate( 4, (index) => Offset(coordArr.ref.coordsPtr[index].x * width, coordArr.ref.coordsPtr[index].y * height)); return coords; } } catch (e, st) { debugPrint("exception:${e.toString()}"); debugPrintStack(stackTrace: st); } return null; } Future singleScan( String imgPath, String outPath, ROI roi) async { Pointer? imgPathPtr; Pointer? roiPtr; Pointer? outPathPtr; try { if (_release) throw "The DocScanner has release!"; outPathPtr = outPath.toNativeUtf8(); imgPathPtr = imgPath.toNativeUtf8(); roiPtr = roi.toNativeROI(); _sendport!.send({ "data_source": "File", "imgPathPtr": imgPathPtr.address, "roiPtr": roiPtr.address, "outPathPtr": outPathPtr.address }); final msg = await _events?.next; if (_release) throw "The DocScanner has release!"; Pointer coordArr = Pointer.fromAddress(msg["coordsPtr"]); return coordArr.toEdgeDetectionResult(); } catch (e, st) { debugPrintStack(stackTrace: st); } finally { if (imgPathPtr != null) malloc.free(imgPathPtr); if (roiPtr != null) malloc.free(roiPtr); if (outPathPtr != null) malloc.free(outPathPtr); } return null; } bool available() { return !_release; } Future release() async { _release = true; _cameraImgPtr?.release(); _sendport?.send(null); await _events?.cancel(); debugPrint("DocScanner release!"); } static Future isolateHandler(dynamic args) async { final commandPort = ReceivePort(); SendPort p = args["sendPort"]; String docModelPath = args["docModelPath"]; String cornerModelPath = args["cornerModelPath"]; double precise = args["precise"]; final tmpDirPtr = (args["tmpDir"] as String).toNativeUtf8(); p.send(commandPort.sendPort); final docFinderPtr = await _initNativeDocFinder(docModelPath, cornerModelPath, precise); final coordsPtr = CoordinateArrayExtention.createCoordsPtr(); await for (final msg in commandPort) { if (msg == null) break; final type = msg["data_source"]; var succ = false; if (type == "CameraImage") { succ = _findDocWithCameraImg( coordsPtr, Pointer.fromAddress(msg["CameraImagePtr"]), docFinderPtr, tmpDirPtr); } else if (type == "File") { succ = _findDocWithImgPath( Pointer.fromAddress(msg["imgPathPtr"]), Pointer.fromAddress(msg["outPathPtr"]), docFinderPtr, coordsPtr, Pointer.fromAddress(msg["roiPtr"])); } p.send({"succ": succ, "coordsPtr": coordsPtr.address}); } // release res!!! _nativeDeleteDocFinder(docFinderPtr); malloc.free(tmpDirPtr); coordsPtr.release(); // debugPrint('DocScan isolate has finished.'); Isolate.exit(); } static Future _initNativeDocFinder( String docModelPath, String cornerModelPath, double precise) async { final nativeDocModelPath = docModelPath.toNativeUtf8(); final nativeCornerModelPath = cornerModelPath.toNativeUtf8(); final docFinderPtr = _nativeDocFinder(nativeDocModelPath, nativeCornerModelPath, precise); malloc.free(nativeDocModelPath); malloc.free(nativeCornerModelPath); return docFinderPtr; } }