ImageProcess/native_vision/lib/native/doc_scanner.dart

import 'dart:ffi';
import 'dart:io';
import 'dart:isolate';

import 'package:async/async.dart';
import 'package:camera/camera.dart';
import 'package:ffi/ffi.dart';
import 'package:flutter/material.dart';
import 'package:native_vision/native/dynamic_lib_singleton.dart';
import 'package:native_vision/edge_detection.dart';
import 'package:native_vision/native_vision_helper.dart';
import 'package:path_provider/path_provider.dart';

class NativeROI extends Struct {
  @Int32()
  external int left;
  @Int32()
  external int top;
  @Int32()
  external int width;
  @Int32()
  external int height;
}

class NativeCameraImage extends Struct {
  external Pointer<Uint8> buf;
  @Int32()
  external int height;
  @Int32()
  external int width;
  external Pointer<Utf8> format;
  @Int32()
  external int bytesPerRow;
  external Pointer<NativeROI> roiPtr;

  static Pointer<NativeCameraImage> New(imgSizeInByte, String format) {
    final nativeCameraImgPtr =
        malloc.allocate<NativeCameraImage>(sizeOf<NativeCameraImage>());
    final ref = nativeCameraImgPtr.ref;
    ref.buf = malloc.allocate<Uint8>(imgSizeInByte);
    ref.roiPtr = malloc.allocate<NativeROI>(sizeOf<NativeROI>());
    ref.format = format.toNativeUtf8();
    return nativeCameraImgPtr;
  }
}

extension CameraImageExtention on CameraImage {
  void fillNativeCameraImage(int left, int top, int cropW, int cropH,
      Pointer<NativeCameraImage> cameraImgPtr) {
    final imgSize = planes[0].bytes.length;

    cameraImgPtr.ref.height = height;
    cameraImgPtr.ref.width = width;

    final buf = cameraImgPtr.ref.buf;
    buf.asTypedList(imgSize).setRange(0, imgSize, planes[0].bytes);
    cameraImgPtr.ref.bytesPerRow = planes[0].bytesPerRow;

    final roiPtr = cameraImgPtr.ref.roiPtr;
    roiPtr.ref.left = left;
    roiPtr.ref.top = top;
    roiPtr.ref.width = cropW;
    roiPtr.ref.height = cropH;
  }
}

extension NativeCameraImageExtention on Pointer<NativeCameraImage> {
  void release() {
    malloc.free(ref.buf);
    malloc.free(ref.format);
    malloc.free(ref.roiPtr);
    malloc.free(this);
  }
}

class DocScanner {
  static final DynamicLibrary _docScan = DynamicLibSingleton().dl;

  static final int Function(
          Pointer<Utf8> docModelPath, Pointer<Utf8> cornerModelPath, double)
      _nativeDocFinder = _docScan
          .lookup<
              NativeFunction<
                  IntPtr Function(
                      Pointer<Utf8>, Pointer<Utf8>, Float)>>('initDocFinder')
          .asFunction();

  static final void Function(int docFinderPtr) _nativeDeleteDocFinder = _docScan
      .lookup<NativeFunction<Void Function(IntPtr)>>('deleteDocFinder')
      .asFunction();

  static final bool Function(Pointer<CoordinateArray>,
          Pointer<NativeCameraImage>, int docFinderPtr, Pointer<Utf8> tmpDirPtr)
      _findDocWithCameraImg = _docScan
          .lookup<
              NativeFunction<
                  Bool Function(
                      Pointer<CoordinateArray> coordsPtr,
                      Pointer<NativeCameraImage>,
                      IntPtr,
                      Pointer<Utf8>)>>('findDocWithCameraImg')
          .asFunction();
  static final bool Function(Pointer<Utf8> imgPathPtr, Pointer<Utf8> outPathPtr,
          int docFinderPtr, Pointer<CoordinateArray>, Pointer<NativeROI>)
      _findDocWithImgPath = _docScan
          .lookup<
              NativeFunction<
                  Bool Function(
                      Pointer<Utf8>,
                      Pointer<Utf8>,
                      IntPtr,
                      Pointer<CoordinateArray>,
                      Pointer<NativeROI>)>>("findDocWithImgPath")
          .asFunction();

  SendPort? _sendport;
  StreamQueue? _events;
  bool _release = true;
  Pointer<NativeCameraImage>? _cameraImgPtr;

  Future<void> init(double precise) async {
    final docModelPath =
        await NativeVisionHelper.getModelPath("doc_model_20220802.tflite");
    final cornerModelPath =
        await NativeVisionHelper.getModelPath("corner_model_20220802.tflite");
    final tmpDir = (await getTemporaryDirectory()).path;
    final p = ReceivePort();
    Isolate.spawn(isolateHandler, {
      "sendPort": p.sendPort,
      "docModelPath": docModelPath,
      "cornerModelPath": cornerModelPath,
      "tmpDir": tmpDir,
      "precise": precise,
    });
    _events = StreamQueue<dynamic>(p);
    _sendport = await _events?.next;
    _release = false;
  }

  Future<List<Offset>?> scanWithCameraImage(CameraImage image, int left,
      int top, int cropW, int cropH, int width, int height) async {
    try {
      if (_release) throw "The DocScanner has release!";
      _cameraImgPtr ??= NativeCameraImage.New(
          image.planes[0].bytes.length, image.format.group.name);
      image.fillNativeCameraImage(left, top, cropW, cropH, _cameraImgPtr!);
      _sendport!.send({
        "data_source": "CameraImage",
        "CameraImagePtr": _cameraImgPtr!.address
      });
      final msg = await _events?.next;
      if (_release) throw "The DocScanner has release!";
      if (msg["succ"] == true) {
        Pointer<CoordinateArray> coordArr =
            Pointer<CoordinateArray>.fromAddress(msg["coordsPtr"]);
        final coords = List.generate(
            4,
            (index) => Offset(coordArr.ref.coordsPtr[index].x * width,
                coordArr.ref.coordsPtr[index].y * height));
        return coords;
      }
    } catch (e, st) {
      debugPrint("exception:${e.toString()}");
      debugPrintStack(stackTrace: st);
    }
    return null;
  }

  Future<EdgeDetectionResult?> singleScan(
      String imgPath, String outPath, ROI roi) async {
    Pointer<Utf8>? imgPathPtr;
    Pointer<NativeROI>? roiPtr;
    Pointer<Utf8>? outPathPtr;
    try {
      if (_release) throw "The DocScanner has release!";
      outPathPtr = outPath.toNativeUtf8();
      imgPathPtr = imgPath.toNativeUtf8();
      roiPtr = roi.toNativeROI();
      _sendport!.send({
        "data_source": "File",
        "imgPathPtr": imgPathPtr.address,
        "roiPtr": roiPtr.address,
        "outPathPtr": outPathPtr.address
      });
      final msg = await _events?.next;
      if (_release) throw "The DocScanner has release!";
      Pointer<CoordinateArray> coordArr =
          Pointer<CoordinateArray>.fromAddress(msg["coordsPtr"]);
      return coordArr.toEdgeDetectionResult();
    } catch (e, st) {
      debugPrintStack(stackTrace: st);
    } finally {
      if (imgPathPtr != null) malloc.free(imgPathPtr);
      if (roiPtr != null) malloc.free(roiPtr);
      if (outPathPtr != null) malloc.free(outPathPtr);
    }
    return null;
  }

  bool available() {
    return !_release;
  }

  Future<void> release() async {
    _release = true;
    _cameraImgPtr?.release();
    _sendport?.send(null);
    await _events?.cancel();
    debugPrint("DocScanner release!");
  }

  static Future<void> isolateHandler(dynamic args) async {
    final commandPort = ReceivePort();
    SendPort p = args["sendPort"];
    String docModelPath = args["docModelPath"];
    String cornerModelPath = args["cornerModelPath"];
    double precise = args["precise"];
    final tmpDirPtr = (args["tmpDir"] as String).toNativeUtf8();
    p.send(commandPort.sendPort);
    final docFinderPtr =
        await _initNativeDocFinder(docModelPath, cornerModelPath, precise);
    final coordsPtr = CoordinateArrayExtention.createCoordsPtr();
    await for (final msg in commandPort) {
      if (msg == null) break;
      final type = msg["data_source"];
      var succ = false;
      if (type == "CameraImage") {
        succ = _findDocWithCameraImg(
            coordsPtr,
            Pointer<NativeCameraImage>.fromAddress(msg["CameraImagePtr"]),
            docFinderPtr,
            tmpDirPtr);
      } else if (type == "File") {
        succ = _findDocWithImgPath(
            Pointer<Utf8>.fromAddress(msg["imgPathPtr"]),
            Pointer<Utf8>.fromAddress(msg["outPathPtr"]),
            docFinderPtr,
            coordsPtr,
            Pointer<NativeROI>.fromAddress(msg["roiPtr"]));
      }
      p.send({"succ": succ, "coordsPtr": coordsPtr.address});
    }
    // release res!!!
    _nativeDeleteDocFinder(docFinderPtr);
    malloc.free(tmpDirPtr);
    coordsPtr.release();
    // debugPrint('DocScan isolate has finished.');
    Isolate.exit();
  }

  static Future<int> _initNativeDocFinder(
      String docModelPath, String cornerModelPath, double precise) async {
    final nativeDocModelPath = docModelPath.toNativeUtf8();
    final nativeCornerModelPath = cornerModelPath.toNativeUtf8();
    final docFinderPtr =
        _nativeDocFinder(nativeDocModelPath, nativeCornerModelPath, precise);
    malloc.free(nativeDocModelPath);
    malloc.free(nativeCornerModelPath);
    return docFinderPtr;
  }
}