dnn.hpp 84 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759
  1. /*M///////////////////////////////////////////////////////////////////////////////////////
  2. //
  3. // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4. //
  5. // By downloading, copying, installing or using the software you agree to this license.
  6. // If you do not agree to this license, do not download, install,
  7. // copy or use the software.
  8. //
  9. //
  10. // License Agreement
  11. // For Open Source Computer Vision Library
  12. //
  13. // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14. // Third party copyrights are property of their respective owners.
  15. //
  16. // Redistribution and use in source and binary forms, with or without modification,
  17. // are permitted provided that the following conditions are met:
  18. //
  19. // * Redistribution's of source code must retain the above copyright notice,
  20. // this list of conditions and the following disclaimer.
  21. //
  22. // * Redistribution's in binary form must reproduce the above copyright notice,
  23. // this list of conditions and the following disclaimer in the documentation
  24. // and/or other materials provided with the distribution.
  25. //
  26. // * The name of the copyright holders may not be used to endorse or promote products
  27. // derived from this software without specific prior written permission.
  28. //
  29. // This software is provided by the copyright holders and contributors "as is" and
  30. // any express or implied warranties, including, but not limited to, the implied
  31. // warranties of merchantability and fitness for a particular purpose are disclaimed.
  32. // In no event shall the Intel Corporation or contributors be liable for any direct,
  33. // indirect, incidental, special, exemplary, or consequential damages
  34. // (including, but not limited to, procurement of substitute goods or services;
  35. // loss of use, data, or profits; or business interruption) however caused
  36. // and on any theory of liability, whether in contract, strict liability,
  37. // or tort (including negligence or otherwise) arising in any way out of
  38. // the use of this software, even if advised of the possibility of such damage.
  39. //
  40. //M*/
  41. #ifndef OPENCV_DNN_DNN_HPP
  42. #define OPENCV_DNN_DNN_HPP
  43. #include <vector>
  44. #include <opencv2/core.hpp>
  45. #include "opencv2/core/async.hpp"
  46. #include "../dnn/version.hpp"
  47. #include <opencv2/dnn/dict.hpp>
  48. namespace cv {
  49. namespace dnn {
  50. CV__DNN_INLINE_NS_BEGIN
  51. //! @addtogroup dnn
  52. //! @{
  53. typedef std::vector<int> MatShape;
  54. /**
  55. * @brief Enum of computation backends supported by layers.
  56. * @see Net::setPreferableBackend
  57. */
  58. enum Backend
  59. {
  60. //! DNN_BACKEND_DEFAULT equals to DNN_BACKEND_INFERENCE_ENGINE if
  61. //! OpenCV is built with Intel's Inference Engine library or
  62. //! DNN_BACKEND_OPENCV otherwise.
  63. DNN_BACKEND_DEFAULT = 0,
  64. DNN_BACKEND_HALIDE,
  65. DNN_BACKEND_INFERENCE_ENGINE, //!< Intel's Inference Engine computational backend
  66. //!< @sa setInferenceEngineBackendType
  67. DNN_BACKEND_OPENCV,
  68. DNN_BACKEND_VKCOM,
  69. DNN_BACKEND_CUDA,
  70. DNN_BACKEND_WEBNN,
  71. DNN_BACKEND_TIMVX,
  72. #ifdef __OPENCV_BUILD
  73. DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
  74. DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
  75. #endif
  76. };
  77. /**
  78. * @brief Enum of target devices for computations.
  79. * @see Net::setPreferableTarget
  80. */
  81. enum Target
  82. {
  83. DNN_TARGET_CPU = 0,
  84. DNN_TARGET_OPENCL,
  85. DNN_TARGET_OPENCL_FP16,
  86. DNN_TARGET_MYRIAD,
  87. DNN_TARGET_VULKAN,
  88. DNN_TARGET_FPGA, //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin.
  89. DNN_TARGET_CUDA,
  90. DNN_TARGET_CUDA_FP16,
  91. DNN_TARGET_HDDL,
  92. DNN_TARGET_NPU,
  93. };
  94. CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends();
  95. CV_EXPORTS_W std::vector<Target> getAvailableTargets(dnn::Backend be);
  96. /**
  97. * @brief Enables detailed logging of the DNN model loading with CV DNN API.
  98. * @param[in] isDiagnosticsMode Indicates whether diagnostic mode should be set.
  99. *
  100. * Diagnostic mode provides detailed logging of the model loading stage to explore
  101. * potential problems (ex.: not implemented layer type).
  102. *
  103. * @note In diagnostic mode series of assertions will be skipped, it can lead to the
  104. * expected application crash.
  105. */
  106. CV_EXPORTS void enableModelDiagnostics(bool isDiagnosticsMode);
  107. /** @brief This class provides all data needed to initialize layer.
  108. *
  109. * It includes dictionary with scalar params (which can be read by using Dict interface),
  110. * blob params #blobs and optional meta information: #name and #type of layer instance.
  111. */
  112. class CV_EXPORTS LayerParams : public Dict
  113. {
  114. public:
  115. //TODO: Add ability to name blob params
  116. std::vector<Mat> blobs; //!< List of learned parameters stored as blobs.
  117. String name; //!< Name of the layer instance (optional, can be used internal purposes).
  118. String type; //!< Type name which was used for creating layer by layer factory (optional).
  119. };
  120. /**
  121. * @brief Derivatives of this class encapsulates functions of certain backends.
  122. */
  123. class BackendNode
  124. {
  125. public:
  126. explicit BackendNode(int backendId);
  127. virtual ~BackendNode(); //!< Virtual destructor to make polymorphism.
  128. int backendId; //!< Backend identifier.
  129. };
  130. /**
  131. * @brief Derivatives of this class wraps cv::Mat for different backends and targets.
  132. */
  133. class BackendWrapper
  134. {
  135. public:
  136. BackendWrapper(int backendId, int targetId);
  137. /**
  138. * @brief Wrap cv::Mat for specific backend and target.
  139. * @param[in] targetId Target identifier.
  140. * @param[in] m cv::Mat for wrapping.
  141. *
  142. * Make CPU->GPU data transfer if it's require for the target.
  143. */
  144. BackendWrapper(int targetId, const cv::Mat& m);
  145. /**
  146. * @brief Make wrapper for reused cv::Mat.
  147. * @param[in] base Wrapper of cv::Mat that will be reused.
  148. * @param[in] shape Specific shape.
  149. *
  150. * Initialize wrapper from another one. It'll wrap the same host CPU
  151. * memory and mustn't allocate memory on device(i.e. GPU). It might
  152. * has different shape. Use in case of CPU memory reusing for reuse
  153. * associated memory on device too.
  154. */
  155. BackendWrapper(const Ptr<BackendWrapper>& base, const MatShape& shape);
  156. virtual ~BackendWrapper(); //!< Virtual destructor to make polymorphism.
  157. /**
  158. * @brief Transfer data to CPU host memory.
  159. */
  160. virtual void copyToHost() = 0;
  161. /**
  162. * @brief Indicate that an actual data is on CPU.
  163. */
  164. virtual void setHostDirty() = 0;
  165. int backendId; //!< Backend identifier.
  166. int targetId; //!< Target identifier.
  167. };
  168. class CV_EXPORTS ActivationLayer;
  169. /** @brief This interface class allows to build new Layers - are building blocks of networks.
  170. *
  171. * Each class, derived from Layer, must implement allocate() methods to declare own outputs and forward() to compute outputs.
  172. * Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros.
  173. */
  174. class CV_EXPORTS_W Layer : public Algorithm
  175. {
  176. public:
  177. //! List of learned parameters must be stored here to allow read them by using Net::getParam().
  178. CV_PROP_RW std::vector<Mat> blobs;
  179. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  180. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  181. * @param[in] input vector of already allocated input blobs
  182. * @param[out] output vector of already allocated output blobs
  183. *
  184. * If this method is called after network has allocated all memory for input and output blobs
  185. * and before inferencing.
  186. */
  187. CV_DEPRECATED_EXTERNAL
  188. virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output);
  189. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  190. * @param[in] inputs vector of already allocated input blobs
  191. * @param[out] outputs vector of already allocated output blobs
  192. *
  193. * If this method is called after network has allocated all memory for input and output blobs
  194. * and before inferencing.
  195. */
  196. CV_WRAP virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs);
  197. /** @brief Given the @p input blobs, computes the output @p blobs.
  198. * @deprecated Use Layer::forward(InputArrayOfArrays, OutputArrayOfArrays, OutputArrayOfArrays) instead
  199. * @param[in] input the input blobs.
  200. * @param[out] output allocated output blobs, which will store results of the computation.
  201. * @param[out] internals allocated internal blobs
  202. */
  203. CV_DEPRECATED_EXTERNAL
  204. virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals);
  205. /** @brief Given the @p input blobs, computes the output @p blobs.
  206. * @param[in] inputs the input blobs.
  207. * @param[out] outputs allocated output blobs, which will store results of the computation.
  208. * @param[out] internals allocated internal blobs
  209. */
  210. virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
  211. /** @brief Tries to quantize the given layer and compute the quantization parameters required for fixed point implementation.
  212. * @param[in] scales input and output scales.
  213. * @param[in] zeropoints input and output zeropoints.
  214. * @param[out] params Quantized parameters required for fixed point implementation of that layer.
  215. * @returns True if layer can be quantized.
  216. */
  217. virtual bool tryQuantize(const std::vector<std::vector<float> > &scales,
  218. const std::vector<std::vector<int> > &zeropoints, LayerParams& params);
  219. /** @brief Given the @p input blobs, computes the output @p blobs.
  220. * @param[in] inputs the input blobs.
  221. * @param[out] outputs allocated output blobs, which will store results of the computation.
  222. * @param[out] internals allocated internal blobs
  223. */
  224. void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
  225. /** @brief
  226. * @overload
  227. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  228. */
  229. CV_DEPRECATED_EXTERNAL
  230. void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs);
  231. /** @brief
  232. * @overload
  233. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  234. */
  235. CV_DEPRECATED std::vector<Mat> finalize(const std::vector<Mat> &inputs);
  236. /** @brief Allocates layer and computes output.
  237. * @deprecated This method will be removed in the future release.
  238. */
  239. CV_DEPRECATED CV_WRAP void run(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs,
  240. CV_IN_OUT std::vector<Mat> &internals);
  241. /** @brief Returns index of input blob into the input array.
  242. * @param inputName label of input blob
  243. *
  244. * Each layer input and output can be labeled to easily identify them using "%<layer_name%>[.output_name]" notation.
  245. * This method maps label of input blob to its index into input vector.
  246. */
  247. virtual int inputNameToIndex(String inputName); // FIXIT const
  248. /** @brief Returns index of output blob in output array.
  249. * @see inputNameToIndex()
  250. */
  251. CV_WRAP virtual int outputNameToIndex(const String& outputName); // FIXIT const
  252. /**
  253. * @brief Ask layer if it support specific backend for doing computations.
  254. * @param[in] backendId computation backend identifier.
  255. * @see Backend
  256. */
  257. virtual bool supportBackend(int backendId); // FIXIT const
  258. /**
  259. * @brief Returns Halide backend node.
  260. * @param[in] inputs Input Halide buffers.
  261. * @see BackendNode, BackendWrapper
  262. *
  263. * Input buffers should be exactly the same that will be used in forward invocations.
  264. * Despite we can use Halide::ImageParam based on input shape only,
  265. * it helps prevent some memory management issues (if something wrong,
  266. * Halide tests will be failed).
  267. */
  268. virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs);
  269. virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
  270. virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs);
  271. virtual Ptr<BackendNode> initWebnn(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
  272. /**
  273. * @brief Returns a CUDA backend node
  274. *
  275. * @param context void pointer to CSLContext object
  276. * @param inputs layer inputs
  277. * @param outputs layer outputs
  278. */
  279. virtual Ptr<BackendNode> initCUDA(
  280. void *context,
  281. const std::vector<Ptr<BackendWrapper>>& inputs,
  282. const std::vector<Ptr<BackendWrapper>>& outputs
  283. );
  284. /**
  285. * @brief Returns a TimVX backend node
  286. *
  287. * @param timVxInfo void pointer to CSLContext object
  288. * @param inputsWrapper layer inputs
  289. * @param outputsWrapper layer outputs
  290. * @param isLast if the node is the last one of the TimVX Graph.
  291. */
  292. virtual Ptr<BackendNode> initTimVX(void* timVxInfo,
  293. const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
  294. const std::vector<Ptr<BackendWrapper> > &outputsWrapper,
  295. bool isLast);
  296. /**
  297. * @brief Automatic Halide scheduling based on layer hyper-parameters.
  298. * @param[in] node Backend node with Halide functions.
  299. * @param[in] inputs Blobs that will be used in forward invocations.
  300. * @param[in] outputs Blobs that will be used in forward invocations.
  301. * @param[in] targetId Target identifier
  302. * @see BackendNode, Target
  303. *
  304. * Layer don't use own Halide::Func members because we can have applied
  305. * layers fusing. In this way the fused function should be scheduled.
  306. */
  307. virtual void applyHalideScheduler(Ptr<BackendNode>& node,
  308. const std::vector<Mat*> &inputs,
  309. const std::vector<Mat> &outputs,
  310. int targetId) const;
  311. /**
  312. * @brief Implement layers fusing.
  313. * @param[in] node Backend node of bottom layer.
  314. * @see BackendNode
  315. *
  316. * Actual for graph-based backends. If layer attached successfully,
  317. * returns non-empty cv::Ptr to node of the same backend.
  318. * Fuse only over the last function.
  319. */
  320. virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node);
  321. /**
  322. * @brief Tries to attach to the layer the subsequent activation layer, i.e. do the layer fusion in a partial case.
  323. * @param[in] layer The subsequent activation layer.
  324. *
  325. * Returns true if the activation layer has been attached successfully.
  326. */
  327. virtual bool setActivation(const Ptr<ActivationLayer>& layer);
  328. /**
  329. * @brief Try to fuse current layer with a next one
  330. * @param[in] top Next layer to be fused.
  331. * @returns True if fusion was performed.
  332. */
  333. virtual bool tryFuse(Ptr<Layer>& top);
  334. /**
  335. * @brief Returns parameters of layers with channel-wise multiplication and addition.
  336. * @param[out] scale Channel-wise multipliers. Total number of values should
  337. * be equal to number of channels.
  338. * @param[out] shift Channel-wise offsets. Total number of values should
  339. * be equal to number of channels.
  340. *
  341. * Some layers can fuse their transformations with further layers.
  342. * In example, convolution + batch normalization. This way base layer
  343. * use weights from layer after it. Fused layer is skipped.
  344. * By default, @p scale and @p shift are empty that means layer has no
  345. * element-wise multiplications or additions.
  346. */
  347. virtual void getScaleShift(Mat& scale, Mat& shift) const;
  348. /**
  349. * @brief Returns scale and zeropoint of layers
  350. * @param[out] scale Output scale
  351. * @param[out] zeropoint Output zeropoint
  352. *
  353. * By default, @p scale is 1 and @p zeropoint is 0.
  354. */
  355. virtual void getScaleZeropoint(float& scale, int& zeropoint) const;
  356. /**
  357. * @brief "Detaches" all the layers, attached to particular layer.
  358. */
  359. virtual void unsetAttached();
  360. virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
  361. const int requiredOutputs,
  362. std::vector<MatShape> &outputs,
  363. std::vector<MatShape> &internals) const;
  364. virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
  365. const std::vector<MatShape> &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;}
  366. virtual bool updateMemoryShapes(const std::vector<MatShape> &inputs);
  367. CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes.
  368. CV_PROP String type; //!< Type name which was used for creating layer by layer factory.
  369. CV_PROP int preferableTarget; //!< prefer target for layer forwarding
  370. Layer();
  371. explicit Layer(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  372. void setParamsFrom(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  373. virtual ~Layer();
  374. };
  375. /** @brief This class allows to create and manipulate comprehensive artificial neural networks.
  376. *
  377. * Neural network is presented as directed acyclic graph (DAG), where vertices are Layer instances,
  378. * and edges specify relationships between layers inputs and outputs.
  379. *
  380. * Each network layer has unique integer id and unique string name inside its network.
  381. * LayerId can store either layer name or layer id.
  382. *
  383. * This class supports reference counting of its instances, i. e. copies point to the same instance.
  384. */
  385. class CV_EXPORTS_W_SIMPLE Net
  386. {
  387. public:
  388. CV_WRAP Net(); //!< Default constructor.
  389. CV_WRAP ~Net(); //!< Destructor frees the net only if there aren't references to the net anymore.
  390. /** @brief Create a network from Intel's Model Optimizer intermediate representation (IR).
  391. * @param[in] xml XML configuration file with network's topology.
  392. * @param[in] bin Binary file with trained weights.
  393. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  394. * backend.
  395. */
  396. CV_WRAP static Net readFromModelOptimizer(const String& xml, const String& bin);
  397. /** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
  398. * @param[in] bufferModelConfig buffer with model's configuration.
  399. * @param[in] bufferWeights buffer with model's trained weights.
  400. * @returns Net object.
  401. */
  402. CV_WRAP static
  403. Net readFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
  404. /** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
  405. * @param[in] bufferModelConfigPtr buffer pointer of model's configuration.
  406. * @param[in] bufferModelConfigSize buffer size of model's configuration.
  407. * @param[in] bufferWeightsPtr buffer pointer of model's trained weights.
  408. * @param[in] bufferWeightsSize buffer size of model's trained weights.
  409. * @returns Net object.
  410. */
  411. static
  412. Net readFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
  413. const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
  414. /** Returns true if there are no layers in the network. */
  415. CV_WRAP bool empty() const;
  416. /** @brief Dump net to String
  417. * @returns String with structure, hyperparameters, backend, target and fusion
  418. * Call method after setInput(). To see correct backend, target and fusion run after forward().
  419. */
  420. CV_WRAP String dump();
  421. /** @brief Dump net structure, hyperparameters, backend, target and fusion to dot file
  422. * @param path path to output file with .dot extension
  423. * @see dump()
  424. */
  425. CV_WRAP void dumpToFile(const String& path);
  426. /** @brief Adds new layer to the net.
  427. * @param name unique name of the adding layer.
  428. * @param type typename of the adding layer (type must be registered in LayerRegister).
  429. * @param dtype datatype of output blobs.
  430. * @param params parameters which will be used to initialize the creating layer.
  431. * @returns unique identifier of created layer, or -1 if a failure will happen.
  432. */
  433. int addLayer(const String &name, const String &type, const int &dtype, LayerParams &params);
  434. /** @overload Datatype of output blobs set to default CV_32F */
  435. int addLayer(const String &name, const String &type, LayerParams &params);
  436. /** @brief Adds new layer and connects its first input to the first output of previously added layer.
  437. * @see addLayer()
  438. */
  439. int addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams &params);
  440. /** @overload */
  441. int addLayerToPrev(const String &name, const String &type, LayerParams &params);
  442. /** @brief Converts string name of the layer to the integer identifier.
  443. * @returns id of the layer, or -1 if the layer wasn't found.
  444. */
  445. CV_WRAP int getLayerId(const String &layer) const;
  446. CV_WRAP std::vector<String> getLayerNames() const;
  447. /** @brief Container for strings and integers.
  448. *
  449. * @deprecated Use getLayerId() with int result.
  450. */
  451. typedef DictValue LayerId;
  452. /** @brief Returns pointer to layer with specified id or name which the network use. */
  453. CV_WRAP Ptr<Layer> getLayer(int layerId) const;
  454. /** @overload
  455. * @deprecated Use int getLayerId(const String &layer)
  456. */
  457. CV_WRAP inline Ptr<Layer> getLayer(const String& layerName) const { return getLayer(getLayerId(layerName)); }
  458. /** @overload
  459. * @deprecated to be removed
  460. */
  461. CV_WRAP Ptr<Layer> getLayer(const LayerId& layerId) const;
  462. /** @brief Returns pointers to input layers of specific layer. */
  463. std::vector<Ptr<Layer> > getLayerInputs(int layerId) const; // FIXIT: CV_WRAP
  464. /** @brief Connects output of the first layer to input of the second layer.
  465. * @param outPin descriptor of the first layer output.
  466. * @param inpPin descriptor of the second layer input.
  467. *
  468. * Descriptors have the following template <DFN>&lt;layer_name&gt;[.input_number]</DFN>:
  469. * - the first part of the template <DFN>layer_name</DFN> is string name of the added layer.
  470. * If this part is empty then the network input pseudo layer will be used;
  471. * - the second optional part of the template <DFN>input_number</DFN>
  472. * is either number of the layer input, either label one.
  473. * If this part is omitted then the first layer input will be used.
  474. *
  475. * @see setNetInputs(), Layer::inputNameToIndex(), Layer::outputNameToIndex()
  476. */
  477. CV_WRAP void connect(String outPin, String inpPin);
  478. /** @brief Connects #@p outNum output of the first layer to #@p inNum input of the second layer.
  479. * @param outLayerId identifier of the first layer
  480. * @param outNum number of the first layer output
  481. * @param inpLayerId identifier of the second layer
  482. * @param inpNum number of the second layer input
  483. */
  484. void connect(int outLayerId, int outNum, int inpLayerId, int inpNum);
  485. /** @brief Registers network output with name
  486. *
  487. * Function may create additional 'Identity' layer.
  488. *
  489. * @param outputName identifier of the output
  490. * @param layerId identifier of the second layer
  491. * @param outputPort number of the second layer input
  492. *
  493. * @returns index of bound layer (the same as layerId or newly created)
  494. */
  495. int registerOutput(const std::string& outputName, int layerId, int outputPort);
  496. /** @brief Sets outputs names of the network input pseudo layer.
  497. *
  498. * Each net always has special own the network input pseudo layer with id=0.
  499. * This layer stores the user blobs only and don't make any computations.
  500. * In fact, this layer provides the only way to pass user data into the network.
  501. * As any other layer, this layer can label its outputs and this function provides an easy way to do this.
  502. */
  503. CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames);
  504. /** @brief Specify shape of network input.
  505. */
  506. CV_WRAP void setInputShape(const String &inputName, const MatShape& shape);
  507. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  508. * @param outputName name for layer which output is needed to get
  509. * @return blob for first output of specified layer.
  510. * @details By default runs forward pass for the whole network.
  511. */
  512. CV_WRAP Mat forward(const String& outputName = String());
  513. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  514. * @param outputName name for layer which output is needed to get
  515. * @details By default runs forward pass for the whole network.
  516. *
  517. * This is an asynchronous version of forward(const String&).
  518. * dnn::DNN_BACKEND_INFERENCE_ENGINE backend is required.
  519. */
  520. CV_WRAP AsyncArray forwardAsync(const String& outputName = String());
  521. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  522. * @param outputBlobs contains all output blobs for specified layer.
  523. * @param outputName name for layer which output is needed to get
  524. * @details If @p outputName is empty, runs forward pass for the whole network.
  525. */
  526. CV_WRAP void forward(OutputArrayOfArrays outputBlobs, const String& outputName = String());
  527. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  528. * @param outputBlobs contains blobs for first outputs of specified layers.
  529. * @param outBlobNames names for layers which outputs are needed to get
  530. */
  531. CV_WRAP void forward(OutputArrayOfArrays outputBlobs,
  532. const std::vector<String>& outBlobNames);
  533. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  534. * @param outputBlobs contains all output blobs for each layer specified in @p outBlobNames.
  535. * @param outBlobNames names for layers which outputs are needed to get
  536. */
  537. CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector<std::vector<Mat> >& outputBlobs,
  538. const std::vector<String>& outBlobNames);
  539. /** @brief Returns a quantized Net from a floating-point Net.
  540. * @param calibData Calibration data to compute the quantization parameters.
  541. * @param inputsDtype Datatype of quantized net's inputs. Can be CV_32F or CV_8S.
  542. * @param outputsDtype Datatype of quantized net's outputs. Can be CV_32F or CV_8S.
  543. */
  544. CV_WRAP Net quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype);
  545. /** @brief Returns input scale and zeropoint for a quantized Net.
  546. * @param scales output parameter for returning input scales.
  547. * @param zeropoints output parameter for returning input zeropoints.
  548. */
  549. CV_WRAP void getInputDetails(CV_OUT std::vector<float>& scales, CV_OUT std::vector<int>& zeropoints) const;
  550. /** @brief Returns output scale and zeropoint for a quantized Net.
  551. * @param scales output parameter for returning output scales.
  552. * @param zeropoints output parameter for returning output zeropoints.
  553. */
  554. CV_WRAP void getOutputDetails(CV_OUT std::vector<float>& scales, CV_OUT std::vector<int>& zeropoints) const;
  555. /**
  556. * @brief Compile Halide layers.
  557. * @param[in] scheduler Path to YAML file with scheduling directives.
  558. * @see setPreferableBackend
  559. *
  560. * Schedule layers that support Halide backend. Then compile them for
  561. * specific target. For layers that not represented in scheduling file
  562. * or if no manual scheduling used at all, automatic scheduling will be applied.
  563. */
  564. CV_WRAP void setHalideScheduler(const String& scheduler);
  565. /**
  566. * @brief Ask network to use specific computation backend where it supported.
  567. * @param[in] backendId backend identifier.
  568. * @see Backend
  569. *
  570. * If OpenCV is compiled with Intel's Inference Engine library, DNN_BACKEND_DEFAULT
  571. * means DNN_BACKEND_INFERENCE_ENGINE. Otherwise it equals to DNN_BACKEND_OPENCV.
  572. */
  573. CV_WRAP void setPreferableBackend(int backendId);
  574. /**
  575. * @brief Ask network to make computations on specific target device.
  576. * @param[in] targetId target identifier.
  577. * @see Target
  578. *
  579. * List of supported combinations backend / target:
  580. * | | DNN_BACKEND_OPENCV | DNN_BACKEND_INFERENCE_ENGINE | DNN_BACKEND_HALIDE | DNN_BACKEND_CUDA |
  581. * |------------------------|--------------------|------------------------------|--------------------|-------------------|
  582. * | DNN_TARGET_CPU | + | + | + | |
  583. * | DNN_TARGET_OPENCL | + | + | + | |
  584. * | DNN_TARGET_OPENCL_FP16 | + | + | | |
  585. * | DNN_TARGET_MYRIAD | | + | | |
  586. * | DNN_TARGET_FPGA | | + | | |
  587. * | DNN_TARGET_CUDA | | | | + |
  588. * | DNN_TARGET_CUDA_FP16 | | | | + |
  589. * | DNN_TARGET_HDDL | | + | | |
  590. */
  591. CV_WRAP void setPreferableTarget(int targetId);
  592. /** @brief Sets the new input value for the network
  593. * @param blob A new blob. Should have CV_32F or CV_8U depth.
  594. * @param name A name of input layer.
  595. * @param scalefactor An optional normalization scale.
  596. * @param mean An optional mean subtraction values.
  597. * @see connect(String, String) to know format of the descriptor.
  598. *
  599. * If scale or mean values are specified, a final input blob is computed
  600. * as:
  601. * \f[input(n,c,h,w) = scalefactor \times (blob(n,c,h,w) - mean_c)\f]
  602. */
  603. CV_WRAP void setInput(InputArray blob, const String& name = "",
  604. double scalefactor = 1.0, const Scalar& mean = Scalar());
  605. /** @brief Sets the new value for the learned param of the layer.
  606. * @param layer name or id of the layer.
  607. * @param numParam index of the layer parameter in the Layer::blobs array.
  608. * @param blob the new value.
  609. * @see Layer::blobs
  610. * @note If shape of the new blob differs from the previous shape,
  611. * then the following forward pass may fail.
  612. */
  613. CV_WRAP void setParam(int layer, int numParam, const Mat &blob);
  614. CV_WRAP inline void setParam(const String& layerName, int numParam, const Mat &blob) { return setParam(getLayerId(layerName), numParam, blob); }
  615. /** @brief Returns parameter blob of the layer.
  616. * @param layer name or id of the layer.
  617. * @param numParam index of the layer parameter in the Layer::blobs array.
  618. * @see Layer::blobs
  619. */
  620. CV_WRAP Mat getParam(int layer, int numParam = 0) const;
  621. CV_WRAP inline Mat getParam(const String& layerName, int numParam = 0) const { return getParam(getLayerId(layerName), numParam); }
  622. /** @brief Returns indexes of layers with unconnected outputs.
  623. *
  624. * FIXIT: Rework API to registerOutput() approach, deprecate this call
  625. */
  626. CV_WRAP std::vector<int> getUnconnectedOutLayers() const;
  627. /** @brief Returns names of layers with unconnected outputs.
  628. *
  629. * FIXIT: Rework API to registerOutput() approach, deprecate this call
  630. */
  631. CV_WRAP std::vector<String> getUnconnectedOutLayersNames() const;
  632. /** @brief Returns input and output shapes for all layers in loaded model;
  633. * preliminary inferencing isn't necessary.
  634. * @param netInputShapes shapes for all input blobs in net input layer.
  635. * @param layersIds output parameter for layer IDs.
  636. * @param inLayersShapes output parameter for input layers shapes;
  637. * order is the same as in layersIds
  638. * @param outLayersShapes output parameter for output layers shapes;
  639. * order is the same as in layersIds
  640. */
  641. CV_WRAP void getLayersShapes(const std::vector<MatShape>& netInputShapes,
  642. CV_OUT std::vector<int>& layersIds,
  643. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  644. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  645. /** @overload */
  646. CV_WRAP void getLayersShapes(const MatShape& netInputShape,
  647. CV_OUT std::vector<int>& layersIds,
  648. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  649. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  650. /** @brief Returns input and output shapes for layer with specified
  651. * id in loaded model; preliminary inferencing isn't necessary.
  652. * @param netInputShape shape input blob in net input layer.
  653. * @param layerId id for layer.
  654. * @param inLayerShapes output parameter for input layers shapes;
  655. * order is the same as in layersIds
  656. * @param outLayerShapes output parameter for output layers shapes;
  657. * order is the same as in layersIds
  658. */
  659. void getLayerShapes(const MatShape& netInputShape,
  660. const int layerId,
  661. CV_OUT std::vector<MatShape>& inLayerShapes,
  662. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  663. /** @overload */
  664. void getLayerShapes(const std::vector<MatShape>& netInputShapes,
  665. const int layerId,
  666. CV_OUT std::vector<MatShape>& inLayerShapes,
  667. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  668. /** @brief Computes FLOP for whole loaded model with specified input shapes.
  669. * @param netInputShapes vector of shapes for all net inputs.
  670. * @returns computed FLOP.
  671. */
  672. CV_WRAP int64 getFLOPS(const std::vector<MatShape>& netInputShapes) const;
  673. /** @overload */
  674. CV_WRAP int64 getFLOPS(const MatShape& netInputShape) const;
  675. /** @overload */
  676. CV_WRAP int64 getFLOPS(const int layerId,
  677. const std::vector<MatShape>& netInputShapes) const;
  678. /** @overload */
  679. CV_WRAP int64 getFLOPS(const int layerId,
  680. const MatShape& netInputShape) const;
  681. /** @brief Returns list of types for layer used in model.
  682. * @param layersTypes output parameter for returning types.
  683. */
  684. CV_WRAP void getLayerTypes(CV_OUT std::vector<String>& layersTypes) const;
  685. /** @brief Returns count of layers of specified type.
  686. * @param layerType type.
  687. * @returns count of layers
  688. */
  689. CV_WRAP int getLayersCount(const String& layerType) const;
  690. /** @brief Computes bytes number which are required to store
  691. * all weights and intermediate blobs for model.
  692. * @param netInputShapes vector of shapes for all net inputs.
  693. * @param weights output parameter to store resulting bytes for weights.
  694. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  695. */
  696. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  697. CV_OUT size_t& weights, CV_OUT size_t& blobs) const; // FIXIT: CV_WRAP
  698. /** @overload */
  699. CV_WRAP void getMemoryConsumption(const MatShape& netInputShape,
  700. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  701. /** @overload */
  702. CV_WRAP void getMemoryConsumption(const int layerId,
  703. const std::vector<MatShape>& netInputShapes,
  704. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  705. /** @overload */
  706. CV_WRAP void getMemoryConsumption(const int layerId,
  707. const MatShape& netInputShape,
  708. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  709. /** @brief Computes bytes number which are required to store
  710. * all weights and intermediate blobs for each layer.
  711. * @param netInputShapes vector of shapes for all net inputs.
  712. * @param layerIds output vector to save layer IDs.
  713. * @param weights output parameter to store resulting bytes for weights.
  714. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  715. */
  716. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  717. CV_OUT std::vector<int>& layerIds,
  718. CV_OUT std::vector<size_t>& weights,
  719. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  720. /** @overload */
  721. void getMemoryConsumption(const MatShape& netInputShape,
  722. CV_OUT std::vector<int>& layerIds,
  723. CV_OUT std::vector<size_t>& weights,
  724. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  725. /** @brief Enables or disables layer fusion in the network.
  726. * @param fusion true to enable the fusion, false to disable. The fusion is enabled by default.
  727. */
  728. CV_WRAP void enableFusion(bool fusion);
  729. /** @brief Returns overall time for inference and timings (in ticks) for layers.
  730. *
  731. * Indexes in returned vector correspond to layers ids. Some layers can be fused with others,
  732. * in this case zero ticks count will be return for that skipped layers. Supported by DNN_BACKEND_OPENCV on DNN_TARGET_CPU only.
  733. *
  734. * @param[out] timings vector for tick timings for all layers.
  735. * @return overall ticks for model inference.
  736. */
  737. CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings);
  738. private:
  739. struct Impl;
  740. Ptr<Impl> impl;
  741. };
  742. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  743. * @param cfgFile path to the .cfg file with text description of the network architecture.
  744. * @param darknetModel path to the .weights file with learned network.
  745. * @returns Network object that ready to do forward, throw an exception in failure cases.
  746. * @returns Net object.
  747. */
  748. CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String());
  749. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  750. * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
  751. * @param bufferModel A buffer contains a content of .weights file with learned network.
  752. * @returns Net object.
  753. */
  754. CV_EXPORTS_W Net readNetFromDarknet(const std::vector<uchar>& bufferCfg,
  755. const std::vector<uchar>& bufferModel = std::vector<uchar>());
  756. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  757. * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
  758. * @param lenCfg Number of bytes to read from bufferCfg
  759. * @param bufferModel A buffer contains a content of .weights file with learned network.
  760. * @param lenModel Number of bytes to read from bufferModel
  761. * @returns Net object.
  762. */
  763. CV_EXPORTS Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg,
  764. const char *bufferModel = NULL, size_t lenModel = 0);
  765. /** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format.
  766. * @param prototxt path to the .prototxt file with text description of the network architecture.
  767. * @param caffeModel path to the .caffemodel file with learned network.
  768. * @returns Net object.
  769. */
  770. CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String());
  771. /** @brief Reads a network model stored in Caffe model in memory.
  772. * @param bufferProto buffer containing the content of the .prototxt file
  773. * @param bufferModel buffer containing the content of the .caffemodel file
  774. * @returns Net object.
  775. */
  776. CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
  777. const std::vector<uchar>& bufferModel = std::vector<uchar>());
  778. /** @brief Reads a network model stored in Caffe model in memory.
  779. * @details This is an overloaded member function, provided for convenience.
  780. * It differs from the above function only in what argument(s) it accepts.
  781. * @param bufferProto buffer containing the content of the .prototxt file
  782. * @param lenProto length of bufferProto
  783. * @param bufferModel buffer containing the content of the .caffemodel file
  784. * @param lenModel length of bufferModel
  785. * @returns Net object.
  786. */
  787. CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
  788. const char *bufferModel = NULL, size_t lenModel = 0);
  789. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  790. * @param model path to the .pb file with binary protobuf description of the network architecture
  791. * @param config path to the .pbtxt file that contains text graph definition in protobuf format.
  792. * Resulting Net object is built by text graph using weights from a binary one that
  793. * let us make it more flexible.
  794. * @returns Net object.
  795. */
  796. CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String());
  797. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  798. * @param bufferModel buffer containing the content of the pb file
  799. * @param bufferConfig buffer containing the content of the pbtxt file
  800. * @returns Net object.
  801. */
  802. CV_EXPORTS_W Net readNetFromTensorflow(const std::vector<uchar>& bufferModel,
  803. const std::vector<uchar>& bufferConfig = std::vector<uchar>());
  804. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  805. * @details This is an overloaded member function, provided for convenience.
  806. * It differs from the above function only in what argument(s) it accepts.
  807. * @param bufferModel buffer containing the content of the pb file
  808. * @param lenModel length of bufferModel
  809. * @param bufferConfig buffer containing the content of the pbtxt file
  810. * @param lenConfig length of bufferConfig
  811. */
  812. CV_EXPORTS Net readNetFromTensorflow(const char *bufferModel, size_t lenModel,
  813. const char *bufferConfig = NULL, size_t lenConfig = 0);
  814. /**
  815. * @brief Reads a network model stored in <a href="http://torch.ch">Torch7</a> framework's format.
  816. * @param model path to the file, dumped from Torch by using torch.save() function.
  817. * @param isBinary specifies whether the network was serialized in ascii mode or binary.
  818. * @param evaluate specifies testing phase of network. If true, it's similar to evaluate() method in Torch.
  819. * @returns Net object.
  820. *
  821. * @note Ascii mode of Torch serializer is more preferable, because binary mode extensively use `long` type of C language,
  822. * which has various bit-length on different systems.
  823. *
  824. * The loading file must contain serialized <a href="https://github.com/torch/nn/blob/master/doc/module.md">nn.Module</a> object
  825. * with importing network. Try to eliminate a custom objects from serialazing data to avoid importing errors.
  826. *
  827. * List of supported layers (i.e. object instances derived from Torch nn.Module class):
  828. * - nn.Sequential
  829. * - nn.Parallel
  830. * - nn.Concat
  831. * - nn.Linear
  832. * - nn.SpatialConvolution
  833. * - nn.SpatialMaxPooling, nn.SpatialAveragePooling
  834. * - nn.ReLU, nn.TanH, nn.Sigmoid
  835. * - nn.Reshape
  836. * - nn.SoftMax, nn.LogSoftMax
  837. *
  838. * Also some equivalents of these classes from cunn, cudnn, and fbcunn may be successfully imported.
  839. */
  840. CV_EXPORTS_W Net readNetFromTorch(const String &model, bool isBinary = true, bool evaluate = true);
  841. /**
  842. * @brief Read deep learning network represented in one of the supported formats.
  843. * @param[in] model Binary file contains trained weights. The following file
  844. * extensions are expected for models from different frameworks:
  845. * * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/)
  846. * * `*.pb` (TensorFlow, https://www.tensorflow.org/)
  847. * * `*.t7` | `*.net` (Torch, http://torch.ch/)
  848. * * `*.weights` (Darknet, https://pjreddie.com/darknet/)
  849. * * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit)
  850. * * `*.onnx` (ONNX, https://onnx.ai/)
  851. * @param[in] config Text file contains network configuration. It could be a
  852. * file with the following extensions:
  853. * * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/)
  854. * * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/)
  855. * * `*.cfg` (Darknet, https://pjreddie.com/darknet/)
  856. * * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit)
  857. * @param[in] framework Explicit framework name tag to determine a format.
  858. * @returns Net object.
  859. *
  860. * This function automatically detects an origin framework of trained model
  861. * and calls an appropriate function such @ref readNetFromCaffe, @ref readNetFromTensorflow,
  862. * @ref readNetFromTorch or @ref readNetFromDarknet. An order of @p model and @p config
  863. * arguments does not matter.
  864. */
  865. CV_EXPORTS_W Net readNet(const String& model, const String& config = "", const String& framework = "");
  866. /**
  867. * @brief Read deep learning network represented in one of the supported formats.
  868. * @details This is an overloaded member function, provided for convenience.
  869. * It differs from the above function only in what argument(s) it accepts.
  870. * @param[in] framework Name of origin framework.
  871. * @param[in] bufferModel A buffer with a content of binary file with weights
  872. * @param[in] bufferConfig A buffer with a content of text file contains network configuration.
  873. * @returns Net object.
  874. */
  875. CV_EXPORTS_W Net readNet(const String& framework, const std::vector<uchar>& bufferModel,
  876. const std::vector<uchar>& bufferConfig = std::vector<uchar>());
  877. /** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework.
  878. * @warning This function has the same limitations as readNetFromTorch().
  879. */
  880. CV_EXPORTS_W Mat readTorchBlob(const String &filename, bool isBinary = true);
  881. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  882. * @param[in] xml XML configuration file with network's topology.
  883. * @param[in] bin Binary file with trained weights.
  884. * @returns Net object.
  885. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  886. * backend.
  887. */
  888. CV_EXPORTS_W
  889. Net readNetFromModelOptimizer(const String &xml, const String &bin);
  890. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  891. * @param[in] bufferModelConfig Buffer contains XML configuration with network's topology.
  892. * @param[in] bufferWeights Buffer contains binary data with trained weights.
  893. * @returns Net object.
  894. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  895. * backend.
  896. */
  897. CV_EXPORTS_W
  898. Net readNetFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
  899. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  900. * @param[in] bufferModelConfigPtr Pointer to buffer which contains XML configuration with network's topology.
  901. * @param[in] bufferModelConfigSize Binary size of XML configuration data.
  902. * @param[in] bufferWeightsPtr Pointer to buffer which contains binary data with trained weights.
  903. * @param[in] bufferWeightsSize Binary size of trained weights data.
  904. * @returns Net object.
  905. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  906. * backend.
  907. */
  908. CV_EXPORTS
  909. Net readNetFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
  910. const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
  911. /** @brief Reads a network model <a href="https://onnx.ai/">ONNX</a>.
  912. * @param onnxFile path to the .onnx file with text description of the network architecture.
  913. * @returns Network object that ready to do forward, throw an exception in failure cases.
  914. */
  915. CV_EXPORTS_W Net readNetFromONNX(const String &onnxFile);
  916. /** @brief Reads a network model from <a href="https://onnx.ai/">ONNX</a>
  917. * in-memory buffer.
  918. * @param buffer memory address of the first byte of the buffer.
  919. * @param sizeBuffer size of the buffer.
  920. * @returns Network object that ready to do forward, throw an exception
  921. * in failure cases.
  922. */
  923. CV_EXPORTS Net readNetFromONNX(const char* buffer, size_t sizeBuffer);
  924. /** @brief Reads a network model from <a href="https://onnx.ai/">ONNX</a>
  925. * in-memory buffer.
  926. * @param buffer in-memory buffer that stores the ONNX model bytes.
  927. * @returns Network object that ready to do forward, throw an exception
  928. * in failure cases.
  929. */
  930. CV_EXPORTS_W Net readNetFromONNX(const std::vector<uchar>& buffer);
  931. /** @brief Creates blob from .pb file.
  932. * @param path to the .pb file with input tensor.
  933. * @returns Mat.
  934. */
  935. CV_EXPORTS_W Mat readTensorFromONNX(const String& path);
  936. /** @brief Creates 4-dimensional blob from image. Optionally resizes and crops @p image from center,
  937. * subtract @p mean values, scales values by @p scalefactor, swap Blue and Red channels.
  938. * @param image input image (with 1-, 3- or 4-channels).
  939. * @param size spatial size for output image
  940. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  941. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  942. * @param scalefactor multiplier for @p image values.
  943. * @param swapRB flag which indicates that swap first and last channels
  944. * in 3-channel image is necessary.
  945. * @param crop flag which indicates whether image will be cropped after resize or not
  946. * @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
  947. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
  948. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  949. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  950. * @returns 4-dimensional Mat with NCHW dimensions order.
  951. */
  952. CV_EXPORTS_W Mat blobFromImage(InputArray image, double scalefactor=1.0, const Size& size = Size(),
  953. const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  954. int ddepth=CV_32F);
  955. /** @brief Creates 4-dimensional blob from image.
  956. * @details This is an overloaded member function, provided for convenience.
  957. * It differs from the above function only in what argument(s) it accepts.
  958. */
  959. CV_EXPORTS void blobFromImage(InputArray image, OutputArray blob, double scalefactor=1.0,
  960. const Size& size = Size(), const Scalar& mean = Scalar(),
  961. bool swapRB=false, bool crop=false, int ddepth=CV_32F);
  962. /** @brief Creates 4-dimensional blob from series of images. Optionally resizes and
  963. * crops @p images from center, subtract @p mean values, scales values by @p scalefactor,
  964. * swap Blue and Red channels.
  965. * @param images input images (all with 1-, 3- or 4-channels).
  966. * @param size spatial size for output image
  967. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  968. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  969. * @param scalefactor multiplier for @p images values.
  970. * @param swapRB flag which indicates that swap first and last channels
  971. * in 3-channel image is necessary.
  972. * @param crop flag which indicates whether image will be cropped after resize or not
  973. * @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
  974. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
  975. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  976. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  977. * @returns 4-dimensional Mat with NCHW dimensions order.
  978. */
  979. CV_EXPORTS_W Mat blobFromImages(InputArrayOfArrays images, double scalefactor=1.0,
  980. Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  981. int ddepth=CV_32F);
  982. /** @brief Creates 4-dimensional blob from series of images.
  983. * @details This is an overloaded member function, provided for convenience.
  984. * It differs from the above function only in what argument(s) it accepts.
  985. */
  986. CV_EXPORTS void blobFromImages(InputArrayOfArrays images, OutputArray blob,
  987. double scalefactor=1.0, Size size = Size(),
  988. const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  989. int ddepth=CV_32F);
  990. /** @brief Parse a 4D blob and output the images it contains as 2D arrays through a simpler data structure
  991. * (std::vector<cv::Mat>).
  992. * @param[in] blob_ 4 dimensional array (images, channels, height, width) in floating point precision (CV_32F) from
  993. * which you would like to extract the images.
  994. * @param[out] images_ array of 2D Mat containing the images extracted from the blob in floating point precision
  995. * (CV_32F). They are non normalized neither mean added. The number of returned images equals the first dimension
  996. * of the blob (batch size). Every image has a number of channels equals to the second dimension of the blob (depth).
  997. */
  998. CV_EXPORTS_W void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_);
  999. /** @brief Convert all weights of Caffe network to half precision floating point.
  1000. * @param src Path to origin model from Caffe framework contains single
  1001. * precision floating point weights (usually has `.caffemodel` extension).
  1002. * @param dst Path to destination model with updated weights.
  1003. * @param layersTypes Set of layers types which parameters will be converted.
  1004. * By default, converts only Convolutional and Fully-Connected layers'
  1005. * weights.
  1006. *
  1007. * @note Shrinked model has no origin float32 weights so it can't be used
  1008. * in origin Caffe framework anymore. However the structure of data
  1009. * is taken from NVidia's Caffe fork: https://github.com/NVIDIA/caffe.
  1010. * So the resulting model may be used there.
  1011. */
  1012. CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst,
  1013. const std::vector<String>& layersTypes = std::vector<String>());
  1014. /** @brief Create a text representation for a binary network stored in protocol buffer format.
  1015. * @param[in] model A path to binary network.
  1016. * @param[in] output A path to output text file to be created.
  1017. *
  1018. * @note To reduce output file size, trained weights are not included.
  1019. */
  1020. CV_EXPORTS_W void writeTextGraph(const String& model, const String& output);
  1021. /** @brief Performs non maximum suppression given boxes and corresponding scores.
  1022. * @param bboxes a set of bounding boxes to apply NMS.
  1023. * @param scores a set of corresponding confidences.
  1024. * @param score_threshold a threshold used to filter boxes by score.
  1025. * @param nms_threshold a threshold used in non maximum suppression.
  1026. * @param indices the kept indices of bboxes after NMS.
  1027. * @param eta a coefficient in adaptive threshold formula: \f$nms\_threshold_{i+1}=eta\cdot nms\_threshold_i\f$.
  1028. * @param top_k if `>0`, keep at most @p top_k picked indices.
  1029. */
  1030. CV_EXPORTS void NMSBoxes(const std::vector<Rect>& bboxes, const std::vector<float>& scores,
  1031. const float score_threshold, const float nms_threshold,
  1032. CV_OUT std::vector<int>& indices,
  1033. const float eta = 1.f, const int top_k = 0);
  1034. CV_EXPORTS_W void NMSBoxes(const std::vector<Rect2d>& bboxes, const std::vector<float>& scores,
  1035. const float score_threshold, const float nms_threshold,
  1036. CV_OUT std::vector<int>& indices,
  1037. const float eta = 1.f, const int top_k = 0);
  1038. CV_EXPORTS_AS(NMSBoxesRotated) void NMSBoxes(const std::vector<RotatedRect>& bboxes, const std::vector<float>& scores,
  1039. const float score_threshold, const float nms_threshold,
  1040. CV_OUT std::vector<int>& indices,
  1041. const float eta = 1.f, const int top_k = 0);
  1042. /**
  1043. * @brief Enum of Soft NMS methods.
  1044. * @see softNMSBoxes
  1045. */
  1046. enum class SoftNMSMethod
  1047. {
  1048. SOFTNMS_LINEAR = 1,
  1049. SOFTNMS_GAUSSIAN = 2
  1050. };
  1051. /** @brief Performs soft non maximum suppression given boxes and corresponding scores.
  1052. * Reference: https://arxiv.org/abs/1704.04503
  1053. * @param bboxes a set of bounding boxes to apply Soft NMS.
  1054. * @param scores a set of corresponding confidences.
  1055. * @param updated_scores a set of corresponding updated confidences.
  1056. * @param score_threshold a threshold used to filter boxes by score.
  1057. * @param nms_threshold a threshold used in non maximum suppression.
  1058. * @param indices the kept indices of bboxes after NMS.
  1059. * @param top_k keep at most @p top_k picked indices.
  1060. * @param sigma parameter of Gaussian weighting.
  1061. * @param method Gaussian or linear.
  1062. * @see SoftNMSMethod
  1063. */
  1064. CV_EXPORTS_W void softNMSBoxes(const std::vector<Rect>& bboxes,
  1065. const std::vector<float>& scores,
  1066. CV_OUT std::vector<float>& updated_scores,
  1067. const float score_threshold,
  1068. const float nms_threshold,
  1069. CV_OUT std::vector<int>& indices,
  1070. size_t top_k = 0,
  1071. const float sigma = 0.5,
  1072. SoftNMSMethod method = SoftNMSMethod::SOFTNMS_GAUSSIAN);
  1073. /** @brief This class is presented high-level API for neural networks.
  1074. *
  1075. * Model allows to set params for preprocessing input image.
  1076. * Model creates net from file with trained weights and config,
  1077. * sets preprocessing input and runs forward pass.
  1078. */
  1079. class CV_EXPORTS_W_SIMPLE Model
  1080. {
  1081. public:
  1082. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1083. Model();
  1084. Model(const Model&) = default;
  1085. Model(Model&&) = default;
  1086. Model& operator=(const Model&) = default;
  1087. Model& operator=(Model&&) = default;
  1088. /**
  1089. * @brief Create model from deep learning network represented in one of the supported formats.
  1090. * An order of @p model and @p config arguments does not matter.
  1091. * @param[in] model Binary file contains trained weights.
  1092. * @param[in] config Text file contains network configuration.
  1093. */
  1094. CV_WRAP Model(const String& model, const String& config = "");
  1095. /**
  1096. * @brief Create model from deep learning network.
  1097. * @param[in] network Net object.
  1098. */
  1099. CV_WRAP Model(const Net& network);
  1100. /** @brief Set input size for frame.
  1101. * @param[in] size New input size.
  1102. * @note If shape of the new blob less than 0, then frame size not change.
  1103. */
  1104. CV_WRAP Model& setInputSize(const Size& size);
  1105. /** @overload
  1106. * @param[in] width New input width.
  1107. * @param[in] height New input height.
  1108. */
  1109. CV_WRAP inline
  1110. Model& setInputSize(int width, int height) { return setInputSize(Size(width, height)); }
  1111. /** @brief Set mean value for frame.
  1112. * @param[in] mean Scalar with mean values which are subtracted from channels.
  1113. */
  1114. CV_WRAP Model& setInputMean(const Scalar& mean);
  1115. /** @brief Set scalefactor value for frame.
  1116. * @param[in] scale Multiplier for frame values.
  1117. */
  1118. CV_WRAP Model& setInputScale(double scale);
  1119. /** @brief Set flag crop for frame.
  1120. * @param[in] crop Flag which indicates whether image will be cropped after resize or not.
  1121. */
  1122. CV_WRAP Model& setInputCrop(bool crop);
  1123. /** @brief Set flag swapRB for frame.
  1124. * @param[in] swapRB Flag which indicates that swap first and last channels.
  1125. */
  1126. CV_WRAP Model& setInputSwapRB(bool swapRB);
  1127. /** @brief Set preprocessing parameters for frame.
  1128. * @param[in] size New input size.
  1129. * @param[in] mean Scalar with mean values which are subtracted from channels.
  1130. * @param[in] scale Multiplier for frame values.
  1131. * @param[in] swapRB Flag which indicates that swap first and last channels.
  1132. * @param[in] crop Flag which indicates whether image will be cropped after resize or not.
  1133. * blob(n, c, y, x) = scale * resize( frame(y, x, c) ) - mean(c) )
  1134. */
  1135. CV_WRAP void setInputParams(double scale = 1.0, const Size& size = Size(),
  1136. const Scalar& mean = Scalar(), bool swapRB = false, bool crop = false);
  1137. /** @brief Given the @p input frame, create input blob, run net and return the output @p blobs.
  1138. * @param[in] frame The input image.
  1139. * @param[out] outs Allocated output blobs, which will store results of the computation.
  1140. */
  1141. CV_WRAP void predict(InputArray frame, OutputArrayOfArrays outs) const;
  1142. // ============================== Net proxy methods ==============================
  1143. // Never expose methods with network implementation details, like:
  1144. // - addLayer, addLayerToPrev, connect, setInputsNames, setInputShape, setParam, getParam
  1145. // - getLayer*, getUnconnectedOutLayers, getUnconnectedOutLayersNames, getLayersShapes
  1146. // - forward* methods, setInput
  1147. /// @sa Net::setPreferableBackend
  1148. CV_WRAP Model& setPreferableBackend(dnn::Backend backendId);
  1149. /// @sa Net::setPreferableTarget
  1150. CV_WRAP Model& setPreferableTarget(dnn::Target targetId);
  1151. CV_DEPRECATED_EXTERNAL
  1152. operator Net&() const { return getNetwork_(); }
  1153. //protected: - internal/tests usage only
  1154. Net& getNetwork_() const;
  1155. inline Net& getNetwork_() { return const_cast<const Model*>(this)->getNetwork_(); }
  1156. struct Impl;
  1157. inline Impl* getImpl() const { return impl.get(); }
  1158. inline Impl& getImplRef() const { CV_DbgAssert(impl); return *impl.get(); }
  1159. protected:
  1160. Ptr<Impl> impl;
  1161. };
  1162. /** @brief This class represents high-level API for classification models.
  1163. *
  1164. * ClassificationModel allows to set params for preprocessing input image.
  1165. * ClassificationModel creates net from file with trained weights and config,
  1166. * sets preprocessing input, runs forward pass and return top-1 prediction.
  1167. */
  1168. class CV_EXPORTS_W_SIMPLE ClassificationModel : public Model
  1169. {
  1170. public:
  1171. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1172. ClassificationModel();
  1173. /**
  1174. * @brief Create classification model from network represented in one of the supported formats.
  1175. * An order of @p model and @p config arguments does not matter.
  1176. * @param[in] model Binary file contains trained weights.
  1177. * @param[in] config Text file contains network configuration.
  1178. */
  1179. CV_WRAP ClassificationModel(const String& model, const String& config = "");
  1180. /**
  1181. * @brief Create model from deep learning network.
  1182. * @param[in] network Net object.
  1183. */
  1184. CV_WRAP ClassificationModel(const Net& network);
  1185. /**
  1186. * @brief Set enable/disable softmax post processing option.
  1187. *
  1188. * If this option is true, softmax is applied after forward inference within the classify() function
  1189. * to convert the confidences range to [0.0-1.0].
  1190. * This function allows you to toggle this behavior.
  1191. * Please turn true when not contain softmax layer in model.
  1192. * @param[in] enable Set enable softmax post processing within the classify() function.
  1193. */
  1194. CV_WRAP ClassificationModel& setEnableSoftmaxPostProcessing(bool enable);
  1195. /**
  1196. * @brief Get enable/disable softmax post processing option.
  1197. *
  1198. * This option defaults to false, softmax post processing is not applied within the classify() function.
  1199. */
  1200. CV_WRAP bool getEnableSoftmaxPostProcessing() const;
  1201. /** @brief Given the @p input frame, create input blob, run net and return top-1 prediction.
  1202. * @param[in] frame The input image.
  1203. */
  1204. std::pair<int, float> classify(InputArray frame);
  1205. /** @overload */
  1206. CV_WRAP void classify(InputArray frame, CV_OUT int& classId, CV_OUT float& conf);
  1207. };
  1208. /** @brief This class represents high-level API for keypoints models
  1209. *
  1210. * KeypointsModel allows to set params for preprocessing input image.
  1211. * KeypointsModel creates net from file with trained weights and config,
  1212. * sets preprocessing input, runs forward pass and returns the x and y coordinates of each detected keypoint
  1213. */
  1214. class CV_EXPORTS_W_SIMPLE KeypointsModel: public Model
  1215. {
  1216. public:
  1217. /**
  1218. * @brief Create keypoints model from network represented in one of the supported formats.
  1219. * An order of @p model and @p config arguments does not matter.
  1220. * @param[in] model Binary file contains trained weights.
  1221. * @param[in] config Text file contains network configuration.
  1222. */
  1223. CV_WRAP KeypointsModel(const String& model, const String& config = "");
  1224. /**
  1225. * @brief Create model from deep learning network.
  1226. * @param[in] network Net object.
  1227. */
  1228. CV_WRAP KeypointsModel(const Net& network);
  1229. /** @brief Given the @p input frame, create input blob, run net
  1230. * @param[in] frame The input image.
  1231. * @param thresh minimum confidence threshold to select a keypoint
  1232. * @returns a vector holding the x and y coordinates of each detected keypoint
  1233. *
  1234. */
  1235. CV_WRAP std::vector<Point2f> estimate(InputArray frame, float thresh=0.5);
  1236. };
  1237. /** @brief This class represents high-level API for segmentation models
  1238. *
  1239. * SegmentationModel allows to set params for preprocessing input image.
  1240. * SegmentationModel creates net from file with trained weights and config,
  1241. * sets preprocessing input, runs forward pass and returns the class prediction for each pixel.
  1242. */
  1243. class CV_EXPORTS_W_SIMPLE SegmentationModel: public Model
  1244. {
  1245. public:
  1246. /**
  1247. * @brief Create segmentation model from network represented in one of the supported formats.
  1248. * An order of @p model and @p config arguments does not matter.
  1249. * @param[in] model Binary file contains trained weights.
  1250. * @param[in] config Text file contains network configuration.
  1251. */
  1252. CV_WRAP SegmentationModel(const String& model, const String& config = "");
  1253. /**
  1254. * @brief Create model from deep learning network.
  1255. * @param[in] network Net object.
  1256. */
  1257. CV_WRAP SegmentationModel(const Net& network);
  1258. /** @brief Given the @p input frame, create input blob, run net
  1259. * @param[in] frame The input image.
  1260. * @param[out] mask Allocated class prediction for each pixel
  1261. */
  1262. CV_WRAP void segment(InputArray frame, OutputArray mask);
  1263. };
  1264. /** @brief This class represents high-level API for object detection networks.
  1265. *
  1266. * DetectionModel allows to set params for preprocessing input image.
  1267. * DetectionModel creates net from file with trained weights and config,
  1268. * sets preprocessing input, runs forward pass and return result detections.
  1269. * For DetectionModel SSD, Faster R-CNN, YOLO topologies are supported.
  1270. */
  1271. class CV_EXPORTS_W_SIMPLE DetectionModel : public Model
  1272. {
  1273. public:
  1274. /**
  1275. * @brief Create detection model from network represented in one of the supported formats.
  1276. * An order of @p model and @p config arguments does not matter.
  1277. * @param[in] model Binary file contains trained weights.
  1278. * @param[in] config Text file contains network configuration.
  1279. */
  1280. CV_WRAP DetectionModel(const String& model, const String& config = "");
  1281. /**
  1282. * @brief Create model from deep learning network.
  1283. * @param[in] network Net object.
  1284. */
  1285. CV_WRAP DetectionModel(const Net& network);
  1286. CV_DEPRECATED_EXTERNAL // avoid using in C++ code (need to fix bindings first)
  1287. DetectionModel();
  1288. /**
  1289. * @brief nmsAcrossClasses defaults to false,
  1290. * such that when non max suppression is used during the detect() function, it will do so per-class.
  1291. * This function allows you to toggle this behaviour.
  1292. * @param[in] value The new value for nmsAcrossClasses
  1293. */
  1294. CV_WRAP DetectionModel& setNmsAcrossClasses(bool value);
  1295. /**
  1296. * @brief Getter for nmsAcrossClasses. This variable defaults to false,
  1297. * such that when non max suppression is used during the detect() function, it will do so only per-class
  1298. */
  1299. CV_WRAP bool getNmsAcrossClasses();
  1300. /** @brief Given the @p input frame, create input blob, run net and return result detections.
  1301. * @param[in] frame The input image.
  1302. * @param[out] classIds Class indexes in result detection.
  1303. * @param[out] confidences A set of corresponding confidences.
  1304. * @param[out] boxes A set of bounding boxes.
  1305. * @param[in] confThreshold A threshold used to filter boxes by confidences.
  1306. * @param[in] nmsThreshold A threshold used in non maximum suppression.
  1307. */
  1308. CV_WRAP void detect(InputArray frame, CV_OUT std::vector<int>& classIds,
  1309. CV_OUT std::vector<float>& confidences, CV_OUT std::vector<Rect>& boxes,
  1310. float confThreshold = 0.5f, float nmsThreshold = 0.0f);
  1311. };
  1312. /** @brief This class represents high-level API for text recognition networks.
  1313. *
  1314. * TextRecognitionModel allows to set params for preprocessing input image.
  1315. * TextRecognitionModel creates net from file with trained weights and config,
  1316. * sets preprocessing input, runs forward pass and return recognition result.
  1317. * For TextRecognitionModel, CRNN-CTC is supported.
  1318. */
  1319. class CV_EXPORTS_W_SIMPLE TextRecognitionModel : public Model
  1320. {
  1321. public:
  1322. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1323. TextRecognitionModel();
  1324. /**
  1325. * @brief Create Text Recognition model from deep learning network
  1326. * Call setDecodeType() and setVocabulary() after constructor to initialize the decoding method
  1327. * @param[in] network Net object
  1328. */
  1329. CV_WRAP TextRecognitionModel(const Net& network);
  1330. /**
  1331. * @brief Create text recognition model from network represented in one of the supported formats
  1332. * Call setDecodeType() and setVocabulary() after constructor to initialize the decoding method
  1333. * @param[in] model Binary file contains trained weights
  1334. * @param[in] config Text file contains network configuration
  1335. */
  1336. CV_WRAP inline
  1337. TextRecognitionModel(const std::string& model, const std::string& config = "")
  1338. : TextRecognitionModel(readNet(model, config)) { /* nothing */ }
  1339. /**
  1340. * @brief Set the decoding method of translating the network output into string
  1341. * @param[in] decodeType The decoding method of translating the network output into string, currently supported type:
  1342. * - `"CTC-greedy"` greedy decoding for the output of CTC-based methods
  1343. * - `"CTC-prefix-beam-search"` Prefix beam search decoding for the output of CTC-based methods
  1344. */
  1345. CV_WRAP
  1346. TextRecognitionModel& setDecodeType(const std::string& decodeType);
  1347. /**
  1348. * @brief Get the decoding method
  1349. * @return the decoding method
  1350. */
  1351. CV_WRAP
  1352. const std::string& getDecodeType() const;
  1353. /**
  1354. * @brief Set the decoding method options for `"CTC-prefix-beam-search"` decode usage
  1355. * @param[in] beamSize Beam size for search
  1356. * @param[in] vocPruneSize Parameter to optimize big vocabulary search,
  1357. * only take top @p vocPruneSize tokens in each search step, @p vocPruneSize <= 0 stands for disable this prune.
  1358. */
  1359. CV_WRAP
  1360. TextRecognitionModel& setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize = 0);
  1361. /**
  1362. * @brief Set the vocabulary for recognition.
  1363. * @param[in] vocabulary the associated vocabulary of the network.
  1364. */
  1365. CV_WRAP
  1366. TextRecognitionModel& setVocabulary(const std::vector<std::string>& vocabulary);
  1367. /**
  1368. * @brief Get the vocabulary for recognition.
  1369. * @return vocabulary the associated vocabulary
  1370. */
  1371. CV_WRAP
  1372. const std::vector<std::string>& getVocabulary() const;
  1373. /**
  1374. * @brief Given the @p input frame, create input blob, run net and return recognition result
  1375. * @param[in] frame The input image
  1376. * @return The text recognition result
  1377. */
  1378. CV_WRAP
  1379. std::string recognize(InputArray frame) const;
  1380. /**
  1381. * @brief Given the @p input frame, create input blob, run net and return recognition result
  1382. * @param[in] frame The input image
  1383. * @param[in] roiRects List of text detection regions of interest (cv::Rect, CV_32SC4). ROIs is be cropped as the network inputs
  1384. * @param[out] results A set of text recognition results.
  1385. */
  1386. CV_WRAP
  1387. void recognize(InputArray frame, InputArrayOfArrays roiRects, CV_OUT std::vector<std::string>& results) const;
  1388. };
  1389. /** @brief Base class for text detection networks
  1390. */
  1391. class CV_EXPORTS_W_SIMPLE TextDetectionModel : public Model
  1392. {
  1393. protected:
  1394. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1395. TextDetectionModel();
  1396. public:
  1397. /** @brief Performs detection
  1398. *
  1399. * Given the input @p frame, prepare network input, run network inference, post-process network output and return result detections.
  1400. *
  1401. * Each result is quadrangle's 4 points in this order:
  1402. * - bottom-left
  1403. * - top-left
  1404. * - top-right
  1405. * - bottom-right
  1406. *
  1407. * Use cv::getPerspectiveTransform function to retrieve image region without perspective transformations.
  1408. *
  1409. * @note If DL model doesn't support that kind of output then result may be derived from detectTextRectangles() output.
  1410. *
  1411. * @param[in] frame The input image
  1412. * @param[out] detections array with detections' quadrangles (4 points per result)
  1413. * @param[out] confidences array with detection confidences
  1414. */
  1415. CV_WRAP
  1416. void detect(
  1417. InputArray frame,
  1418. CV_OUT std::vector< std::vector<Point> >& detections,
  1419. CV_OUT std::vector<float>& confidences
  1420. ) const;
  1421. /** @overload */
  1422. CV_WRAP
  1423. void detect(
  1424. InputArray frame,
  1425. CV_OUT std::vector< std::vector<Point> >& detections
  1426. ) const;
  1427. /** @brief Performs detection
  1428. *
  1429. * Given the input @p frame, prepare network input, run network inference, post-process network output and return result detections.
  1430. *
  1431. * Each result is rotated rectangle.
  1432. *
  1433. * @note Result may be inaccurate in case of strong perspective transformations.
  1434. *
  1435. * @param[in] frame the input image
  1436. * @param[out] detections array with detections' RotationRect results
  1437. * @param[out] confidences array with detection confidences
  1438. */
  1439. CV_WRAP
  1440. void detectTextRectangles(
  1441. InputArray frame,
  1442. CV_OUT std::vector<cv::RotatedRect>& detections,
  1443. CV_OUT std::vector<float>& confidences
  1444. ) const;
  1445. /** @overload */
  1446. CV_WRAP
  1447. void detectTextRectangles(
  1448. InputArray frame,
  1449. CV_OUT std::vector<cv::RotatedRect>& detections
  1450. ) const;
  1451. };
  1452. /** @brief This class represents high-level API for text detection DL networks compatible with EAST model.
  1453. *
  1454. * Configurable parameters:
  1455. * - (float) confThreshold - used to filter boxes by confidences, default: 0.5f
  1456. * - (float) nmsThreshold - used in non maximum suppression, default: 0.0f
  1457. */
  1458. class CV_EXPORTS_W_SIMPLE TextDetectionModel_EAST : public TextDetectionModel
  1459. {
  1460. public:
  1461. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1462. TextDetectionModel_EAST();
  1463. /**
  1464. * @brief Create text detection algorithm from deep learning network
  1465. * @param[in] network Net object
  1466. */
  1467. CV_WRAP TextDetectionModel_EAST(const Net& network);
  1468. /**
  1469. * @brief Create text detection model from network represented in one of the supported formats.
  1470. * An order of @p model and @p config arguments does not matter.
  1471. * @param[in] model Binary file contains trained weights.
  1472. * @param[in] config Text file contains network configuration.
  1473. */
  1474. CV_WRAP inline
  1475. TextDetectionModel_EAST(const std::string& model, const std::string& config = "")
  1476. : TextDetectionModel_EAST(readNet(model, config)) { /* nothing */ }
  1477. /**
  1478. * @brief Set the detection confidence threshold
  1479. * @param[in] confThreshold A threshold used to filter boxes by confidences
  1480. */
  1481. CV_WRAP
  1482. TextDetectionModel_EAST& setConfidenceThreshold(float confThreshold);
  1483. /**
  1484. * @brief Get the detection confidence threshold
  1485. */
  1486. CV_WRAP
  1487. float getConfidenceThreshold() const;
  1488. /**
  1489. * @brief Set the detection NMS filter threshold
  1490. * @param[in] nmsThreshold A threshold used in non maximum suppression
  1491. */
  1492. CV_WRAP
  1493. TextDetectionModel_EAST& setNMSThreshold(float nmsThreshold);
  1494. /**
  1495. * @brief Get the detection confidence threshold
  1496. */
  1497. CV_WRAP
  1498. float getNMSThreshold() const;
  1499. };
  1500. /** @brief This class represents high-level API for text detection DL networks compatible with DB model.
  1501. *
  1502. * Related publications: @cite liao2020real
  1503. * Paper: https://arxiv.org/abs/1911.08947
  1504. * For more information about the hyper-parameters setting, please refer to https://github.com/MhLiao/DB
  1505. *
  1506. * Configurable parameters:
  1507. * - (float) binaryThreshold - The threshold of the binary map. It is usually set to 0.3.
  1508. * - (float) polygonThreshold - The threshold of text polygons. It is usually set to 0.5, 0.6, and 0.7. Default is 0.5f
  1509. * - (double) unclipRatio - The unclip ratio of the detected text region, which determines the output size. It is usually set to 2.0.
  1510. * - (int) maxCandidates - The max number of the output results.
  1511. */
  1512. class CV_EXPORTS_W_SIMPLE TextDetectionModel_DB : public TextDetectionModel
  1513. {
  1514. public:
  1515. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1516. TextDetectionModel_DB();
  1517. /**
  1518. * @brief Create text detection algorithm from deep learning network.
  1519. * @param[in] network Net object.
  1520. */
  1521. CV_WRAP TextDetectionModel_DB(const Net& network);
  1522. /**
  1523. * @brief Create text detection model from network represented in one of the supported formats.
  1524. * An order of @p model and @p config arguments does not matter.
  1525. * @param[in] model Binary file contains trained weights.
  1526. * @param[in] config Text file contains network configuration.
  1527. */
  1528. CV_WRAP inline
  1529. TextDetectionModel_DB(const std::string& model, const std::string& config = "")
  1530. : TextDetectionModel_DB(readNet(model, config)) { /* nothing */ }
  1531. CV_WRAP TextDetectionModel_DB& setBinaryThreshold(float binaryThreshold);
  1532. CV_WRAP float getBinaryThreshold() const;
  1533. CV_WRAP TextDetectionModel_DB& setPolygonThreshold(float polygonThreshold);
  1534. CV_WRAP float getPolygonThreshold() const;
  1535. CV_WRAP TextDetectionModel_DB& setUnclipRatio(double unclipRatio);
  1536. CV_WRAP double getUnclipRatio() const;
  1537. CV_WRAP TextDetectionModel_DB& setMaxCandidates(int maxCandidates);
  1538. CV_WRAP int getMaxCandidates() const;
  1539. };
  1540. //! @}
  1541. CV__DNN_INLINE_NS_END
  1542. }
  1543. }
  1544. #include <opencv2/dnn/layer.hpp>
  1545. #include <opencv2/dnn/dnn.inl.hpp>
  1546. /// @deprecated Include this header directly from application. Automatic inclusion will be removed
  1547. #include <opencv2/dnn/utils/inference_engine.hpp>
  1548. #endif /* OPENCV_DNN_DNN_HPP */