Advertisement
itruf

Untitled

Feb 18th, 2022
288
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 18.81 KB | None | 0 0
  1. #include <chrono>
  2. #include <iostream>
  3. #include <vector>
  4. #include <utility>
  5. #include <fsdk/FaceEngine.h>
  6.  
  7. #import <UIKit/UIKit.h>
  8. #import "FaceRecognitionManager.h"
  9.  
  10. #import "KYC_Demo-Swift.h"
  11.  
  12. #define OUT(x) std::cout << "[" << __PRETTY_FUNCTION__ << "]: " << x << '\n';
  13.  
  14. struct Constant {
  15.  
  16. };
  17.  
  18. struct VectorArchive: fsdk::IArchive
  19. {
  20. std::vector<uint8_t>& dataOut;
  21. int index = 0;
  22.  
  23. bool write(const void* data, size_t size) noexcept override {
  24. const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
  25. dataOut.insert(dataOut.end(), p, p+size);
  26. return true;
  27. }
  28.  
  29. bool read(void* data, size_t size) noexcept override {
  30. assert(size <= dataOut.size()-index);
  31. memcpy(data, (void*)&dataOut[0+index], size);
  32. index += size;
  33. return true;
  34. }
  35.  
  36. void setSizeHint(size_t /*hint*/) noexcept override {}
  37.  
  38. VectorArchive(std::vector<uint8_t>& dataref):
  39. dataOut(dataref)
  40. {}
  41. };
  42.  
  43. @implementation FaceRecognitionManager {
  44. fsdk::IFaceEngineMobilePtr engine; // Root SDK object
  45. fsdk::IDescriptorExtractorPtr descriptorExtractor; // Face descriptor extractor
  46. fsdk::IDescriptorMatcherPtr descriptorMatcher; // Face descriptor matcher
  47. fsdk::IDetectorPtr detector;
  48. fsdk::IWarperPtr warper; // Face descriptor warper
  49.  
  50. std::mutex mutex;
  51.  
  52. bool loaded;
  53. bool activated;
  54. bool completeEdition;
  55. }
  56.  
  57. static NSString * const kModelsDirectory = @"data";
  58.  
  59. // returns vlf image in rgb format (the one extractor accepts)
  60. // uiimage is expected to be in bgra
  61. fsdk::Image uiimageToVlfImage(UIImage* uiimage) {
  62. CGImage* cgimage = uiimage.CGImage;
  63. NSDictionary *options = @{
  64. (NSString*)kCVPixelBufferCGImageCompatibilityKey : @YES,
  65. (NSString*)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES,
  66. };
  67.  
  68. CVPixelBufferRef pxbuffer = NULL;
  69. CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault,
  70. CGImageGetWidth(cgimage),
  71. CGImageGetHeight(cgimage),
  72. kCVPixelFormatType_32BGRA,
  73. (__bridge CFDictionaryRef) options,
  74. &pxbuffer);
  75. fsdk::Image resultingImage;
  76.  
  77. if (status != kCVReturnSuccess || !pxbuffer) {
  78. OUT("Operation failed");
  79. return resultingImage;
  80. }
  81.  
  82. CVPixelBufferLockBaseAddress(pxbuffer, 0);
  83.  
  84. CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
  85. CGContextRef context = CGBitmapContextCreate(CVPixelBufferGetBaseAddress(pxbuffer),
  86. CGImageGetWidth(cgimage),
  87. CGImageGetHeight(cgimage),
  88. 8,
  89. 4 * CGImageGetWidth(cgimage),
  90. rgbColorSpace,
  91. kCGImageAlphaNoneSkipLast);
  92.  
  93. CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(cgimage), CGImageGetHeight(cgimage)), cgimage);
  94. CGColorSpaceRelease(rgbColorSpace);
  95. CGContextRelease(context);
  96.  
  97. fsdk::Image vlfimage = fsdk::Image((int32_t)CVPixelBufferGetWidth(pxbuffer),
  98. (int32_t)CVPixelBufferGetHeight(pxbuffer),
  99. fsdk::Format::R8G8B8X8,
  100. CVPixelBufferGetBaseAddress(pxbuffer));
  101.  
  102. if(!vlfimage.convert(resultingImage, fsdk::Format::R8G8B8))
  103. OUT("Failed to convert image to rgb");
  104.  
  105. CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
  106. CVPixelBufferRelease(pxbuffer);
  107.  
  108. return resultingImage;
  109. }
  110.  
  111. /********************************************************/
  112. UIImage* vlfImagetoUiimage(const fsdk::Image& vlfimage) {
  113. assert(vlfimage.getFormat() == fsdk::Format::R8G8B8);
  114. const auto channelNum = 3;
  115. NSData *data = [NSData dataWithBytes: vlfimage.getData() length: vlfimage.getWidth() * vlfimage.getHeight() * channelNum];
  116. CGColorSpaceRef colorSpace;
  117. CGBitmapInfo bitmapInfo;
  118.  
  119. colorSpace = CGColorSpaceCreateDeviceRGB();
  120. bitmapInfo = kCGBitmapByteOrder32Big | kCGImageAlphaNone;
  121.  
  122. CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)data);
  123.  
  124. // Creating CGImage from cv::Mat
  125. CGImageRef imageRef = CGImageCreate(
  126. vlfimage.getWidth(), //width
  127. vlfimage.getHeight(), //height
  128. 8, //bits per component
  129. 8 * channelNum, //bits per pixel
  130. vlfimage.getWidth() * channelNum, //bytesPerRow
  131. colorSpace, //colorspace
  132. bitmapInfo, // bitmap info
  133. provider, //CGDataProviderRef
  134. NULL, //decode
  135. false, //should interpolate
  136. kCGRenderingIntentDefault //intent
  137. );
  138.  
  139. // Getting UIImage from CGImage
  140. UIImage *finalImage = [UIImage imageWithCGImage:imageRef scale:1 orientation: UIImageOrientationUp];
  141. CGImageRelease(imageRef);
  142. CGDataProviderRelease(provider);
  143. CGColorSpaceRelease(colorSpace);
  144.  
  145. return finalImage;
  146. }
  147.  
  148. #pragma mark - Public
  149.  
  150. + (FaceRecognitionManager *)shared {
  151. static FaceRecognitionManager *sharedManager = nil;
  152. static dispatch_once_t onceToken;
  153.  
  154. dispatch_once(&onceToken, ^{
  155. sharedManager = [[self alloc] init];
  156. });
  157.  
  158. return sharedManager;
  159. }
  160.  
  161. - (FaceEngineLoadError)load {
  162. std::lock_guard<decltype(mutex)> guard(mutex);
  163. NSString *path = [[[NSBundle bundleWithIdentifier:@"ru.visionlabs.FaceEngine"] resourcePath] stringByAppendingPathComponent:kModelsDirectory];
  164. fsdk::ISettingsProviderPtr config;
  165. fsdk::ISettingsProviderPtr runtimeConfig;
  166. std::string configPath = std::string([path UTF8String]) + "/faceengine.conf";
  167. std::string licenseConfPath = std::string([path UTF8String]) + "/license.conf";
  168. std::string runtimeConfPath = std::string([path UTF8String]) + "/runtime.conf";
  169.  
  170. loaded = false;
  171. activated = false;
  172.  
  173. OUT("face engine data path: " << [path UTF8String]);
  174. auto resEngine = fsdk::createFaceEngineMobile([path UTF8String]);
  175. if (resEngine.isError()) {
  176. OUT("Failed to create face engine. " << resEngine.what())
  177. return FaceEngineLoadError::FaceEngineNotLoaded;
  178. }
  179. engine = resEngine.getValue();
  180.  
  181. if(!engine) {
  182. OUT("Failed to create face engine.")
  183. return FaceEngineLoadError::FaceEngineNotLoaded;
  184. }
  185.  
  186. // engine->setDataDirectory([path UTF8String]);
  187.  
  188. completeEdition = (engine->getFaceEngineEdition() == fsdk::FaceEngineEdition::CompleteEdition);
  189.  
  190. auto resRuntimeConfig = fsdk::createSettingsProvider(runtimeConfPath.c_str());
  191. if (resRuntimeConfig.isError()) {
  192. OUT("[ISettingsProvider creating] Failed to create Runtime ISettingsProvider. " << resRuntimeConfig.what());
  193. return FaceEngineLoadError::FaceEngineNotLoaded;
  194. }
  195. runtimeConfig = resRuntimeConfig.getValue();
  196.  
  197. if (!runtimeConfig) {
  198. OUT("[ISettingsProvider creating] Failed to create Runtime ISettingsProvider. ");
  199. return FaceEngineLoadError::FaceEngineNotLoaded;
  200. }
  201.  
  202. auto resConfig = fsdk::createSettingsProvider(configPath.c_str());
  203. if (resConfig.isError()) {
  204. OUT("[ISettingsProvider creating] Failed to create ISettingsProvider. " << resConfig.what());
  205. return FaceEngineLoadError::FaceEngineNotLoaded;
  206. }
  207. config = resConfig.getValue();
  208. if (!config) {
  209. OUT("[ISettingsProvider creating] Failed to create ISettingsProvider");
  210. return FaceEngineLoadError::FaceEngineNotLoaded;
  211. }
  212.  
  213. config->setValue("DescriptorFactory::Settings", "useMobileNet", 1);
  214.  
  215. config->setValue("FaceDetV2::Settings", "minFaceSize", (int)[Luna5Settings minIndent]);
  216.  
  217. engine->setSettingsProvider(config);
  218. runtimeConfig->setValue("Runtime", "numThreads", -1);
  219. engine->setRuntimeSettingsProvider(runtimeConfig);
  220. fsdk::ILicense* license = engine->getLicense();
  221. if (!license) {
  222. OUT("[License creating] Failed to get license");
  223. return FaceEngineLoadError::FaceEngineNotLoaded;
  224. }
  225.  
  226. // Make activation
  227. bool activationResult = fsdk::activateLicense(license, licenseConfPath.c_str());
  228. if (!(activationResult)) {
  229. OUT("[Descriptor creating] Failed to activate license");
  230. return FaceEngineLoadError::LicenseActivationFailed;
  231. } else {
  232. activated = true;
  233. OUT("[Descriptor creating] license is activated");
  234. }
  235. auto resDescriptorExtractor = engine->createExtractor();
  236. if (resDescriptorExtractor.isError()) {
  237. OUT("Failed to create descriptor extractor. " << resDescriptorExtractor.what());
  238. return FaceEngineLoadError::FaceEngineNotLoaded;
  239. }
  240. descriptorExtractor = resDescriptorExtractor.getValue();
  241. if (!descriptorExtractor) {
  242. OUT("Failed to create descriptor extractor");
  243. return FaceEngineLoadError::FaceEngineNotLoaded;
  244. }
  245. auto resDescriptorMatcher = engine->createMatcher();
  246.  
  247. if (resDescriptorMatcher.isError()) {
  248. OUT("Failed to create descriptor matcher. " << resDescriptorMatcher.what());
  249. return FaceEngineLoadError::FaceEngineNotLoaded;
  250. }
  251. descriptorMatcher = resDescriptorMatcher.getValue();
  252. if (!descriptorMatcher) {
  253. OUT("Failed to create descriptor matcher");
  254. return FaceEngineLoadError::FaceEngineNotLoaded;
  255. }
  256.  
  257. // Create default detector, see faceengine.conf - "defaultDetectorType"
  258. auto resFaceDetector = engine->createDetector();
  259. if (!resFaceDetector) {
  260. OUT("Failed to create face detector instance.");
  261. return FaceEngineLoadError::FaceEngineNotLoaded;
  262. }
  263. detector = resFaceDetector.getValue();
  264.  
  265. auto resWarper = engine->createWarper();
  266. if(!resWarper) {
  267. OUT("Failed to create warper instance.");
  268. return FaceEngineLoadError::FaceEngineNotLoaded;
  269. }
  270. warper = resWarper.getValue();
  271.  
  272. loaded = true;
  273. activated = true;
  274.  
  275. return FaceEngineLoadError::FaceEngineLoadSuccess;
  276. }
  277.  
  278. - (NSData*)extractDescriptorFrom: (UIImage*)image {
  279. std::lock_guard<decltype(mutex)> guard(mutex);
  280.  
  281. if(!loaded) {
  282. OUT("Not loaded properly");
  283. return nil;
  284. }
  285.  
  286. // convert UIImage to fsdk::Image
  287. fsdk::Image rgbimage = uiimageToVlfImage(image);
  288.  
  289. auto resDescriptor = engine->createDescriptor();
  290. if(resDescriptor.isError()) {
  291. OUT("[extractDescriptorFrom] Failed to create descriptor instance. " << resDescriptor.what());
  292. return nil;
  293. }
  294. fsdk::IDescriptorPtr descriptor = resDescriptor.getValue();
  295. if(!descriptor) {
  296. OUT("[extractDescriptorFrom] Failed to create descriptor instance.");
  297. return nil;
  298. }
  299. if(!rgbimage.isValid()) {
  300. OUT("[extractDescriptorFrom] Image is not valid. No extraction");
  301. return nil;
  302. }
  303.  
  304. const auto result = descriptorExtractor->extractFromWarpedImage(rgbimage, descriptor);
  305.  
  306. if(result) {
  307. OUT("Descriptor extracted");
  308. std::vector<uint8_t> descriptorData;
  309. VectorArchive archiveDescriptor(descriptorData);
  310.  
  311. descriptor->save(&archiveDescriptor);
  312. NSData* nsdata = [[NSData alloc] initWithBytes: descriptorData.data() length:descriptorData.size()];
  313.  
  314. if(!nsdata) {
  315. OUT("Couldnt convert descriptor to nsdata");
  316. return nil;
  317. }
  318.  
  319. return nsdata;
  320. } else {
  321. OUT("Failed to extract descriptor: " << result.what());
  322. return nil;
  323. }
  324. }
  325.  
  326. - (UIImage*)warp: (UIImage*)image {
  327. std::lock_guard<decltype(mutex)> guard(mutex);
  328.  
  329. if(!loaded) {
  330. OUT("Not loaded properly");
  331. return nil;
  332. }
  333.  
  334. // convert UIImage to fsdk::Image
  335. fsdk::Image rgbimage = uiimageToVlfImage(image);
  336.  
  337.  
  338. // Facial feature detection confidence threshold.
  339. const float confidenceThreshold = 0.25f;
  340.  
  341. if (!rgbimage) {
  342. OUT("Request image is invalid.");
  343. return nil;
  344. }
  345.  
  346. // Stage 1. Detect a face.
  347. OUT("Detecting faces.");
  348.  
  349. // Detect no more than 10 faces in the image.
  350. const uint32_t maxDetections = 1;
  351.  
  352. // Data used for detection.
  353. uint32_t detectionsCount(maxDetections);
  354. fsdk::Rect imageRect = rgbimage.getRect();
  355.  
  356. fsdk::ResultValue<fsdk::FSDKError, fsdk::Face> detectorResult = detector->detectOne(rgbimage, imageRect, fsdk::DT_LANDMARKS5);
  357.  
  358. // // Detect faces in the image.
  359. // fsdk::ResultValue<fsdk::FSDKError, fsdk::Ref<fsdk::IFaceDetectionBatch>> detectorResult =
  360. // detector->detect(
  361. // fsdk::Span<const fsdk::Image>(&rgbimage, 1),
  362. // fsdk::Span<const fsdk::Rect>(&imageRect, 1),
  363. // detectionsCount,
  364. // fsdk::DT_ALL
  365. // );
  366.  
  367. if (detectorResult.isError()) {
  368. OUT("Failed to detect face detection. Reason: " << detectorResult.what());
  369. return nullptr;
  370. }
  371.  
  372. fsdk::Face face = detectorResult.getValue();
  373. // fsdk::Ref<fsdk::IFaceDetectionBatch> detectionBatch = detectorResult.getValue();
  374. // detectionsCount = static_cast<uint32_t>(detectionBatch->getSize(0));
  375. // if (detectionsCount == 0) {
  376. // OUT("Faces are not found.");
  377. // return nullptr;
  378. // }
  379. //
  380. // OUT("Found " << detectionsCount << " face(s).");
  381.  
  382. // const fsdk::Span<const fsdk::Detection>& detections = detectionBatch->getDetections(0);
  383. // const fsdk::Span<const fsdk::Landmarks5>& landmarks5 = detectionBatch->getLandmarks5(0);
  384.  
  385. // int bestDetectionIndex(-1);
  386. // float bestScore(-1.f);
  387.  
  388. // Loop through all the faces and find one with the best score.
  389. // for (size_t i = 0; i < detectionsCount; ++i) {
  390. // const fsdk::Detection &detection = detections[i];
  391. // OUT("Detecting facial features (" << i + 1 << "/" << detectionsCount << ")");
  392. //
  393. // // Choose the best detection.
  394. // float score = detection.getScore();
  395. // if (score > bestScore) {
  396. // bestScore = score;
  397. // bestDetectionIndex = static_cast<int>(i);
  398. // }
  399. // }
  400.  
  401. // // If detection confidence score is too low, abort.
  402. // if (bestScore < confidenceThreshold) {
  403. // OUT("Face detection succeeded, but no faces with good confidence found.");
  404. // return nullptr;
  405. // }
  406. // OUT("Best face confidence is " << bestScore);
  407.  
  408. // Stage 2. Create face descriptor.
  409. OUT("Extracting descriptor.");
  410.  
  411. // Create face descriptor.
  412. auto resDescriptor = engine->createDescriptor();
  413. if (!resDescriptor) {
  414. OUT("Failed to create face descrtiptor instance.");
  415. return nullptr;
  416. }
  417. fsdk::IDescriptorPtr descriptor = resDescriptor.getValue();
  418.  
  419. // Extract face descriptor.
  420. // This is typically the most time-consuming task.
  421. // fsdk::Transformation transform = warper->createTransformation(
  422. // detections[bestDetectionIndex],
  423. // landmarks5[bestDetectionIndex]
  424. // );
  425.  
  426. if (face.landmarks5.has_value() == false) {
  427. OUT("Failed to get face landmark.");
  428. return nullptr;
  429. }
  430.  
  431. fsdk::Transformation transform = warper->createTransformation(
  432. face.detection,
  433. face.landmarks5.value()
  434. );
  435.  
  436. fsdk::Image warpedImage = {};
  437. fsdk::Result<fsdk::FSDKError> warpResult = warper->warp(rgbimage, transform, warpedImage);
  438. if(warpResult.isError()) {
  439. OUT("Failed to warp image. Reason: " << warpResult.what());
  440. return nullptr;
  441. }
  442.  
  443. UIImage *result = vlfImagetoUiimage(warpedImage);
  444.  
  445. if(result) {
  446. return result;
  447. } else {
  448. return nil;
  449. }
  450. }
  451.  
  452. fsdk::IDescriptorPtr descriptorFromNSData(const fsdk::IFaceEngineMobilePtr engine, NSData* data) {
  453. const uint8_t* buffer = static_cast<const uint8_t*>([data bytes]);
  454. std::vector<uint8_t> descriptorData(buffer, buffer + [data length]);
  455. auto resDescriptor = engine->createDescriptor();
  456. if(resDescriptor.isError()) {
  457. OUT("[descriptorFromNSData] Failed to create descriptor instance. " << resDescriptor.what());
  458. return nil;
  459. }
  460. fsdk::IDescriptorPtr descriptor = resDescriptor.getValue();
  461. VectorArchive archiveDescriptor(descriptorData);
  462. descriptor->load(&archiveDescriptor);
  463.  
  464. if(!descriptor)
  465. OUT("Failed to get descriptor from NSData");
  466.  
  467. return descriptor;
  468. }
  469.  
  470. // MARK: Estimators
  471. // quality estimation
  472. // only pass WARPED image - you can get it from warpImage method
  473. /********************************************************/
  474. //- (bool)estimateQualityOf: (UIImage*)warpedImage estimation: (float*) estimation {
  475. // std::lock_guard<decltype(mutex)> guard(mutex);
  476. //
  477. // if(![self isValid]) {
  478. // OUT("Object is not valid (probably you didnt call load())");
  479. // return nil;
  480. // }
  481. //
  482. // // check args correctness
  483. // if(warpedImage == nullptr || estimation == nullptr) {
  484. // OUT("Invalid arguments");
  485. // return false;
  486. // }
  487. //
  488. // fsdk::Image rgbimage = uiimageToVlfImage(warpedImage);
  489. // fsdk::SubjectiveQuality quality;
  490. //
  491. // const auto result = qualityEstimator->estimate(rgbimage, quality);
  492. //
  493. // *estimation = quality.isGood();
  494. //
  495. // if(result.isError()) {
  496. // OUT("Failed to estimate quality. Reason: " << result.what());
  497. // return false;
  498. // }
  499. //
  500. // return true;
  501. //}
  502.  
  503.  
  504. - (float)qualityOf:(UIImage *)warpedImage {
  505. // TODO: 2022!!!
  506. return 1.0;
  507. }
  508.  
  509. - (float)matchDescriptor: (NSData*) firstDescriptorData and: (NSData*) secondDescriptorData {
  510. std::lock_guard<decltype(mutex)> guard(mutex);
  511. float similarity = 0.f;
  512.  
  513. if(!loaded) {
  514. OUT("Not loaded properly");
  515. return similarity;
  516. }
  517.  
  518. const auto firstDescriptor = descriptorFromNSData(engine, firstDescriptorData);
  519. const auto secondDescriptor = descriptorFromNSData(engine, secondDescriptorData);
  520. const auto result = descriptorMatcher->match(firstDescriptor, secondDescriptor);
  521.  
  522. if(result) {
  523. similarity = result.getValue().similarity;
  524. OUT("Descriptors matched with similarity=" << similarity);
  525. }
  526. else {
  527. OUT("Failed to match descriptors: " << result.what());
  528. }
  529.  
  530. return similarity;
  531. }
  532.  
  533. @end
  534.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement