From 2187612aaa5082b9192f7fb0996b4fe9971db3a9 Mon Sep 17 00:00:00 2001 From: Rafa Ruiz Date: Fri, 11 Oct 2024 19:05:53 +0200 Subject: [PATCH 1/3] Enhance petri-lipponen-movesense's code --- android/build.gradle | 3 + .../mobile_scanner/MobileScanner.kt | 38 ++++++++++- .../mobile_scanner/MobileScannerHandler.kt | 12 ++++ ios/Classes/MobileScanner.swift | 64 +++++++++++++++++-- ios/Classes/MobileScannerPlugin.swift | 19 +++++- .../mobile_scanner_method_channel.dart | 8 +++ lib/src/mobile_scanner_controller.dart | 9 +++ .../mobile_scanner_platform_interface.dart | 5 ++ lib/src/objects/start_options.dart | 5 ++ 9 files changed, 152 insertions(+), 11 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index b53e3e99b..cc8a2c90e 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -80,6 +80,9 @@ dependencies { implementation 'androidx.camera:camera-lifecycle:1.3.3' implementation 'androidx.camera:camera-camera2:1.3.3' + // opencv + implementation 'com.quickbirdstudios:opencv:3.4.15' + testImplementation 'org.jetbrains.kotlin:kotlin-test' testImplementation 'org.mockito:mockito-core:5.12.0' } diff --git a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt index fa6f20c50..4e68c3e8c 100644 --- a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt +++ b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt @@ -36,7 +36,12 @@ import dev.steenbakker.mobile_scanner.utils.YuvToRgbConverter import io.flutter.view.TextureRegistry import java.io.ByteArrayOutputStream import kotlin.math.roundToInt - +import com.google.mlkit.vision.common.internal.ImageConvertUtils +import org.opencv.core.Mat +import org.opencv.core.CvType +import org.opencv.core.Core +import org.opencv.android.Utils +import org.opencv.android.OpenCVLoader class MobileScanner( private val activity: Activity, @@ -57,6 +62,8 @@ class MobileScanner( /// Configurable variables var scanWindow: List? = null + var intervalInvertImage: Boolean = false + var invertImage: Boolean = false private var detectionSpeed: DetectionSpeed = DetectionSpeed.NO_DUPLICATES private var detectionTimeout: Long = 250 private var returnImage = false @@ -67,7 +74,16 @@ class MobileScanner( @ExperimentalGetImage val captureOutput = ImageAnalysis.Analyzer { imageProxy -> // YUV_420_888 format val mediaImage = imageProxy.image ?: return@Analyzer - val inputImage = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees) + var inputImage = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees) + + // Invert + if (intervalInvertImage) { + invertImage = !invertImage + } + + if (invertImage) { + inputImage = invertInputImage(inputImage) + } if (detectionSpeed == DetectionSpeed.NORMAL && scannerTimeout) { imageProxy.close() @@ -218,6 +234,7 @@ class MobileScanner( fun start( barcodeScannerOptions: BarcodeScannerOptions?, returnImage: Boolean, + intervalInvertImage: Boolean, cameraPosition: CameraSelector, torch: Boolean, detectionSpeed: DetectionSpeed, @@ -229,9 +246,13 @@ class MobileScanner( cameraResolution: Size?, newCameraResolutionSelector: Boolean ) { + + OpenCVLoader.initDebug() + this.detectionSpeed = detectionSpeed this.detectionTimeout = detectionTimeout this.returnImage = returnImage + this.intervalInvertImage = intervalInvertImage if (camera?.cameraInfo != null && preview != null && textureEntry != null) { mobileScannerErrorCallback(AlreadyStarted()) @@ -473,5 +494,16 @@ class MobileScanner( if (camera == null) throw ZoomWhenStopped() camera?.cameraControl?.setZoomRatio(1f) } - + /** + * Invert the input image. + */ + fun invertInputImage(image: InputImage): InputImage { + val bitmap = ImageConvertUtils.getInstance().getUpRightBitmap(image); + val tmp = Mat(bitmap.getWidth(), bitmap.getHeight(), CvType.CV_8UC1); + Utils.bitmapToMat(bitmap, tmp); + Core.bitwise_not(tmp, tmp); + Utils.matToBitmap(tmp, bitmap); + val newImage = InputImage.fromBitmap(bitmap, 0); + return newImage + } } diff --git a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScannerHandler.kt b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScannerHandler.kt index ea6a352a7..b1452925a 100644 --- a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScannerHandler.kt +++ b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScannerHandler.kt @@ -128,6 +128,7 @@ class MobileScannerHandler( "setScale" -> setScale(call, result) "resetScale" -> resetScale(result) "updateScanWindow" -> updateScanWindow(call, result) + "setIntervalInvertImage" -> setIntervalInvertImage(call, result) else -> result.notImplemented() } } @@ -138,6 +139,7 @@ class MobileScannerHandler( val facing: Int = call.argument("facing") ?: 0 val formats: List? = call.argument>("formats") val returnImage: Boolean = call.argument("returnImage") ?: false + val intervalInvertImage: Boolean = call.argument("intervalInvertImage") ?: false val speed: Int = call.argument("speed") ?: 1 val timeout: Int = call.argument("timeout") ?: 250 val cameraResolutionValues: List? = call.argument>("cameraResolution") @@ -174,6 +176,7 @@ class MobileScannerHandler( mobileScanner!!.start( barcodeScannerOptions, returnImage, + intervalInvertImage, position, torch, detectionSpeed, @@ -275,4 +278,13 @@ class MobileScannerHandler( result.success(null) } + + private fun setIntervalInvertImage(call: MethodCall, result: MethodChannel.Result) { + val intervalInvertImage = call.argument("intervalInvertImage") + + if (intervalInvertImage != null) + mobileScanner?.intervalInvertImage = intervalInvertImage + + result.success(null) + } } diff --git a/ios/Classes/MobileScanner.swift b/ios/Classes/MobileScanner.swift index e9c066f25..104d8f9c0 100644 --- a/ios/Classes/MobileScanner.swift +++ b/ios/Classes/MobileScanner.swift @@ -28,6 +28,12 @@ public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelega /// Return image buffer with the Barcode event var returnImage: Bool = false + /// Analyze inverted image (useful for scanning white barcodes on black background) and inverted at the same time + var intervalInvertImage: Bool = false + + /// If [intervalInvertImage] is true, this will interval between true and false on each image so we can detect both images + var invertImage: Bool = false + /// Default position of camera var videoPosition: AVCaptureDevice.Position = AVCaptureDevice.Position.back @@ -124,6 +130,14 @@ public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelega AVCaptureDevice.requestAccess(for: .video, completionHandler: { result($0) }) } + func convertCIImageToCGImage(inputImage: CIImage) -> CGImage? { + let context = CIContext(options: nil) + if let cgImage = context.createCGImage(inputImage, from: inputImage.extent) { + return cgImage + } + return nil + } + /// Gets called when a new image is added to the buffer public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { @@ -141,9 +155,22 @@ public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelega nextScanTime = currentTime + timeoutSeconds imagesCurrentlyBeingProcessed = true - let ciImage = latestBuffer.image + let uiImage : UIImage + if (intervalInvertImage) { + invertImage = !invertImage + } + + if (invertImage) { + let temp_image = self.invertImage(image: latestBuffer.image) + uiImage = temp_image + } + else + { + uiImage = latestBuffer.image + } - let image = VisionImage(image: ciImage) + var image = VisionImage(image: uiImage) + image.orientation = imageOrientation( deviceOrientation: UIDevice.current.orientation, defaultOrientation: .portrait, @@ -165,19 +192,20 @@ public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelega } } - mobileScannerCallback(barcodes, error, ciImage) + mobileScannerCallback(barcodes, error, uiImage) } } } /// Start scanning for barcodes - func start(barcodeScannerOptions: BarcodeScannerOptions?, returnImage: Bool, cameraPosition: AVCaptureDevice.Position, torch: Bool, detectionSpeed: DetectionSpeed, completion: @escaping (MobileScannerStartParameters) -> ()) throws { + func start(barcodeScannerOptions: BarcodeScannerOptions?, returnImage: Bool, intervalInvertImage: Bool, cameraPosition: AVCaptureDevice.Position, torch: Bool, detectionSpeed: DetectionSpeed, completion: @escaping (MobileScannerStartParameters) -> ()) throws { self.detectionSpeed = detectionSpeed if (device != nil || captureSession != nil) { throw MobileScannerError.alreadyStarted } barcodesString = nil + self.intervalInvertImage = intervalInvertImage scanner = barcodeScannerOptions != nil ? BarcodeScanner.barcodeScanner(options: barcodeScannerOptions!) : BarcodeScanner.barcodeScanner() captureSession = AVCaptureSession() textureId = registry?.register(self) @@ -390,6 +418,10 @@ public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelega } } + func setIntervalInvertImage(_ intervalInvertImage: Bool) { + self.intervalInvertImage = intervalInvertImage + } + /// Set the zoom factor of the camera func setScale(_ scale: CGFloat) throws { if (device == nil) { @@ -434,14 +466,32 @@ public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelega /// Analyze a single image func analyzeImage(image: UIImage, position: AVCaptureDevice.Position, callback: @escaping BarcodeScanningCallback) { - let image = VisionImage(image: image) - image.orientation = imageOrientation( + var uiimage = image + + if (intervalInvertImage) { + invertImage = !invertImage + } + + if (invertImage) { + uiimage = self.invertImage(image: uiimage) + } + let visImage = VisionImage(image: uiimage) + visImage.orientation = imageOrientation( deviceOrientation: UIDevice.current.orientation, defaultOrientation: .portrait, position: position ) + scanner.process(visImage, completion: callback) + } - scanner.process(image, completion: callback) + func invertImage(image: UIImage) -> UIImage { + let ciImage = CIImage(image: image) + let filter = CIFilter(name: "CIColorInvert") + filter?.setValue(ciImage, forKey: kCIInputImageKey) + let outputImage = filter?.outputImage + let cgImage = convertCIImageToCGImage(inputImage: outputImage!) + + return UIImage(cgImage: cgImage!, scale: image.scale, orientation: image.imageOrientation) } var barcodesString: Array? diff --git a/ios/Classes/MobileScannerPlugin.swift b/ios/Classes/MobileScannerPlugin.swift index eba359a36..1826d8153 100644 --- a/ios/Classes/MobileScannerPlugin.swift +++ b/ios/Classes/MobileScannerPlugin.swift @@ -84,6 +84,8 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin { toggleTorch(result) case "analyzeImage": analyzeImage(call, result) + case "setIntervalInvertImage": + setIntervalInvertImage(call, result) case "setScale": setScale(call, result) case "resetScale": @@ -101,6 +103,7 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin { let facing: Int = (call.arguments as! Dictionary)["facing"] as? Int ?? 1 let formats: Array = (call.arguments as! Dictionary)["formats"] as? Array ?? [] let returnImage: Bool = (call.arguments as! Dictionary)["returnImage"] as? Bool ?? false + let intervalInvertImage: Bool = (call.arguments as! Dictionary)["intervalInvertImage"] as? Bool ?? false let speed: Int = (call.arguments as! Dictionary)["speed"] as? Int ?? 0 let timeoutMs: Int = (call.arguments as! Dictionary)["timeout"] as? Int ?? 0 self.mobileScanner.timeoutSeconds = Double(timeoutMs) / Double(1000) @@ -120,7 +123,7 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin { let detectionSpeed: DetectionSpeed = DetectionSpeed(rawValue: speed)! do { - try mobileScanner.start(barcodeScannerOptions: barcodeOptions, returnImage: returnImage, cameraPosition: position, torch: torch, detectionSpeed: detectionSpeed) { parameters in + try mobileScanner.start(barcodeScannerOptions: barcodeOptions, returnImage: returnImage, intervalInvertImage: intervalInvertImage, cameraPosition: position, torch: torch, detectionSpeed: detectionSpeed) { parameters in DispatchQueue.main.async { result([ "textureId": parameters.textureId, @@ -162,6 +165,20 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin { result(nil) } + + /// Sets interval inverting + private func setIntervalInvertImage(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { + let intervalInvertImage = call.arguments as? Bool + if (intervalInvertImage == nil) { + result(FlutterError(code: "MobileScanner", + message: "You must provide a invert (bool) when calling setIntervalInvertImage", + details: nil)) + return + } + mobileScanner.setIntervalInvertImage(invert!) + result(nil) + } + /// Sets the zoomScale. private func setScale(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { let scale = call.arguments as? CGFloat diff --git a/lib/src/method_channel/mobile_scanner_method_channel.dart b/lib/src/method_channel/mobile_scanner_method_channel.dart index 1cc937cfd..6d6b7e02b 100644 --- a/lib/src/method_channel/mobile_scanner_method_channel.dart +++ b/lib/src/method_channel/mobile_scanner_method_channel.dart @@ -295,6 +295,14 @@ class MethodChannelMobileScanner extends MobileScannerPlatform { ); } + @override + Future setIntervalInvertImage(bool intervalInvertImage) async { + await methodChannel.invokeMethod( + 'setIntervalInvertImage', + {'intervalInvertImage': intervalInvertImage}, + ); + } + @override Future dispose() async { await stop(); diff --git a/lib/src/mobile_scanner_controller.dart b/lib/src/mobile_scanner_controller.dart index f01743b45..834f42aef 100644 --- a/lib/src/mobile_scanner_controller.dart +++ b/lib/src/mobile_scanner_controller.dart @@ -25,6 +25,7 @@ class MobileScannerController extends ValueNotifier { this.formats = const [], this.returnImage = false, this.torchEnabled = false, + this.intervalInvertImage = false, this.useNewCameraSelector = false, }) : detectionTimeoutMs = detectionSpeed == DetectionSpeed.normal ? detectionTimeoutMs : 0, @@ -88,6 +89,13 @@ class MobileScannerController extends ValueNotifier { /// Defaults to false. final bool torchEnabled; + /// Whether the image should be inverted on a 1 to 1 interval. + /// So images are normal - inverted - normal - inverted + /// and we can read both kind of images. + /// + /// Defaults to false. + final bool intervalInvertImage; + /// Use the new resolution selector. /// /// This feature is experimental and not fully tested yet. @@ -268,6 +276,7 @@ class MobileScannerController extends ValueNotifier { formats: formats, returnImage: returnImage, torchEnabled: torchEnabled, + intervalInvertImage: intervalInvertImage, ); try { diff --git a/lib/src/mobile_scanner_platform_interface.dart b/lib/src/mobile_scanner_platform_interface.dart index 553602160..282e991ea 100644 --- a/lib/src/mobile_scanner_platform_interface.dart +++ b/lib/src/mobile_scanner_platform_interface.dart @@ -104,6 +104,11 @@ abstract class MobileScannerPlatform extends PlatformInterface { throw UnimplementedError('updateScanWindow() has not been implemented.'); } + /// Set inverting image colors (for negative Data Matrixes) on intervals. + Future setIntervalInvertImage(bool intervalInvertImage) { + throw UnimplementedError('setIntervalInvertImage() has not been implemented.'); + } + /// Dispose of this [MobileScannerPlatform] instance. Future dispose() { throw UnimplementedError('dispose() has not been implemented.'); diff --git a/lib/src/objects/start_options.dart b/lib/src/objects/start_options.dart index 71f368d69..4cf58dd61 100644 --- a/lib/src/objects/start_options.dart +++ b/lib/src/objects/start_options.dart @@ -14,6 +14,7 @@ class StartOptions { required this.formats, required this.returnImage, required this.torchEnabled, + required this.intervalInvertImage, }); /// The direction for the camera. @@ -37,6 +38,9 @@ class StartOptions { /// Whether the torch should be turned on when the scanner starts. final bool torchEnabled; + /// Whether the image should be inverted on intervals. + final bool intervalInvertImage; + Map toMap() { return { if (cameraResolution != null) @@ -51,6 +55,7 @@ class StartOptions { 'speed': detectionSpeed.rawValue, 'timeout': detectionTimeoutMs, 'torch': torchEnabled, + 'intervalInvertImage': intervalInvertImage, }; } } From 9ef449f609edefdeddc6e7a982c4a56e8c3fec94 Mon Sep 17 00:00:00 2001 From: Rafa Ruiz Date: Fri, 11 Oct 2024 19:28:40 +0200 Subject: [PATCH 2/3] Missed the bool --- ios/Classes/MobileScannerPlugin.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/Classes/MobileScannerPlugin.swift b/ios/Classes/MobileScannerPlugin.swift index 1826d8153..3df2fcca8 100644 --- a/ios/Classes/MobileScannerPlugin.swift +++ b/ios/Classes/MobileScannerPlugin.swift @@ -175,7 +175,7 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin { details: nil)) return } - mobileScanner.setIntervalInvertImage(invert!) + mobileScanner.setIntervalInvertImage(intervalInvertImage!) result(nil) } From 041d7f024d2e238b5a99627332e8072b2c77ef1f Mon Sep 17 00:00:00 2001 From: Rafa Ruiz Date: Sat, 12 Oct 2024 00:32:37 +0200 Subject: [PATCH 3/3] Invert properly --- .../mobile_scanner/MobileScanner.kt | 32 ++++++++++++++----- lib/src/mobile_scanner_controller.dart | 2 +- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt index 4e68c3e8c..07b2965b6 100644 --- a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt +++ b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt @@ -497,13 +497,29 @@ class MobileScanner( /** * Invert the input image. */ - fun invertInputImage(image: InputImage): InputImage { - val bitmap = ImageConvertUtils.getInstance().getUpRightBitmap(image); - val tmp = Mat(bitmap.getWidth(), bitmap.getHeight(), CvType.CV_8UC1); - Utils.bitmapToMat(bitmap, tmp); - Core.bitwise_not(tmp, tmp); - Utils.matToBitmap(tmp, bitmap); - val newImage = InputImage.fromBitmap(bitmap, 0); - return newImage + private fun invertInputImage(image: InputImage): InputImage { + val bitmap = ImageConvertUtils.getInstance().getUpRightBitmap(image) + + val mat = Mat() + Utils.bitmapToMat(bitmap, mat) + + val channels = ArrayList(4) + Core.split(mat, channels) + + // Invert only RGB channels + for (i in 0..2) { + Core.bitwise_not(channels[i], channels[i]) + } + + // Merge channels back + Core.merge(channels, mat) + + Utils.matToBitmap(mat, bitmap) + mat.release() + for (channel in channels) { + channel.release() + } + + return InputImage.fromBitmap(bitmap, 0) } } diff --git a/lib/src/mobile_scanner_controller.dart b/lib/src/mobile_scanner_controller.dart index 834f42aef..0663aba7c 100644 --- a/lib/src/mobile_scanner_controller.dart +++ b/lib/src/mobile_scanner_controller.dart @@ -90,7 +90,7 @@ class MobileScannerController extends ValueNotifier { final bool torchEnabled; /// Whether the image should be inverted on a 1 to 1 interval. - /// So images are normal - inverted - normal - inverted + /// So images are normal - inverted - normal - inverted... /// and we can read both kind of images. /// /// Defaults to false.