From 0742f17f2a4b37ed4e5385d5dc61cf3265f189df Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Thu, 19 Sep 2024 03:29:43 +0900 Subject: [PATCH] Add the Bitmap Comparison with CRC32 hashing --- .../WelsonJS.Service/ScreenMatch.cs | 60 +++++++++++++------ .../WelsonJS.Service/WelsonJS.Service.csproj | 3 + .../WelsonJS.Service/packages.config | 1 + settings.example.ini | 2 +- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/WelsonJS.Toolkit/WelsonJS.Service/ScreenMatch.cs b/WelsonJS.Toolkit/WelsonJS.Service/ScreenMatch.cs index 4db1984..809de94 100644 --- a/WelsonJS.Toolkit/WelsonJS.Service/ScreenMatch.cs +++ b/WelsonJS.Toolkit/WelsonJS.Service/ScreenMatch.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Drawing; using System.IO; +using System.IO.Hashing; using System.Runtime.InteropServices; using System.ServiceProcess; using System.Text; @@ -14,6 +15,7 @@ using System.Windows.Forms; using System.Linq; using Tesseract; using WelsonJS.Service; +using System.Security.Cryptography; public class ScreenMatch { @@ -114,12 +116,12 @@ public class ScreenMatch private List _params = new List(); private bool isSearchFromEnd = false; private bool isSaveToFile = false; - private bool isUseSampleClipboard = false; - private bool isUseSampleOCR = false; private Size sampleSize; - private int sampleAdjustX = 0; - private int sampleAdjustY = 0; + private int sampleAdjustX; + private int sampleAdjustY; private List sampleAny; + private List sampleClipboard; + private List sampleOcr; private List sampleNodup; private Size sampleNodupSize; private Queue sampleOutdated; @@ -133,13 +135,20 @@ public class ScreenMatch templateDirectoryPath = Path.Combine(workingDirectory, "app/assets/img/_templates"); outputDirectoryPath = Path.Combine(workingDirectory, "app/assets/img/_captured"); - templateImages = new List(); + + // Initialize variables for sampling process sampleSize = new Size { Width = 128, Height = 128 }; + sampleAdjustX = 0; + sampleAdjustY = 0; + sampleAny = new List(); + sampleClipboard = new List(); + sampleOcr = new List(); + sampleNodup = new List(); sampleNodupSize = new Size { Width = 128, @@ -212,8 +221,7 @@ public class ScreenMatch case "sample_clipboard": { - isUseSampleClipboard = true; - this.parent.Log($"Use Clipboard within a {sampleSize.Width}x{sampleSize.Height} pixel range around specific coordinates."); + sampleClipboard = new List(config_value.Split(':')); break; } @@ -221,8 +229,7 @@ public class ScreenMatch { tesseractDataPath = Path.Combine(workingDirectory, "app/assets/tessdata_best"); tesseractLanguage = "eng"; - isUseSampleOCR = true; - this.parent.Log($"Use OCR within a {sampleSize.Width}x{sampleSize.Height} pixel range around specific coordinates."); + sampleOcr = new List(config_value.Split(':')); break; } @@ -395,7 +402,7 @@ public class ScreenMatch } List matchPositions = FindTemplate(_mainImage, (Bitmap)image.Clone()); - matchPositions.ForEach((matchPosition) => + foreach (Point matchPosition in matchPositions) { try { @@ -409,12 +416,14 @@ public class ScreenMatch Position = matchPosition, Text = text }); + + break; // Only one } catch (Exception ex) { parent.Log($"Ignore the match. {ex.Message}"); } - }); + } } if (results.Count > 0) @@ -431,7 +440,7 @@ public class ScreenMatch return results; } - public Bitmap cropBitmap(Bitmap bitmap, Point matchPosition, Size templateSize, Size sampleSize, int dx = 0, int dy = 0) + public Bitmap CropBitmap(Bitmap bitmap, Point matchPosition, Size templateSize, Size sampleSize, int dx = 0, int dy = 0) { // Adjust coordinates to the center int x = matchPosition.X + (templateSize.Width / 2); @@ -464,14 +473,15 @@ public class ScreenMatch string text = ""; // Crop image - Bitmap croppedBitmap = cropBitmap(bitmap, matchPosition, templateSize, sampleSize, sampleAdjustX, sampleAdjustY); + Bitmap croppedBitmap = CropBitmap(bitmap, matchPosition, templateSize, sampleSize, sampleAdjustX, sampleAdjustY); // Save to the outdated samples if (sampleNodup.Contains(templateName)) { - Bitmap croppedNodupBitmap = cropBitmap(bitmap, matchPosition, templateSize, sampleNodupSize); - int bitmapHash = croppedNodupBitmap.GetHashCode(); - bool bitmapExists = sampleOutdated.Any(x => x.GetHashCode() == bitmapHash); + Bitmap croppedNodupBitmap = CropBitmap(bitmap, matchPosition, templateSize, sampleNodupSize); + uint bitmapCrc32 = ComputeBitmapCrc32(croppedNodupBitmap); + parent.Log($"bitmapCrc32: {bitmapCrc32}"); + bool bitmapExists = sampleOutdated.Any(x => ComputeBitmapCrc32(x) == bitmapCrc32); if (bitmapExists) { throw new InvalidOperationException($"This may be a duplicate request. {templateName}"); @@ -484,7 +494,7 @@ public class ScreenMatch } // if use Clipboard - if (isUseSampleClipboard) + if (sampleClipboard.Contains(templateName)) { Thread th = new Thread(new ThreadStart(() => { @@ -503,7 +513,7 @@ public class ScreenMatch } // if use OCR - if (isUseSampleOCR) + if (sampleOcr.Contains(templateName)) { try { @@ -755,4 +765,18 @@ public class ScreenMatch return binaryImage; } + + private uint ComputeBitmapCrc32(Bitmap bitmap) + { + using (MemoryStream ms = new MemoryStream()) + { + bitmap.Save(ms, System.Drawing.Imaging.ImageFormat.Png); + + byte[] bitmapBytes = ms.ToArray(); + Crc32 crc32 = new Crc32(); + crc32.Append(bitmapBytes); + + return BitConverter.ToUInt32(crc32.GetCurrentHash(), 0); + } + } } \ No newline at end of file diff --git a/WelsonJS.Toolkit/WelsonJS.Service/WelsonJS.Service.csproj b/WelsonJS.Toolkit/WelsonJS.Service/WelsonJS.Service.csproj index 33d8bfb..c1bd326 100644 --- a/WelsonJS.Toolkit/WelsonJS.Service/WelsonJS.Service.csproj +++ b/WelsonJS.Toolkit/WelsonJS.Service/WelsonJS.Service.csproj @@ -142,6 +142,9 @@ ..\packages\System.Diagnostics.DiagnosticSource.8.0.1\lib\net462\System.Diagnostics.DiagnosticSource.dll + + ..\packages\System.IO.Hashing.8.0.0\lib\net462\System.IO.Hashing.dll + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll diff --git a/WelsonJS.Toolkit/WelsonJS.Service/packages.config b/WelsonJS.Toolkit/WelsonJS.Service/packages.config index 2a01cb9..7580f55 100644 --- a/WelsonJS.Toolkit/WelsonJS.Service/packages.config +++ b/WelsonJS.Toolkit/WelsonJS.Service/packages.config @@ -15,6 +15,7 @@ + diff --git a/settings.example.ini b/settings.example.ini index df638d9..3cfefe5 100644 --- a/settings.example.ini +++ b/settings.example.ini @@ -13,7 +13,7 @@ DISABLE_SCREEN_TIME=true DISABLE_FILE_MONITOR=true ; window or screen SCREEN_TIME_MODE=screen -; backward,save,sample_ocr,sample_clipboard,sample_width=128,sample_height=128,sample_adjust_x=0,sample_adjust_y=0,sample_only=button.png:message.png,sample_nodup=message.png,process_name=notepad.exe +; backward,save,sample_width=128,sample_height=128,sample_adjust_x=0,sample_adjust_y=0,sample_any=button.png:message.png:like.png,sample_ocr=like.png,sample_clipboard=button.png,sample_nodup=message.png,process_name=notepad.exe SCREEN_TIME_PARAMS= ; default: http://localhost:50051 GRPC_HOST=http://localhost:50051