Add option ocr128 to the screen time feature

WelsonJS.Toolkit/WelsonJS.Service/Model/ScreenMatchResult.cs

@@ -13,6 +13,7 @@ namespace WelsonJS.Service
         public Point WindowPosition { get; set; }
         public Point Position { get; set; }
         public double MaxCorrelation { get; set; }
+        public string Text { get; set; }
 
         public override string ToString()
         {

WelsonJS.Toolkit/WelsonJS.Service/ScreenMatch.cs

@@ -1,5 +1,6 @@
 // ScreenMatching.cs
 // https://github.com/gnh1201/welsonjs
+// https://github.com/gnh1201/welsonjs/wiki/Screen-Time-Feature
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -9,6 +10,8 @@ using System.Runtime.InteropServices;
 using System.ServiceProcess;
 using System.Text;
 using System.Windows.Forms;
+using Tesseract;
+using WelsonJS.Cryptography;
 using WelsonJS.Service;
 
 public class ScreenMatch
@@ -111,6 +114,9 @@ public class ScreenMatch
     private byte thresholdConvertToBinary = 255;
     private bool isSaveToFile = false;
     private bool isMatching = false;
+    private bool isOCR128 = false;
+    private string tesseractDataPath = @"./tessdata";
+    private string tesseractLanguage = "eng";
 
     public ScreenMatch(ServiceBase parent, string workingDirectory)
     {
@@ -154,6 +160,12 @@ public class ScreenMatch
             this.parent.Log("Will be save an image file when capture the screens");
         }
 
+        if (_params.Contains("ocr128"))
+        {
+            isOCR128 = true;
+            this.parent.Log("Use OCR within a 128x128 pixel range around specific coordinates.");
+        }
+
         SetMode(screen_time_mode);
         LoadTemplateImages();
     }
@@ -269,20 +281,37 @@ public class ScreenMatch
             parent.Log($"Trying match the template {image.Tag as string} on the screen {i}...");
 
             string filename = image.Tag as string;
+
+            Bitmap _mainImage;
             if (filename.StartsWith("binary_"))
             {
-                mainImage = ConvertToBinary((Bitmap)mainImage.Clone(), thresholdConvertToBinary);
+                _mainImage = ConvertToBinary((Bitmap)mainImage.Clone(), thresholdConvertToBinary);
+            }
+            else
+            {
+                _mainImage = mainImage;
             }
 
-            Point matchPosition = FindTemplate(mainImage, (Bitmap)image.Clone(), out double maxCorrelation);
+            Point matchPosition = FindTemplate(_mainImage, (Bitmap)image.Clone(), out double maxCorrelation);
             if (matchPosition != Point.Empty)
             {
+                string text;
+                if (isOCR128)
+                {
+                    text = OCR((Bitmap)mainImage.Clone(), matchPosition.X, matchPosition.Y, 128, 128);
+                }
+                else
+                {
+                    text = "";
+                }
+
                 results.Add(new ScreenMatchResult
                 {
                     FileName = image.Tag.ToString(),
                     ScreenNumber = i,
                     Position = matchPosition,
-                    MaxCorrelation = maxCorrelation
+                    MaxCorrelation = maxCorrelation,
+                    Text = text
                 });
             }
         }
@@ -301,6 +330,35 @@ public class ScreenMatch
         return results;
     }
 
+    public string OCR(Bitmap bitmap, int x, int y, int w, int h)
+    {
+        string text = "";
+
+        int cropX = Math.Max(x - w / 2, 0);
+        int cropY = Math.Max(y - h / 2, 0);
+        int cropWidth = Math.Min(w, bitmap.Width - cropX);
+        int cropHeight = Math.Min(h, bitmap.Height - cropY);
+
+        Rectangle cropArea = new Rectangle(cropX, cropY, cropWidth, cropHeight);
+        Bitmap croppedBitmap = bitmap.Clone(cropArea, bitmap.PixelFormat);
+
+        using (var engine = new TesseractEngine(tesseractDataPath, tesseractLanguage, EngineMode.Default))
+        {
+            using (var img = PixConverter.ToPix(croppedBitmap))
+            {
+                using (var page = engine.Process(img))
+                {
+                    text = page.GetText();
+
+                    parent.Log($"Mean confidence: {page.GetMeanConfidence()}");
+                    parent.Log($"Text (GetText): {text}");
+                }
+            }
+        }
+
+        return text;
+    }
+
     public Bitmap CaptureScreen(Screen screen)
     {
         Rectangle screenSize = screen.Bounds;

WelsonJS.Toolkit/WelsonJS.Service/WelsonJS.Service.csproj

@@ -171,6 +171,9 @@
     <Reference Include="System.Web" />
     <Reference Include="System.Windows.Forms" />
     <Reference Include="System.Xml" />
+    <Reference Include="Tesseract, Version=5.2.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Tesseract.5.2.0\lib\net48\Tesseract.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="FileEventMonitor.cs" />
@@ -245,7 +248,9 @@
     <Error Condition="!Exists('..\packages\Grpc.Core.2.46.6\build\net45\Grpc.Core.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Grpc.Core.2.46.6\build\net45\Grpc.Core.targets'))" />
     <Error Condition="!Exists('..\packages\Grpc.Tools.2.66.0\build\Grpc.Tools.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Grpc.Tools.2.66.0\build\Grpc.Tools.props'))" />
     <Error Condition="!Exists('..\packages\Grpc.Tools.2.66.0\build\Grpc.Tools.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Grpc.Tools.2.66.0\build\Grpc.Tools.targets'))" />
+    <Error Condition="!Exists('..\packages\Tesseract.5.2.0\build\Tesseract.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Tesseract.5.2.0\build\Tesseract.targets'))" />
   </Target>
   <Import Project="..\packages\Grpc.Core.2.46.6\build\net45\Grpc.Core.targets" Condition="Exists('..\packages\Grpc.Core.2.46.6\build\net45\Grpc.Core.targets')" />
   <Import Project="..\packages\Grpc.Tools.2.66.0\build\Grpc.Tools.targets" Condition="Exists('..\packages\Grpc.Tools.2.66.0\build\Grpc.Tools.targets')" />
+  <Import Project="..\packages\Tesseract.5.2.0\build\Tesseract.targets" Condition="Exists('..\packages\Tesseract.5.2.0\build\Tesseract.targets')" />
 </Project>

WelsonJS.Toolkit/WelsonJS.Service/packages.config

@@ -23,4 +23,5 @@
   <package id="System.Text.Json" version="8.0.4" targetFramework="net48" />
   <package id="System.Threading.Tasks.Extensions" version="4.5.4" targetFramework="net48" />
   <package id="System.ValueTuple" version="4.5.0" targetFramework="net48" />
+  <package id="Tesseract" version="5.2.0" targetFramework="net48" />
 </packages>