gnh1201/welsonjs
1
0
Fork 0
mirror of https://github.com/gnh1201/welsonjs.git synced 2024-11-26 15:31:42 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Save file in the droptop coffee

Browse Source
This commit is contained in:
Namhyeon Go 2024-09-18 14:33:15 +09:00
parent 83efe8a156
commit 27e9a328d4
6 changed files with 120 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
WelsonJS.Toolkit/WelsonJS.Service/Model/ScreenMatchResult.cs
View File

@ -12,14 +12,13 @@ namespace WelsonJS.Service
public string ProcessName { get; set; } public string ProcessName { get; set; }
public Point WindowPosition { get; set; } public Point WindowPosition { get; set; }
public Point Position { get; set; } public Point Position { get; set; }
public double MaxCorrelation { get; set; }
public string Text { get; set; } public string Text { get; set; }
public override string ToString() public override string ToString()
{ {
return $"Template: {FileName}, Screen Number: {ScreenNumber}, Window Title: {WindowTitle}, " + return $"Template: {FileName}, Screen Number: {ScreenNumber}, Window Title: {WindowTitle}, " +
$"Process Name: {ProcessName}, Window Position: (x: {WindowPosition.X}, y: {WindowPosition.Y}), " + $"Process Name: {ProcessName}, Window Position: (x: {WindowPosition.X}, y: {WindowPosition.Y}), " +
$"Location: (x: {Position.X}, y: {Position.Y}), Max Correlation: {MaxCorrelation}"; $"Location: (x: {Position.X}, y: {Position.Y})";
} }
} }
} }

165
WelsonJS.Toolkit/WelsonJS.Service/ScreenMatch.cs
View File

@ -116,11 +116,12 @@ public class ScreenMatch
private bool isSaveToFile = false; private bool isSaveToFile = false;
private bool isUseSampleClipboard = false; private bool isUseSampleClipboard = false;
private bool isUseSampleOCR = false; private bool isUseSampleOCR = false;
private int sampleWidth = 128; private Size sampleSize;
private int sampleHeight = 128;
private int sampleAdjustX = 0; private int sampleAdjustX = 0;
private int sampleAdjustY = 0; private int sampleAdjustY = 0;
private string sampleOnly = ""; private List<string> sampleAny;
private List<string> sampleNodup;
private Queue<Bitmap> sampleOutdated;
private byte thresholdConvertToBinary = 191; private byte thresholdConvertToBinary = 191;
private string tesseractDataPath; private string tesseractDataPath;
private string tesseractLanguage; private string tesseractLanguage;
@ -128,9 +129,17 @@ public class ScreenMatch
public ScreenMatch(ServiceBase parent, string workingDirectory) public ScreenMatch(ServiceBase parent, string workingDirectory)
{ {
this.parent = (ServiceMain)parent; this.parent = (ServiceMain)parent;
templateDirectoryPath = Path.Combine(workingDirectory, "app/assets/img/_templates"); templateDirectoryPath = Path.Combine(workingDirectory, "app/assets/img/_templates");
outputDirectoryPath = Path.Combine(workingDirectory, "app/assets/img/_captured"); outputDirectoryPath = Path.Combine(workingDirectory, "app/assets/img/_captured");
templateImages = new List<Bitmap>(); templateImages = new List<Bitmap>();
sampleSize = new Size
{
Width = 128,
Height = 128
};
sampleOutdated = new Queue<Bitmap>();
// Read values from configration file // Read values from configration file
string screen_time_mode; string screen_time_mode;
@ -164,7 +173,8 @@ public class ScreenMatch
"sample_height", "sample_height",
"sample_adjust_x", "sample_adjust_x",
"sample_adjust_y", "sample_adjust_y",
"sample_only", "sample_any",
"sample_nodup",
"backward", "backward",
"save", "save",
"sample_clipboard", "sample_clipboard",
@ -197,7 +207,7 @@ public class ScreenMatch
case "sample_clipboard": case "sample_clipboard":
{ {
isUseSampleClipboard = true; isUseSampleClipboard = true;
this.parent.Log($"Use Clipboard within a {sampleWidth}x{sampleHeight} pixel range around specific coordinates."); this.parent.Log($"Use Clipboard within a {sampleSize.Width}x{sampleSize.Height} pixel range around specific coordinates.");
break; break;
} }
@ -206,19 +216,21 @@ public class ScreenMatch
tesseractDataPath = Path.Combine(workingDirectory, "app/assets/tessdata_best"); tesseractDataPath = Path.Combine(workingDirectory, "app/assets/tessdata_best");
tesseractLanguage = "eng"; tesseractLanguage = "eng";
isUseSampleOCR = true; isUseSampleOCR = true;
this.parent.Log($"Use OCR within a {sampleWidth}x{sampleHeight} pixel range around specific coordinates."); this.parent.Log($"Use OCR within a {sampleSize.Width}x{sampleSize.Height} pixel range around specific coordinates.");
break; break;
} }
case "sample_width": case "sample_width":
{ {
int.TryParse(config_value, out sampleWidth); int.TryParse(config_value, out int w);
sampleSize.Width = w;
break; break;
} }
case "sample_height": case "sample_height":
{ {
int.TryParse(config_value, out sampleHeight); int.TryParse(config_value, out int h);
sampleSize.Height = h;
break; break;
} }
@ -234,9 +246,15 @@ public class ScreenMatch
break; break;
} }
case "sample_only": case "sample_any":
{ {
sampleOnly = config_value; sampleAny = new List<string>(config_value.Split(':'));
break;
}
case "sample_nodup":
{
sampleNodup = new List<string>(config_value.Split(':'));
break; break;
} }
} }
@ -353,12 +371,15 @@ public class ScreenMatch
Bitmap image = templateImages[templateCurrentIndex]; Bitmap image = templateImages[templateCurrentIndex];
parent.Log($"Trying match the template {image.Tag as string} on the screen {i}..."); parent.Log($"Trying match the template {image.Tag as string} on the screen {i}...");
string filename = image.Tag as string; string templateName = image.Tag as string;
int imageWidth = image.Width; Size templateSize = new Size
int imageHeight = image.Height; {
Width = image.Width,
Height = image.Height
};
Bitmap _mainImage; Bitmap _mainImage;
if (filename.StartsWith("binary_")) if (templateName.StartsWith("binary_"))
{ {
_mainImage = ConvertToBinary((Bitmap)mainImage.Clone(), thresholdConvertToBinary); _mainImage = ConvertToBinary((Bitmap)mainImage.Clone(), thresholdConvertToBinary);
} }
@ -367,29 +388,27 @@ public class ScreenMatch
_mainImage = mainImage; _mainImage = mainImage;
} }
Point matchPosition = FindTemplate(_mainImage, (Bitmap)image.Clone(), out double maxCorrelation); List<Point> matchPositions = FindTemplate(_mainImage, (Bitmap)image.Clone());
if (matchPosition != Point.Empty) matchPositions.ForEach((matchPosition) =>
{ {
string text = ""; try
{
string text = sampleAny.Contains(templateName) ?
InspectSample((Bitmap)mainImage.Clone(), matchPosition, templateSize, templateName, sampleSize) : string.Empty;
if (String.IsNullOrEmpty(sampleOnly) || (!String.IsNullOrEmpty(sampleOnly) && sampleOnly == filename)) results.Add(new ScreenMatchResult
{ {
text = InspectSample((Bitmap)mainImage.Clone(), matchPosition, imageWidth, imageHeight, sampleWidth, sampleHeight); FileName = image.Tag.ToString(),
ScreenNumber = i,
Position = matchPosition,
Text = text
});
} }
else catch (Exception ex)
{ {
parent.Log("Skipped inspect the image sample."); parent.Log($"Ignore the match. {ex.Message}");
} }
});
results.Add(new ScreenMatchResult
{
FileName = image.Tag.ToString(),
ScreenNumber = i,
Position = matchPosition,
MaxCorrelation = maxCorrelation,
Text = text
});
}
} }
if (results.Count > 0) if (results.Count > 0)
@ -406,7 +425,7 @@ public class ScreenMatch
return results; return results;
} }
public string InspectSample(Bitmap bitmap, Point matchPosition, int a, int b, int w, int h) public string InspectSample(Bitmap bitmap, Point matchPosition, Size templateSize, string templateName, Size sampleSize)
{ {
if (bitmap == null) if (bitmap == null)
{ {
@ -415,26 +434,43 @@ public class ScreenMatch
if (matchPosition == null || matchPosition == Point.Empty) if (matchPosition == null || matchPosition == Point.Empty)
{ {
throw new ArgumentNullException("matchPosition cannot be empty."); throw new ArgumentException("matchPosition cannot be empty.");
} }
// initial text // initial text
string text = ""; string text = "";
// Adjust coordinates // Adjust coordinates
int x = matchPosition.X + (a / 2); int positionX = matchPosition.X + (templateSize.Width / 2);
int y = matchPosition.Y + (b / 2); int positionY = matchPosition.Y + (templateSize.Height / 2);
// Set range of crop image // Set range of crop image
int cropX = Math.Max((x - w / 2) + sampleAdjustX, 0); int cropX = Math.Max((positionX - sampleSize.Width / 2) + sampleAdjustX, 0);
int cropY = Math.Max((y - h / 2) + sampleAdjustY, 0); int cropY = Math.Max((positionY - sampleSize.Height / 2) + sampleAdjustY, 0);
int cropWidth = Math.Min(w, bitmap.Width - cropX); int cropWidth = Math.Min(sampleSize.Width, bitmap.Width - cropX);
int cropHeight = Math.Min(h, bitmap.Height - cropY); int cropHeight = Math.Min(sampleSize.Height, bitmap.Height - cropY);
Rectangle cropArea = new Rectangle(cropX, cropY, cropWidth, cropHeight); Rectangle cropArea = new Rectangle(cropX, cropY, cropWidth, cropHeight);
// Crop image // Crop image
Bitmap croppedBitmap = bitmap.Clone(cropArea, bitmap.PixelFormat); Bitmap croppedBitmap = bitmap.Clone(cropArea, bitmap.PixelFormat);
// Save to the outdated samples
if (sampleNodup.Contains(templateName))
{
int croppedBitmapHash = croppedBitmap.GetHashCode();
bool bitmapExists = sampleOutdated.Any(x => x.GetHashCode() == croppedBitmapHash);
if (bitmapExists)
{
throw new InvalidOperationException($"This may be a duplicate request. {templateName}");
}
else
{
sampleOutdated.Enqueue((Bitmap)croppedBitmap.Clone());
parent.Log($"Added to the image queue. {templateName}");
}
}
// if use Clipboard // if use Clipboard
if (isUseSampleClipboard) if (isUseSampleClipboard)
{ {
@ -447,7 +483,7 @@ public class ScreenMatch
} }
catch (Exception ex) catch (Exception ex)
{ {
parent.Log($"Error in Clipboard: {ex.Message}"); parent.Log($"Failed to copy to the clipboard: {ex.Message}");
} }
})); }));
th.SetApartmentState(ApartmentState.STA); th.SetApartmentState(ApartmentState.STA);
@ -472,7 +508,7 @@ public class ScreenMatch
} }
catch (Exception ex) catch (Exception ex)
{ {
parent.Log($"Error in OCR: {ex.Message}"); parent.Log($"Failed to OCR: {ex.Message}");
} }
} }
@ -517,29 +553,31 @@ public class ScreenMatch
string windowTitle = GetWindowTitle(hWnd); string windowTitle = GetWindowTitle(hWnd);
string processName = GetProcessName(hWnd); string processName = GetProcessName(hWnd);
GetWindowRect(hWnd, out RECT windowRect); GetWindowRect(hWnd, out RECT windowRect);
Point windowPosition = new Point(windowRect.Left, windowRect.Top); // 창 위치 계산 Point windowPosition = new Point(windowRect.Left, windowRect.Top);
Bitmap windowImage = CaptureWindow(hWnd); Bitmap windowImage = CaptureWindow(hWnd);
if (windowImage != null) if (windowImage != null)
{ {
Bitmap image = templateImages[templateCurrentIndex]; Bitmap image = templateImages[templateCurrentIndex];
Point matchPosition = FindTemplate(windowImage, image, out double maxCorrelation); List<Point> matchPositions = FindTemplate(windowImage, image);
string templateFileName = image.Tag as string; matchPositions.ForEach((matchPosition) =>
var result = new ScreenMatchResult
{ {
FileName = templateFileName, string templateName = image.Tag as string;
WindowHandle = hWnd, results.Add(new ScreenMatchResult
WindowTitle = windowTitle, {
ProcessName = processName, FileName = templateName,
WindowPosition = windowPosition, WindowHandle = hWnd,
Position = matchPosition, WindowTitle = windowTitle,
MaxCorrelation = maxCorrelation ProcessName = processName,
}; WindowPosition = windowPosition,
results.Add(result); Position = matchPosition
});
});
} }
} }
catch { } catch (Exception ex) {
parent.Log($"Error {ex.Message}");
}
} }
return true; return true;
}, IntPtr.Zero); }, IntPtr.Zero);
@ -586,16 +624,15 @@ public class ScreenMatch
return success ? bitmap : null; return success ? bitmap : null;
} }
public Point FindTemplate(Bitmap mainImage, Bitmap templateImage, out double maxCorrelation) public List<Point> FindTemplate(Bitmap mainImage, Bitmap templateImage)
{ {
var matches = new List<Point>();
int mainWidth = mainImage.Width; int mainWidth = mainImage.Width;
int mainHeight = mainImage.Height; int mainHeight = mainImage.Height;
int templateWidth = templateImage.Width; int templateWidth = templateImage.Width;
int templateHeight = templateImage.Height; int templateHeight = templateImage.Height;
Point bestMatch = Point.Empty;
maxCorrelation = 0;
int startX = isSearchFromEnd ? mainWidth - templateWidth : 0; int startX = isSearchFromEnd ? mainWidth - templateWidth : 0;
int endX = isSearchFromEnd ? -1 : mainWidth - templateWidth + 1; int endX = isSearchFromEnd ? -1 : mainWidth - templateWidth + 1;
int stepX = isSearchFromEnd ? -1 : 1; int stepX = isSearchFromEnd ? -1 : 1;
@ -610,14 +647,12 @@ public class ScreenMatch
{ {
if (IsTemplateMatch(mainImage, templateImage, x, y, threshold)) if (IsTemplateMatch(mainImage, templateImage, x, y, threshold))
{ {
bestMatch = new Point(x, y); matches.Add(new Point(x, y));
maxCorrelation = 1.0;
return bestMatch;
} }
} }
} }
return bestMatch; return matches;
} }
private void toggleBusy() private void toggleBusy()

17
WelsonJS.Toolkit/WelsonJS.Service/ServiceMain.cs
View File

@ -349,16 +349,13 @@ namespace WelsonJS.Service
List<ScreenMatchResult> matchedResults = screenMatcher.CaptureAndMatch(); List<ScreenMatchResult> matchedResults = screenMatcher.CaptureAndMatch();
matchedResults.ForEach(result => matchedResults.ForEach(result =>
{ {
if (result.MaxCorrelation > 0.0) { Log(DispatchServiceEvent("screenTime", new string[]
Log(DispatchServiceEvent("screenTime", new string[] {
{ result.FileName,
result.FileName, result.ScreenNumber.ToString(),
result.ScreenNumber.ToString(), result.Position.X.ToString(),
result.Position.X.ToString(), result.Position.Y.ToString()
result.Position.Y.ToString(), }));
result.MaxCorrelation.ToString()
}));
}
}); });
} }
catch (Exception ex) catch (Exception ex)

6
lib/fsm.js
View File

@ -34,7 +34,7 @@ FSM.prototype.addState = function(stateName) {
}; };
// Method to add a transition between states // Method to add a transition between states
FSM.prototype.addTransition = function(eventType, eventListener, fromStateName, toStateName) { FSM.prototype.addTransition = function(fromStateName, toStateName, eventType, eventListener) {
if (!(fromStateName in this.states) || !(toStateName in this.states)) { if (!(fromStateName in this.states) || !(toStateName in this.states)) {
console.error('Invalid state names provided.'); console.error('Invalid state names provided.');
return false; return false;
@ -66,10 +66,10 @@ FSM.prototype.setState = function(stateName) {
}; };
// Method to attempt a state transition // Method to attempt a state transition
FSM.prototype.tryTransition = function(eventType, stateName) { FSM.prototype.tryTransition = function(stateName, eventType, options) {
if (this.currentState && stateName in this.currentState.transitions) { if (this.currentState && stateName in this.currentState.transitions) {
var transition = this.currentState.transitions[stateName]; var transition = this.currentState.transitions[stateName];
var event = new STD.Event(eventType); var event = new STD.Event(eventType, options);
transition.dispatchEvent(event); transition.dispatchEvent(event);
this.setState(stateName); this.setState(stateName);
} else { } else {

9
lib/std.js
View File

@ -268,7 +268,7 @@ function confirm(message) {
} }
// Standard Event Object // Standard Event Object
function StdEvent(type) { function StdEvent(type, options) {
this.defaultPrevented = false; this.defaultPrevented = false;
this.timeStamp = new Date(); this.timeStamp = new Date();
this.type = type; this.type = type;
@ -296,6 +296,13 @@ function StdEvent(type) {
// Not used but to be compatible // Not used but to be compatible
this.stopPropagation = function() {}; this.stopPropagation = function() {};
// Apply the options
for (var optionKey in options) {
if (!(optionKey in this)) {
this[optionKey] = options[optionKey];
}
}
}; };
StdEvent.NONE = 0; StdEvent.NONE = 0;
StdEvent.CAPTURING_PHASE = 1; // Not used but to be compatible StdEvent.CAPTURING_PHASE = 1; // Not used but to be compatible

2
settings.example.ini
View File

@ -13,7 +13,7 @@ DISABLE_SCREEN_TIME=true
DISABLE_FILE_MONITOR=true DISABLE_FILE_MONITOR=true
; window or screen ; window or screen
SCREEN_TIME_MODE=screen SCREEN_TIME_MODE=screen
; backward,save,sample_ocr,sample_clipboard,sample_width=128,sample_height=128,sample_adjust_x=0,sample_adjust_y=0,sample_only=,process_name=notepad.exe ; backward,save,sample_ocr,sample_clipboard,sample_width=128,sample_height=128,sample_adjust_x=0,sample_adjust_y=0,sample_only=button.png:message.png,sample_nodup=message.png,process_name=notepad.exe
SCREEN_TIME_PARAMS= SCREEN_TIME_PARAMS=
; default: http://localhost:50051 ; default: http://localhost:50051
GRPC_HOST=http://localhost:50051 GRPC_HOST=http://localhost:50051