mirror of
https://github.com/gnh1201/welsonjs.git
synced 2025-02-06 06:54:58 +00:00
Fix a proxy services and HTTP request issues
1. Separated the data listing available HTTP proxy services into a separate file. (data/available_proxies.json) 2. Excluded the architecture of the user's PC (e.g., 32-bit, 64-bit) from the User-Agent in HTTP requests.
This commit is contained in:
parent
79295de4ce
commit
638b172e77
80
data/available_proxies.json
Normal file
80
data/available_proxies.json
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
[
|
||||
{
|
||||
"type": "stateless",
|
||||
"provider": "scrapeops",
|
||||
"url": "https://proxy.scrapeops.io/v1/?api_key={api_key}&url={url}&render_js={render_js}&residential={residential}&country={country}&keep_headers={keep_headers}",
|
||||
"documentation": "https://scrapeops.io?fpr=namhyeon75"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "scrapeops",
|
||||
"url": "http://scrapeops:{api_key}@residential-proxy.scrapeops.io:8181",
|
||||
"documentation": "https://scrapeops.io?fpr=namhyeon75"
|
||||
},
|
||||
{
|
||||
"type": "serp",
|
||||
"provider": "searchapi",
|
||||
"url": "https://www.searchapi.io/api/v1/search?api_key={api_key}&engine={engine}&q={q}",
|
||||
"documentation": "https://www.searchapi.io/?via=namhyeon"
|
||||
},
|
||||
{
|
||||
"type": "serp",
|
||||
"provider": "librey",
|
||||
"url": "https://serp.catswords.net/librex/api.php?q={q}&p=1&t=0",
|
||||
"documentation": "https://github.com/Ahwxorg/LibreY"
|
||||
},
|
||||
{
|
||||
"type": "serp",
|
||||
"provider": "invidious",
|
||||
"url": "https://invidious.jing.rocks/invidious/api/v1/search?q={q}",
|
||||
"documentation": "https://docs.invidious.io/instances/"
|
||||
},
|
||||
{
|
||||
"type": "stateless-jsonrpc2",
|
||||
"provider": "github.com/gnh1201/caterpillar",
|
||||
"url": "http://localhost:8080",
|
||||
"documentation": "https://github.com/gnh1201/caterpillar"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "github.com/gnh1201/caterpillar",
|
||||
"url": "http://localhost:5555",
|
||||
"documentation": "https://github.com/gnh1201/caterpillar"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "cloudflare",
|
||||
"url": "http://localhost:40000",
|
||||
"documentation": "https://developers.cloudflare.com/warp-client/warp-modes/"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "fiddler",
|
||||
"url": "http://localhost:8888",
|
||||
"documentation": "https://www.telerik.com/fiddler/fiddler-classic"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "fiddler2",
|
||||
"url": "http://localhost:8866",
|
||||
"documentation": "https://www.telerik.com/fiddler/fiddler-everywhere"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "mitmproxy",
|
||||
"url": "http://localhost:8080",
|
||||
"documetation": "https://mitmproxy.org/"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "burpsuite",
|
||||
"url": "http://localhost:8080",
|
||||
"documetation": "https://portswigger.net/burp"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "zaproxy",
|
||||
"url": "http://localhost:8080",
|
||||
"documetation": "https://www.zaproxy.org/"
|
||||
}
|
||||
]
|
||||
164
lib/http.js
164
lib/http.js
|
|
@ -1,5 +1,5 @@
|
|||
// http.js
|
||||
// HTTP Client for WelsonJS framework
|
||||
// HTTP REST API client for WelsonJS framework
|
||||
// Namhyeon Go (Catswords Research) <abuse@catswords.net>
|
||||
// https://github.com/gnh1201/welsonjs
|
||||
var SYS = require("lib/system");
|
||||
|
|
@ -9,92 +9,19 @@ var RAND = require("lib/rand");
|
|||
var BASE64 = require("lib/base64");
|
||||
var PipeIPC = require("lib/pipe-ipc");
|
||||
var JsonRpc2 = require("lib/jsonrpc2");
|
||||
var SERP = require("lib/serp");
|
||||
|
||||
var OS_NAME = SYS.getOS();
|
||||
var OS_ARCH = SYS.getArch();
|
||||
var DEVICE_UUID = SYS.getUUID();
|
||||
var PROCESS_VERSION = SYS.getProcessVersion();
|
||||
var DEFAULT_USER_AGENT = "WelsonJS/0.2.7 (" + OS_NAME + "; " + OS_ARCH + "; " + PROCESS_VERSION + "; " + DEVICE_UUID + "; abuse@catswords.net)";
|
||||
var DEFAULT_USER_AGENT = "WelsonJS/0.2.7 (" + OS_NAME + "; " + PROCESS_VERSION + "; " + DEVICE_UUID + ")";
|
||||
|
||||
// If you have any suggestions for partnerships, please contact us at: abuse@catswords.net
|
||||
var AVAILABLE_PROXIES = [
|
||||
{
|
||||
"type": "stateless",
|
||||
"provider": "scrapeops",
|
||||
"url": "https://proxy.scrapeops.io/v1/?api_key={api_key}&url={url}&render_js={render_js}&residential={residential}&country={country}&keep_headers={keep_headers}",
|
||||
"documentation": "https://scrapeops.io?fpr=namhyeon75"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "scrapeops",
|
||||
"url": "http://scrapeops:{api_key}@residential-proxy.scrapeops.io:8181",
|
||||
"documentation": "https://scrapeops.io?fpr=namhyeon75"
|
||||
},
|
||||
{
|
||||
"type": "serp",
|
||||
"provider": "searchapi",
|
||||
"url": "https://www.searchapi.io/api/v1/search?api_key={api_key}&engine={engine}&q={q}",
|
||||
"documentation": "https://www.searchapi.io/?via=namhyeon"
|
||||
},
|
||||
{
|
||||
"type": "serp",
|
||||
"provider": "librey",
|
||||
"url": "https://serp.catswords.net/librex/api.php?q={q}&p=1&t=0",
|
||||
"documentation": "https://github.com/Ahwxorg/LibreY"
|
||||
},
|
||||
{
|
||||
"type": "serp",
|
||||
"provider": "invidious",
|
||||
"url": "https://invidious.jing.rocks/invidious/api/v1/search?q={q}",
|
||||
"documentation": "https://docs.invidious.io/instances/"
|
||||
},
|
||||
{
|
||||
"type": "stateless-jsonrpc2",
|
||||
"provider": "github.com/gnh1201/caterpillar",
|
||||
"url": "http://localhost:8080",
|
||||
"documentation": "https://github.com/gnh1201/caterpillar"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "github.com/gnh1201/caterpillar",
|
||||
"url": "http://localhost:5555",
|
||||
"documentation": "https://github.com/gnh1201/caterpillar"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "cloudflare",
|
||||
"url": "http://localhost:40000",
|
||||
"documentation": "https://developers.cloudflare.com/warp-client/warp-modes/"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "fiddler",
|
||||
"url": "http://localhost:8888",
|
||||
"documentation": "https://www.telerik.com/fiddler/fiddler-classic"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "fiddler2",
|
||||
"url": "http://localhost:8866",
|
||||
"documentation": "https://www.telerik.com/fiddler/fiddler-everywhere"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "mitmproxy",
|
||||
"url": "http://localhost:8080",
|
||||
"documetation": "https://mitmproxy.org/"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "burpsuite",
|
||||
"url": "http://localhost:8080",
|
||||
"documetation": "https://portswigger.net/burp"
|
||||
},
|
||||
{
|
||||
"type": "stateful",
|
||||
"provider": "zaproxy",
|
||||
"url": "http://localhost:8080",
|
||||
"documetation": "https://www.zaproxy.org/"
|
||||
"type": "file",
|
||||
"provider": "",
|
||||
"url": "data/available_proxies.json",
|
||||
"documentation": ""
|
||||
}
|
||||
];
|
||||
|
||||
|
|
@ -554,7 +481,7 @@ var HTTPObject = function(engine) {
|
|||
if (!this.proxy.enabled) return url;
|
||||
|
||||
if (this.proxy.type == "serp") {
|
||||
var serp = this.parseSerpUrl(url);
|
||||
var serp = SERP.parseUrl(url);
|
||||
this.setVariable("engine", serp.engine);
|
||||
this.setVariable("q", encodeURIComponent(serp.keyword));
|
||||
}
|
||||
|
|
@ -567,60 +494,6 @@ var HTTPObject = function(engine) {
|
|||
return url;
|
||||
};
|
||||
|
||||
this.parseSerpUrl = function(url) {
|
||||
var getEngine = function(url) {
|
||||
var defaultEngine = "google";
|
||||
var engines = {
|
||||
"google": ["naver", "daum"],
|
||||
"google_shopping": [
|
||||
"aliexpress", "temu", "coupang", "shopping.naver",
|
||||
"ssg", "gmarket", "11st", "store.kakao",
|
||||
"lotteon", "tmon", "wemakeprice"
|
||||
],
|
||||
"default": [
|
||||
"google", "youtube", "bing", "baidu",
|
||||
"amazon", "duckduckgo"
|
||||
]
|
||||
};
|
||||
|
||||
var match = url.match(/^(?:https?:\/\/)?(?:www\.)?([\w.-]+)\.\w+$/);
|
||||
if (!match) {
|
||||
return defaultEngine;
|
||||
}
|
||||
|
||||
var domain = match[1];
|
||||
|
||||
for (var key in engines) {
|
||||
if (engines.hasOwnProperty(key)) {
|
||||
var group = engines[key];
|
||||
if (group.indexOf(domain) !== -1) {
|
||||
if (key === "default") {
|
||||
return domain;
|
||||
}
|
||||
return key;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return defaultEngine;
|
||||
};
|
||||
|
||||
var getKeyword = function(url) {
|
||||
var regex = /[?&](q|wd|query|keyword|search_query|k|SearchText|search_key)=([^&]*)/g;
|
||||
//var regex = /(?:[?&](q|wd|keyword|query|search_query|k|SearchText|search_key)=|\/pdsearch\/)([^&?]*)/g;
|
||||
var match, keywords = [];
|
||||
while ((match = regex.exec(url)) !== null) {
|
||||
keywords.push(match[2]);
|
||||
}
|
||||
return keywords.join(' ');
|
||||
};
|
||||
|
||||
return {
|
||||
"engine": getEngine(url),
|
||||
"keyword": getKeyword(url)
|
||||
}
|
||||
};
|
||||
|
||||
this.open = function(method, url) {
|
||||
var url = this.serializeParameters(url);
|
||||
|
||||
|
|
@ -1312,6 +1185,25 @@ function parseURL(url) {
|
|||
};
|
||||
}
|
||||
|
||||
// Check an available proxies
|
||||
AVAILABLE_PROXIES.forEach(function(proxy) {
|
||||
if (proxy.type == "file") {
|
||||
if (FILE.fileExists(proxy.url)) {
|
||||
try {
|
||||
var fileContents = FILE.readFile(proxy.url, FILE.CdoCharset.CdoUTF_8);
|
||||
var data = JSON.parse(fileContents);
|
||||
data.forEach(function(x) {
|
||||
AVAILABLE_PROXIES.push(x);
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn(proxy.url, "is not a valid file");
|
||||
}
|
||||
} else {
|
||||
console.warn(proxy.url, "does not exists");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
exports.create = create;
|
||||
exports.get = get;
|
||||
exports.post = post;
|
||||
|
|
@ -1323,7 +1215,7 @@ exports.parseURL = parseURL;
|
|||
exports.DEFAULT_USER_AGENT = DEFAULT_USER_AGENT;
|
||||
exports.defaultUserAgent = DEFAULT_USER_AGENT; // compatible
|
||||
|
||||
exports.VERSIONINFO = "HTTP Client for WelsonJS framework (http.js) version 0.7.44";
|
||||
exports.VERSIONINFO = "HTTP REST API client for WelsonJS framework (http.js) version 0.7.45";
|
||||
exports.AUTHOR = "abuse@catswords.net";
|
||||
exports.global = global;
|
||||
exports.require = global.require;
|
||||
|
|
|
|||
64
lib/serp.js
Normal file
64
lib/serp.js
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
// serp.js
|
||||
// SERP/SEO tools integration for WelsonJS framework
|
||||
// Namhyeon Go <abuse@catswords.net>
|
||||
// https://github.com/gnh1201/welsonjs
|
||||
function parseUrl(url) {
|
||||
var getEngine = function(url) {
|
||||
var defaultEngine = "google";
|
||||
var engines = {
|
||||
"google": ["naver", "daum"],
|
||||
"google_shopping": [
|
||||
"aliexpress", "temu", "coupang", "shopping.naver",
|
||||
"ssg", "gmarket", "11st", "store.kakao",
|
||||
"lotteon", "tmon", "wemakeprice"
|
||||
],
|
||||
"default": [
|
||||
"google", "youtube", "bing", "baidu",
|
||||
"amazon", "duckduckgo"
|
||||
]
|
||||
};
|
||||
|
||||
var match = url.match(/^(?:https?:\/\/)?(?:www\.)?([\w.-]+)\.\w+$/);
|
||||
if (!match) {
|
||||
return defaultEngine;
|
||||
}
|
||||
|
||||
var domain = match[1];
|
||||
|
||||
for (var key in engines) {
|
||||
if (engines.hasOwnProperty(key)) {
|
||||
var group = engines[key];
|
||||
if (group.indexOf(domain) !== -1) {
|
||||
if (key === "default") {
|
||||
return domain;
|
||||
}
|
||||
return key;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return defaultEngine;
|
||||
};
|
||||
|
||||
var getKeyword = function(url) {
|
||||
var regex = /[?&](q|wd|query|keyword|search_query|k|SearchText|search_key)=([^&]*)/g;
|
||||
//var regex = /(?:[?&](q|wd|keyword|query|search_query|k|SearchText|search_key)=|\/pdsearch\/)([^&?]*)/g;
|
||||
var match, keywords = [];
|
||||
while ((match = regex.exec(url)) !== null) {
|
||||
keywords.push(match[2]);
|
||||
}
|
||||
return keywords.join(' ');
|
||||
};
|
||||
|
||||
return {
|
||||
"engine": getEngine(url),
|
||||
"keyword": getKeyword(url)
|
||||
}
|
||||
};
|
||||
|
||||
exports.parseUrl = parseUrl;
|
||||
|
||||
exports.VERSIONINFO = "SERP/SEO tools integration (serp.js) version 0.1";
|
||||
exports.AUTHOR = "abuse@catswords.net";
|
||||
exports.global = global;
|
||||
exports.require = global.require;
|
||||
Loading…
Reference in New Issue
Block a user