diff --git a/data/available_proxies.json b/data/available_proxies.json new file mode 100644 index 0000000..4284dee --- /dev/null +++ b/data/available_proxies.json @@ -0,0 +1,80 @@ +[ + { + "type": "stateless", + "provider": "scrapeops", + "url": "https://proxy.scrapeops.io/v1/?api_key={api_key}&url={url}&render_js={render_js}&residential={residential}&country={country}&keep_headers={keep_headers}", + "documentation": "https://scrapeops.io?fpr=namhyeon75" + }, + { + "type": "stateful", + "provider": "scrapeops", + "url": "http://scrapeops:{api_key}@residential-proxy.scrapeops.io:8181", + "documentation": "https://scrapeops.io?fpr=namhyeon75" + }, + { + "type": "serp", + "provider": "searchapi", + "url": "https://www.searchapi.io/api/v1/search?api_key={api_key}&engine={engine}&q={q}", + "documentation": "https://www.searchapi.io/?via=namhyeon" + }, + { + "type": "serp", + "provider": "librey", + "url": "https://serp.catswords.net/librex/api.php?q={q}&p=1&t=0", + "documentation": "https://github.com/Ahwxorg/LibreY" + }, + { + "type": "serp", + "provider": "invidious", + "url": "https://invidious.jing.rocks/invidious/api/v1/search?q={q}", + "documentation": "https://docs.invidious.io/instances/" + }, + { + "type": "stateless-jsonrpc2", + "provider": "github.com/gnh1201/caterpillar", + "url": "http://localhost:8080", + "documentation": "https://github.com/gnh1201/caterpillar" + }, + { + "type": "stateful", + "provider": "github.com/gnh1201/caterpillar", + "url": "http://localhost:5555", + "documentation": "https://github.com/gnh1201/caterpillar" + }, + { + "type": "stateful", + "provider": "cloudflare", + "url": "http://localhost:40000", + "documentation": "https://developers.cloudflare.com/warp-client/warp-modes/" + }, + { + "type": "stateful", + "provider": "fiddler", + "url": "http://localhost:8888", + "documentation": "https://www.telerik.com/fiddler/fiddler-classic" + }, + { + "type": "stateful", + "provider": "fiddler2", + "url": "http://localhost:8866", + "documentation": "https://www.telerik.com/fiddler/fiddler-everywhere" + }, + { + "type": "stateful", + "provider": "mitmproxy", + "url": "http://localhost:8080", + "documetation": "https://mitmproxy.org/" + }, + { + "type": "stateful", + "provider": "burpsuite", + "url": "http://localhost:8080", + "documetation": "https://portswigger.net/burp" + }, + { + "type": "stateful", + "provider": "zaproxy", + "url": "http://localhost:8080", + "documetation": "https://www.zaproxy.org/" + } +] \ No newline at end of file diff --git a/lib/http.js b/lib/http.js index 7932ae1..8189f21 100644 --- a/lib/http.js +++ b/lib/http.js @@ -1,5 +1,5 @@ // http.js -// HTTP Client for WelsonJS framework +// HTTP REST API client for WelsonJS framework // Namhyeon Go (Catswords Research) // https://github.com/gnh1201/welsonjs var SYS = require("lib/system"); @@ -9,92 +9,19 @@ var RAND = require("lib/rand"); var BASE64 = require("lib/base64"); var PipeIPC = require("lib/pipe-ipc"); var JsonRpc2 = require("lib/jsonrpc2"); +var SERP = require("lib/serp"); var OS_NAME = SYS.getOS(); -var OS_ARCH = SYS.getArch(); var DEVICE_UUID = SYS.getUUID(); var PROCESS_VERSION = SYS.getProcessVersion(); -var DEFAULT_USER_AGENT = "WelsonJS/0.2.7 (" + OS_NAME + "; " + OS_ARCH + "; " + PROCESS_VERSION + "; " + DEVICE_UUID + "; abuse@catswords.net)"; +var DEFAULT_USER_AGENT = "WelsonJS/0.2.7 (" + OS_NAME + "; " + PROCESS_VERSION + "; " + DEVICE_UUID + ")"; -// If you have any suggestions for partnerships, please contact us at: abuse@catswords.net var AVAILABLE_PROXIES = [ { - "type": "stateless", - "provider": "scrapeops", - "url": "https://proxy.scrapeops.io/v1/?api_key={api_key}&url={url}&render_js={render_js}&residential={residential}&country={country}&keep_headers={keep_headers}", - "documentation": "https://scrapeops.io?fpr=namhyeon75" - }, - { - "type": "stateful", - "provider": "scrapeops", - "url": "http://scrapeops:{api_key}@residential-proxy.scrapeops.io:8181", - "documentation": "https://scrapeops.io?fpr=namhyeon75" - }, - { - "type": "serp", - "provider": "searchapi", - "url": "https://www.searchapi.io/api/v1/search?api_key={api_key}&engine={engine}&q={q}", - "documentation": "https://www.searchapi.io/?via=namhyeon" - }, - { - "type": "serp", - "provider": "librey", - "url": "https://serp.catswords.net/librex/api.php?q={q}&p=1&t=0", - "documentation": "https://github.com/Ahwxorg/LibreY" - }, - { - "type": "serp", - "provider": "invidious", - "url": "https://invidious.jing.rocks/invidious/api/v1/search?q={q}", - "documentation": "https://docs.invidious.io/instances/" - }, - { - "type": "stateless-jsonrpc2", - "provider": "github.com/gnh1201/caterpillar", - "url": "http://localhost:8080", - "documentation": "https://github.com/gnh1201/caterpillar" - }, - { - "type": "stateful", - "provider": "github.com/gnh1201/caterpillar", - "url": "http://localhost:5555", - "documentation": "https://github.com/gnh1201/caterpillar" - }, - { - "type": "stateful", - "provider": "cloudflare", - "url": "http://localhost:40000", - "documentation": "https://developers.cloudflare.com/warp-client/warp-modes/" - }, - { - "type": "stateful", - "provider": "fiddler", - "url": "http://localhost:8888", - "documentation": "https://www.telerik.com/fiddler/fiddler-classic" - }, - { - "type": "stateful", - "provider": "fiddler2", - "url": "http://localhost:8866", - "documentation": "https://www.telerik.com/fiddler/fiddler-everywhere" - }, - { - "type": "stateful", - "provider": "mitmproxy", - "url": "http://localhost:8080", - "documetation": "https://mitmproxy.org/" - }, - { - "type": "stateful", - "provider": "burpsuite", - "url": "http://localhost:8080", - "documetation": "https://portswigger.net/burp" - }, - { - "type": "stateful", - "provider": "zaproxy", - "url": "http://localhost:8080", - "documetation": "https://www.zaproxy.org/" + "type": "file", + "provider": "", + "url": "data/available_proxies.json", + "documentation": "" } ]; @@ -554,7 +481,7 @@ var HTTPObject = function(engine) { if (!this.proxy.enabled) return url; if (this.proxy.type == "serp") { - var serp = this.parseSerpUrl(url); + var serp = SERP.parseUrl(url); this.setVariable("engine", serp.engine); this.setVariable("q", encodeURIComponent(serp.keyword)); } @@ -567,60 +494,6 @@ var HTTPObject = function(engine) { return url; }; - this.parseSerpUrl = function(url) { - var getEngine = function(url) { - var defaultEngine = "google"; - var engines = { - "google": ["naver", "daum"], - "google_shopping": [ - "aliexpress", "temu", "coupang", "shopping.naver", - "ssg", "gmarket", "11st", "store.kakao", - "lotteon", "tmon", "wemakeprice" - ], - "default": [ - "google", "youtube", "bing", "baidu", - "amazon", "duckduckgo" - ] - }; - - var match = url.match(/^(?:https?:\/\/)?(?:www\.)?([\w.-]+)\.\w+$/); - if (!match) { - return defaultEngine; - } - - var domain = match[1]; - - for (var key in engines) { - if (engines.hasOwnProperty(key)) { - var group = engines[key]; - if (group.indexOf(domain) !== -1) { - if (key === "default") { - return domain; - } - return key; - } - } - } - - return defaultEngine; - }; - - var getKeyword = function(url) { - var regex = /[?&](q|wd|query|keyword|search_query|k|SearchText|search_key)=([^&]*)/g; - //var regex = /(?:[?&](q|wd|keyword|query|search_query|k|SearchText|search_key)=|\/pdsearch\/)([^&?]*)/g; - var match, keywords = []; - while ((match = regex.exec(url)) !== null) { - keywords.push(match[2]); - } - return keywords.join(' '); - }; - - return { - "engine": getEngine(url), - "keyword": getKeyword(url) - } - }; - this.open = function(method, url) { var url = this.serializeParameters(url); @@ -1312,6 +1185,25 @@ function parseURL(url) { }; } +// Check an available proxies +AVAILABLE_PROXIES.forEach(function(proxy) { + if (proxy.type == "file") { + if (FILE.fileExists(proxy.url)) { + try { + var fileContents = FILE.readFile(proxy.url, FILE.CdoCharset.CdoUTF_8); + var data = JSON.parse(fileContents); + data.forEach(function(x) { + AVAILABLE_PROXIES.push(x); + }); + } catch (e) { + console.warn(proxy.url, "is not a valid file"); + } + } else { + console.warn(proxy.url, "does not exists"); + } + } +}); + exports.create = create; exports.get = get; exports.post = post; @@ -1323,7 +1215,7 @@ exports.parseURL = parseURL; exports.DEFAULT_USER_AGENT = DEFAULT_USER_AGENT; exports.defaultUserAgent = DEFAULT_USER_AGENT; // compatible -exports.VERSIONINFO = "HTTP Client for WelsonJS framework (http.js) version 0.7.44"; +exports.VERSIONINFO = "HTTP REST API client for WelsonJS framework (http.js) version 0.7.45"; exports.AUTHOR = "abuse@catswords.net"; exports.global = global; exports.require = global.require; diff --git a/lib/serp.js b/lib/serp.js new file mode 100644 index 0000000..405eea5 --- /dev/null +++ b/lib/serp.js @@ -0,0 +1,64 @@ +// serp.js +// SERP/SEO tools integration for WelsonJS framework +// Namhyeon Go +// https://github.com/gnh1201/welsonjs +function parseUrl(url) { + var getEngine = function(url) { + var defaultEngine = "google"; + var engines = { + "google": ["naver", "daum"], + "google_shopping": [ + "aliexpress", "temu", "coupang", "shopping.naver", + "ssg", "gmarket", "11st", "store.kakao", + "lotteon", "tmon", "wemakeprice" + ], + "default": [ + "google", "youtube", "bing", "baidu", + "amazon", "duckduckgo" + ] + }; + + var match = url.match(/^(?:https?:\/\/)?(?:www\.)?([\w.-]+)\.\w+$/); + if (!match) { + return defaultEngine; + } + + var domain = match[1]; + + for (var key in engines) { + if (engines.hasOwnProperty(key)) { + var group = engines[key]; + if (group.indexOf(domain) !== -1) { + if (key === "default") { + return domain; + } + return key; + } + } + } + + return defaultEngine; + }; + + var getKeyword = function(url) { + var regex = /[?&](q|wd|query|keyword|search_query|k|SearchText|search_key)=([^&]*)/g; + //var regex = /(?:[?&](q|wd|keyword|query|search_query|k|SearchText|search_key)=|\/pdsearch\/)([^&?]*)/g; + var match, keywords = []; + while ((match = regex.exec(url)) !== null) { + keywords.push(match[2]); + } + return keywords.join(' '); + }; + + return { + "engine": getEngine(url), + "keyword": getKeyword(url) + } +}; + +exports.parseUrl = parseUrl; + +exports.VERSIONINFO = "SERP/SEO tools integration (serp.js) version 0.1"; +exports.AUTHOR = "abuse@catswords.net"; +exports.global = global; +exports.require = global.require;