From 4bef7a241756ef786477028aa84761d80ad5859a Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Thu, 4 Jul 2024 14:44:57 +0900 Subject: [PATCH 01/20] Create portscan.py --- plugins/portscan.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 plugins/portscan.py diff --git a/plugins/portscan.py b/plugins/portscan.py new file mode 100644 index 0000000..47692be --- /dev/null +++ b/plugins/portscan.py @@ -0,0 +1,34 @@ +#!/usr/bin/python3 +# +# portscan.py +# NMAP port scanning wrapper for Caterpillar Proxy +# +# Caterpillar Proxy - The simple web debugging proxy (formerly, php-httpproxy) +# Namyheon Go (Catswords Research) +# https://github.com/gnh1201/caterpillar +# Created at: 2022-01-26 (github.com/gnh1201/welsonjs) +# Updated at: 2024-07-04 +# +import sys +import nmap +import json + +from base import Extension + +class PortScanner(Extension): + def __init__(self): + self.type = "rpcmethod" + self.method = "discover_ports_by_hosts" + self.exported_methods = [] + + def dispatch(self, type, id, params, conn): + hosts = params['hosts'] + binpath = params['binpath'] + + #result = nm.scan(hosts=hosts, arguments='-T5 -sV -p21-25,80,139,443,445,1883,2179,2323,3389,7547,8080,8443,8883') + result = nm.scan(hosts=hosts, arguments='-T5 -sV -p0-65535 --max-retries 0') + + return result; + +if __name__ == "__main__": + main(sys.argv) From 7d5d997881dc73eca5b917ce14e9c10f5d49d54d Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Thu, 4 Jul 2024 14:46:40 +0900 Subject: [PATCH 02/20] Update portscan.py --- plugins/portscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/portscan.py b/plugins/portscan.py index 47692be..112028c 100644 --- a/plugins/portscan.py +++ b/plugins/portscan.py @@ -18,7 +18,7 @@ from base import Extension class PortScanner(Extension): def __init__(self): self.type = "rpcmethod" - self.method = "discover_ports_by_hosts" + self.method = "scan_ports_by_hosts" self.exported_methods = [] def dispatch(self, type, id, params, conn): From 823c97015f163246c178ed728b0f0f8376d983e0 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Thu, 4 Jul 2024 15:00:58 +0900 Subject: [PATCH 03/20] Update server.py --- server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 11b4649..a221916 100644 --- a/server.py +++ b/server.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2022-10-06 -# Updated at: 2024-06-20 +# Updated at: 2024-07-04 # import argparse @@ -502,6 +502,8 @@ if __name__== "__main__": #Extension.register(importlib.import_module("plugins.fediverse").Fediverse()) #Extension.register(importlib.import_module("plugins.container").Container()) #Extension.register(importlib.import_module("plugins.wayback").Wayback()) - + #Extension.register(importlib.import_module("plugins.bio").Bio()) + #Extension.register(importlib.import_module("plugins.nmap").PortScanner()) + # start Caterpillar start() From 32af8bd70187343cf35b47c48257981c31d9d642 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Thu, 4 Jul 2024 15:02:51 +0900 Subject: [PATCH 04/20] Rename portscan.py to nmap.py --- plugins/{portscan.py => nmap.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/{portscan.py => nmap.py} (100%) diff --git a/plugins/portscan.py b/plugins/nmap.py similarity index 100% rename from plugins/portscan.py rename to plugins/nmap.py From f953341330887a00935bd3bb49a86376b177176e Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Sat, 6 Jul 2024 22:52:53 +0900 Subject: [PATCH 05/20] Fix `Extension.register()` API and related files --- .gitignore | 1 + base.py | 14 ++++++++++---- plugins/container.py | 4 ++-- plugins/fediverse.py | 4 ++-- plugins/wayback.py | 5 +++-- server.py | 12 ++++++------ web.py | 4 ++-- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 7930d50..57760a2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ certs/ savedfiles/ settings.ini .env +*.pyc \ No newline at end of file diff --git a/base.py b/base.py index c2406e8..1d121e1 100644 --- a/base.py +++ b/base.py @@ -3,16 +3,17 @@ # base.py # base (common) file # -# Caterpillar Proxy - The simple and parasitic web proxy SPAM spam filter +# Caterpillar Proxy - The simple web debugging proxy (formerly, php-httpproxy) # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-05-20 -# Updated at: 2024-05-21 +# Updated at: 2024-07-06 # import hashlib import json import re +import importlib client_encoding = 'utf-8' @@ -71,8 +72,13 @@ class Extension(): cls.buffer_size = _buffer_size @classmethod - def register(cls, f): - cls.extensions.append(f) + def register(cls, module_path, class_name): + try: + module = importlib.import_module(module_path) + _class = getattr(module, class_name) + cls.extensions.append(_class()) + except (ImportError, AttributeError) as e: + raise ImportError(class_name + " in " + module_path) @classmethod def get_filters(cls): diff --git a/plugins/container.py b/plugins/container.py index 11dbbb4..b2f6429 100644 --- a/plugins/container.py +++ b/plugins/container.py @@ -7,12 +7,12 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-03-04 -# Updated at: 2024-03-13 +# Updated at: 2024-07-06 # import docker -from server import Extension +from base import Extension class Container(Extension): def __init__(self): diff --git a/plugins/fediverse.py b/plugins/fediverse.py index f6f910b..7dcd6d4 100644 --- a/plugins/fediverse.py +++ b/plugins/fediverse.py @@ -8,7 +8,7 @@ # https://github.com/gnh1201/caterpillar # # Created in: 2022-10-06 -# Updated in: 2024-06-05 +# Updated in: 2024-07-06 # import io @@ -19,7 +19,7 @@ import os.path from decouple import config from PIL import Image -from server import Extension +from base import Extension try: client_encoding = config('CLIENT_ENCODING', default='utf-8') diff --git a/plugins/wayback.py b/plugins/wayback.py index 51da3b9..cace667 100644 --- a/plugins/wayback.py +++ b/plugins/wayback.py @@ -7,12 +7,13 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-03-13 -# Updated at: 2024-03-13 +# Updated at: 2024-07-06 # import requests +from decouple import config -from server import Extension +from base import Extension try: client_encoding = config('CLIENT_ENCODING') diff --git a/server.py b/server.py index 11b4649..c9227b6 100644 --- a/server.py +++ b/server.py @@ -23,7 +23,6 @@ import time import hashlib import traceback import textwrap -import importlib from datetime import datetime from platform import python_version @@ -499,9 +498,10 @@ def start(): #Main Program if __name__== "__main__": # load extensions - #Extension.register(importlib.import_module("plugins.fediverse").Fediverse()) - #Extension.register(importlib.import_module("plugins.container").Container()) - #Extension.register(importlib.import_module("plugins.wayback").Wayback()) - - # start Caterpillar + #Extension.register("plugins.fediverse", "Fediverse") + #Extension.register("plugins.container", "Container") + Extension.register("plugins.wayback", "Wayback") + #Extension.register("plugins.bio", "PyBio") + + # start Caterpillar start() diff --git a/web.py b/web.py index 236651e..857e360 100644 --- a/web.py +++ b/web.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-05-20 -# Updated at: 2024-05-20 +# Updated at: 2024-07-06 # from flask import Flask, request, redirect, url_for, render_template @@ -94,6 +94,6 @@ if __name__ == "__main__": Extension.set_protocol('http') # load extensions - #Extension.register(importlib.import_module("plugins.yourownplugin").YourOwnPlugin()) + #Extension.register("plugins.YOUR_OWN_MODULE_NAME", "YOUR_OWN_CLASS_NAME"); app.run(debug=True, host='0.0.0.0', port=listening_port) From bf8ea7be95e80fd7add90fa3d2cc02ec8100b1aa Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 15:53:34 +0900 Subject: [PATCH 06/20] One more fix #24 --- plugins/nmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nmap.py b/plugins/nmap.py index 112028c..ecf0238 100644 --- a/plugins/nmap.py +++ b/plugins/nmap.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2022-01-26 (github.com/gnh1201/welsonjs) -# Updated at: 2024-07-04 +# Updated at: 2024-07-09 # import sys import nmap @@ -25,7 +25,7 @@ class PortScanner(Extension): hosts = params['hosts'] binpath = params['binpath'] - #result = nm.scan(hosts=hosts, arguments='-T5 -sV -p21-25,80,139,443,445,1883,2179,2323,3389,7547,8080,8443,8883') + nm = nmap.PortScanner(nmap_search_path=(binpath,)) result = nm.scan(hosts=hosts, arguments='-T5 -sV -p0-65535 --max-retries 0') return result; From 3f185a237ffba6c5ee7c445439466d110a98d07f Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 15:56:20 +0900 Subject: [PATCH 07/20] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 13b2fab..bc6da18 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ CERT_KEY=cert.key CERT_DIR=certs/ OPENSSL_BINPATH=openssl CLIENT_ENCODING=utf-8 +LOADED_EXTENSIONS=wayback ``` - (Optional) Create a certificate for SSL decryption From d0b1cc2bf502c854d1065e65c2249e8eb8b36e97 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:08:52 +0900 Subject: [PATCH 08/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc6da18..1766995 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ CERT_KEY=cert.key CERT_DIR=certs/ OPENSSL_BINPATH=openssl CLIENT_ENCODING=utf-8 -LOADED_EXTENSIONS=wayback +ENABLED_EXTENSIONS=wayback ``` - (Optional) Create a certificate for SSL decryption From c08731245535a673e7f2e13c09976957399d8850 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:11:05 +0900 Subject: [PATCH 09/20] Update server.py --- server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index 317e8bb..c25a3ca 100644 --- a/server.py +++ b/server.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2022-10-06 -# Updated at: 2024-07-04 +# Updated at: 2024-07-09 # import argparse @@ -47,6 +47,7 @@ try: client_encoding = config('CLIENT_ENCODING', default='utf-8') local_domain = config('LOCAL_DOMAIN', default='') proxy_pass = config('PROXY_PASS', default='') + enabled_extensions = config('ENABLED_EXTENSIONS', default='') except KeyboardInterrupt: print("\n[*] User has requested an interrupt") print("[*] Application Exiting.....") @@ -498,10 +499,9 @@ def start(): #Main Program if __name__== "__main__": # load extensions - #Extension.register("plugins.fediverse", "Fediverse") - #Extension.register("plugins.container", "Container") - Extension.register("plugins.wayback", "Wayback") - #Extension.register("plugins.bio", "PyBio") + extensions = list(map(str.strip, enabled_extension.split(','))) + for extension in extensions: + Extension.register(extension) # start Caterpillar start() From e539e3e670bab7afe74b8918fcf82210e2de5afe Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:15:48 +0900 Subject: [PATCH 10/20] Revert "Update server.py" This reverts commit c08731245535a673e7f2e13c09976957399d8850. --- server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index c25a3ca..317e8bb 100644 --- a/server.py +++ b/server.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2022-10-06 -# Updated at: 2024-07-09 +# Updated at: 2024-07-04 # import argparse @@ -47,7 +47,6 @@ try: client_encoding = config('CLIENT_ENCODING', default='utf-8') local_domain = config('LOCAL_DOMAIN', default='') proxy_pass = config('PROXY_PASS', default='') - enabled_extensions = config('ENABLED_EXTENSIONS', default='') except KeyboardInterrupt: print("\n[*] User has requested an interrupt") print("[*] Application Exiting.....") @@ -499,9 +498,10 @@ def start(): #Main Program if __name__== "__main__": # load extensions - extensions = list(map(str.strip, enabled_extension.split(','))) - for extension in extensions: - Extension.register(extension) + #Extension.register("plugins.fediverse", "Fediverse") + #Extension.register("plugins.container", "Container") + Extension.register("plugins.wayback", "Wayback") + #Extension.register("plugins.bio", "PyBio") # start Caterpillar start() From 2e23938ca723c13a43312e1e0206038766880c5f Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:17:33 +0900 Subject: [PATCH 11/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1766995..31797a3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ CERT_KEY=cert.key CERT_DIR=certs/ OPENSSL_BINPATH=openssl CLIENT_ENCODING=utf-8 -ENABLED_EXTENSIONS=wayback +USE_EXTENSIONS=Wayback,PyBio ``` - (Optional) Create a certificate for SSL decryption From 0b94de24e9c028880f4a51aa2213ac7880bfb36d Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:31:41 +0900 Subject: [PATCH 12/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 31797a3..710be35 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ CERT_KEY=cert.key CERT_DIR=certs/ OPENSSL_BINPATH=openssl CLIENT_ENCODING=utf-8 -USE_EXTENSIONS=Wayback,PyBio +USE_EXTENSIONS=wayback.Wayback,bio.PyBio ``` - (Optional) Create a certificate for SSL decryption From 2d2e54cd2da6fde157041b6dcfdd498c9dcd4c84 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:38:19 +0900 Subject: [PATCH 13/20] Update base.py --- base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/base.py b/base.py index 1d121e1..1f98663 100644 --- a/base.py +++ b/base.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-05-20 -# Updated at: 2024-07-06 +# Updated at: 2024-07-09 # import hashlib @@ -72,13 +72,17 @@ class Extension(): cls.buffer_size = _buffer_size @classmethod - def register(cls, module_path, class_name): + def register(cls, s) + module_name = s.split('.')[0] + module_path = 'plugins.' + module_name + class_name = s.split('.')[-1] + try: module = importlib.import_module(module_path) _class = getattr(module, class_name) cls.extensions.append(_class()) except (ImportError, AttributeError) as e: - raise ImportError(class_name + " in " + module_path) + raise ImportError(class_name + " in the extension " + module_name) @classmethod def get_filters(cls): From bbb8c7fe5512cba427d7587822ca729ad1bae387 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:41:02 +0900 Subject: [PATCH 14/20] Update server.py --- server.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index 317e8bb..41f84da 100644 --- a/server.py +++ b/server.py @@ -47,6 +47,7 @@ try: client_encoding = config('CLIENT_ENCODING', default='utf-8') local_domain = config('LOCAL_DOMAIN', default='') proxy_pass = config('PROXY_PASS', default='') + use_extensions = config('USE_EXTENSIONS', default='') except KeyboardInterrupt: print("\n[*] User has requested an interrupt") print("[*] Application Exiting.....") @@ -498,10 +499,7 @@ def start(): #Main Program if __name__== "__main__": # load extensions - #Extension.register("plugins.fediverse", "Fediverse") - #Extension.register("plugins.container", "Container") - Extension.register("plugins.wayback", "Wayback") - #Extension.register("plugins.bio", "PyBio") + map(Extension.register, use_extension.split(','))) - # start Caterpillar + # start Caterpillar start() From d3f3b423c6b61f33716ac19fe537612818284053 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:41:15 +0900 Subject: [PATCH 15/20] Update server.py --- server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 41f84da..c3fbc9e 100644 --- a/server.py +++ b/server.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2022-10-06 -# Updated at: 2024-07-04 +# Updated at: 2024-07-09 # import argparse @@ -499,7 +499,7 @@ def start(): #Main Program if __name__== "__main__": # load extensions - map(Extension.register, use_extension.split(','))) + map(Extension.register, use_extension.split(',')) # start Caterpillar start() From 16bbddcd9445401a0aae39577c56dc0dd4be9a88 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:44:45 +0900 Subject: [PATCH 16/20] Update base.py --- base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base.py b/base.py index 1f98663..a538e2a 100644 --- a/base.py +++ b/base.py @@ -73,9 +73,8 @@ class Extension(): @classmethod def register(cls, s) - module_name = s.split('.')[0] + module_name, class_name = s.trim().split('.')[0:2] module_path = 'plugins.' + module_name - class_name = s.split('.')[-1] try: module = importlib.import_module(module_path) From 810d5041cb9130d4678104f4e824a04407587b27 Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:50:40 +0900 Subject: [PATCH 17/20] Update web.py --- web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web.py b/web.py index 857e360..422a8c3 100644 --- a/web.py +++ b/web.py @@ -94,6 +94,6 @@ if __name__ == "__main__": Extension.set_protocol('http') # load extensions - #Extension.register("plugins.YOUR_OWN_MODULE_NAME", "YOUR_OWN_CLASS_NAME"); + map(Extension.register, use_extension.split(',')) app.run(debug=True, host='0.0.0.0', port=listening_port) From 1064dc017b40a4b235f2b244e649f5fd3365cbed Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 16:50:48 +0900 Subject: [PATCH 18/20] Update web.py --- web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web.py b/web.py index 422a8c3..00586be 100644 --- a/web.py +++ b/web.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-05-20 -# Updated at: 2024-07-06 +# Updated at: 2024-07-09 # from flask import Flask, request, redirect, url_for, render_template From f5caf1cac75e315378029943812edb30fbe8ef3f Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Tue, 9 Jul 2024 17:02:29 +0900 Subject: [PATCH 19/20] Fix fix fix --- base.py | 4 ++-- server.py | 3 ++- web.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base.py b/base.py index a538e2a..47fcb50 100644 --- a/base.py +++ b/base.py @@ -72,8 +72,8 @@ class Extension(): cls.buffer_size = _buffer_size @classmethod - def register(cls, s) - module_name, class_name = s.trim().split('.')[0:2] + def register(cls, s): + module_name, class_name = s.strip().split('.')[0:2] module_path = 'plugins.' + module_name try: diff --git a/server.py b/server.py index c3fbc9e..b5679c8 100644 --- a/server.py +++ b/server.py @@ -499,7 +499,8 @@ def start(): #Main Program if __name__== "__main__": # load extensions - map(Extension.register, use_extension.split(',')) + for s in use_extensions.split(','): + Extension.register(s) # start Caterpillar start() diff --git a/web.py b/web.py index 00586be..88c0c17 100644 --- a/web.py +++ b/web.py @@ -94,6 +94,6 @@ if __name__ == "__main__": Extension.set_protocol('http') # load extensions - map(Extension.register, use_extension.split(',')) + map(Extension.register, use_extensions.split(',')) app.run(debug=True, host='0.0.0.0', port=listening_port) From a9783c6081baa62a635410525249dd079a0bc5aa Mon Sep 17 00:00:00 2001 From: "Namhyeon, Go" Date: Wed, 10 Jul 2024 09:28:06 +0900 Subject: [PATCH 20/20] One more fix #27 --- web.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web.py b/web.py index 88c0c17..6e4affa 100644 --- a/web.py +++ b/web.py @@ -7,7 +7,7 @@ # Namyheon Go (Catswords Research) # https://github.com/gnh1201/caterpillar # Created at: 2024-05-20 -# Updated at: 2024-07-09 +# Updated at: 2024-07-10 # from flask import Flask, request, redirect, url_for, render_template @@ -94,6 +94,7 @@ if __name__ == "__main__": Extension.set_protocol('http') # load extensions - map(Extension.register, use_extensions.split(',')) + for s in use_extensions.split(','): + Extension.register(s) app.run(debug=True, host='0.0.0.0', port=listening_port)