Merge branch 'main' into logging-system

.gitignore

@@ -1,6 +1,178 @@
 certs/
 savedfiles/
+logs/
 settings.ini
 .env
-*.pyc
+*.crt
-logs/**
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json

README.md

@@ -36,6 +36,7 @@ CERT_KEY=cert.key
 CERT_DIR=certs/
 OPENSSL_BINPATH=openssl
 CLIENT_ENCODING=utf-8
+USE_EXTENSIONS=wayback.Wayback,bio.PyBio
 ```
 
 - (Optional) Create a certificate for SSL decryption

base.py

@@ -8,7 +8,7 @@
 # Euiseo Cha (Wonkwang University) <zeroday0619_dev@outlook.com>
 # https://github.com/gnh1201/caterpillar
 # Created at: 2024-05-20
-# Updated at: 2024-07-06
+# Updated at: 2024-07-09
 #
 
 import logging
@@ -78,13 +78,16 @@ class Extension():
         cls.buffer_size = _buffer_size
 
     @classmethod
-    def register(cls, module_path, class_name):
+    def register(cls, s):
+        module_name, class_name = s.strip().split('.')[0:2]
+        module_path = 'plugins.' + module_name
+
         try:
             module = importlib.import_module(module_path)
             _class = getattr(module, class_name)
             cls.extensions.append(_class())
         except (ImportError, AttributeError) as e:
-            raise ImportError(class_name + " in " + module_path)
+            raise ImportError(class_name + " in the extension " + module_name)
 
     @classmethod
     def get_filters(cls):

plugins/nmap.py

@@ -7,7 +7,7 @@
 # Namyheon Go (Catswords Research) <gnh1201@gmail.com>
 # https://github.com/gnh1201/caterpillar
 # Created at: 2022-01-26 (github.com/gnh1201/welsonjs)
-# Updated at: 2024-07-04
+# Updated at: 2024-07-09
 #
 import sys
 import nmap
@@ -25,7 +25,7 @@ class PortScanner(Extension):
         hosts = params['hosts']
         binpath = params['binpath']
 
-        #result = nm.scan(hosts=hosts, arguments='-T5 -sV -p21-25,80,139,443,445,1883,2179,2323,3389,7547,8080,8443,8883')
+        nm = nmap.PortScanner(nmap_search_path=(binpath,))
         result = nm.scan(hosts=hosts, arguments='-T5 -sV -p0-65535 --max-retries 0')
 
         return result;

requirements-dev.txt

@@ -0,0 +1 @@
+ruff==0.5.1

ruff.toml

@@ -0,0 +1,69 @@
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+    "assets",
+    "data"
+]
+
+target-version = "py310"
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = ["E501"]
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+
+[format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"

server.py

@@ -7,7 +7,7 @@
 # Namyheon Go (Catswords Research) <gnh1201@gmail.com>
 # https://github.com/gnh1201/caterpillar
 # Created at: 2022-10-06
-# Updated at: 2024-07-04
+# Updated at: 2024-07-09
 #
 
 import argparse
@@ -49,6 +49,7 @@ try:
     client_encoding = config('CLIENT_ENCODING', default='utf-8')
     local_domain = config('LOCAL_DOMAIN', default='')
     proxy_pass = config('PROXY_PASS', default='')
+    use_extensions = config('USE_EXTENSIONS', default='')
 except KeyboardInterrupt:
     logger.warning("[*] User has requested an interrupt")
     logger.warning("[*] Application Exiting.....")
@@ -500,10 +501,8 @@ def start():    #Main Program
 
 if __name__== "__main__":
     # load extensions
-    #Extension.register("plugins.fediverse", "Fediverse")
+    for s in use_extensions.split(','):
-    #Extension.register("plugins.container", "Container")
+        Extension.register(s)
-    Extension.register("plugins.wayback", "Wayback")
-    #Extension.register("plugins.bio", "PyBio")
 
-   # start Caterpillar
+    # start Caterpillar
     start()

web.py

@@ -7,7 +7,7 @@
 # Namyheon Go (Catswords Research) <gnh1201@gmail.com>
 # https://github.com/gnh1201/caterpillar
 # Created at: 2024-05-20
-# Updated at: 2024-07-06
+# Updated at: 2024-07-10
 #
 
 from flask import Flask, request, redirect, url_for, render_template
@@ -97,6 +97,7 @@ if __name__ == "__main__":
     Extension.set_protocol('http')
 
     # load extensions
-    #Extension.register("plugins.YOUR_OWN_MODULE_NAME", "YOUR_OWN_CLASS_NAME");
+    for s in use_extensions.split(','):
+        Extension.register(s)
 
     app.run(debug=True, host='0.0.0.0', port=listening_port)