[Analyzer] Bbot

.github/workflows/pull_request_automation.yml

@@ -155,4 +155,4 @@ jobs:
       - name: Test with Jest
         run: |
           npm run test -- --silent --coverage
-        working-directory: ./frontend
+        working-directory: ./frontend

api_app/analyzers_manager/classes.py

@@ -433,14 +433,19 @@ def _docker_run(
             self._raise_container_not_running()
 
         # step #2: raise AnalyzerRunException in case of error
-        if not self.__raise_in_case_bad_request(self.name, resp1):
-            raise AssertionError
-
-        # step #3: if no error, continue and try to fetch result
-        key = resp1.json().get("key")
-        final_resp = self.__poll_for_result(key)
-        err = final_resp.get("error", None)
-        report = final_resp.get("report", None)
+        # Modified to support synchronous analyzer BBOT that return results directly in the initial response, avoiding unnecessary polling.
+        if analyzer_name == "BBOT_Analyzer":
+            report = resp1.json().get("report", None)
+            err = resp1.json().get("error", None)
+        else:
+            if not self.__raise_in_case_bad_request(self.name, resp1):
+                raise AssertionError
+
+            # step #3: if no error, continue and try to fetch result
+            key = resp1.json().get("key")
+            final_resp = self.__poll_for_result(key)
+            err = final_resp.get("error", None)
+            report = final_resp.get("report", None)
 
         # APKiD provides empty result in case it does not support the binary type
         if not report and (analyzer_name != "APKiD"):

api_app/analyzers_manager/migrations/0154_analyzer_config_bbot.py

@@ -0,0 +1,193 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+    ReverseManyToOneDescriptor,
+    ReverseOneToOneDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "bbot.BBOT",
+        "base_path": "api_app.analyzers_manager.observable_analyzers",
+    },
+    "name": "BBOT",
+    "description": "[BBOT](https://github.com/blacklanternsecurity/bbot) (Bighuge BLS Open Threat) domain/URL scanner.\r\nLeverages BBOT's Python library to perform scans with configurable modules and presets.",
+    "disabled": False,
+    "soft_time_limit": 600,
+    "routing_key": "default",
+    "health_check_status": True,
+    "type": "observable",
+    "docker_based": True,
+    "maximum_tlp": "CLEAR",
+    "observable_supported": ["ip", "url", "domain"],
+    "supported_filetypes": [],
+    "run_hash": False,
+    "run_hash_type": "",
+    "not_supported_filetypes": [],
+    "mapping_data_model": {},
+    "model": "analyzers_manager.AnalyzerConfig",
+}
+
+params = [
+    {
+        "python_module": {
+            "module": "bbot.BBOT",
+            "base_path": "api_app.analyzers_manager.observable_analyzers",
+        },
+        "name": "modules",
+        "type": "list",
+        "description": "",
+        "is_secret": False,
+        "required": False,
+    },
+    {
+        "python_module": {
+            "module": "bbot.BBOT",
+            "base_path": "api_app.analyzers_manager.observable_analyzers",
+        },
+        "name": "presets",
+        "type": "list",
+        "description": "",
+        "is_secret": False,
+        "required": False,
+    },
+]
+
+values = [
+    {
+        "parameter": {
+            "python_module": {
+                "module": "bbot.BBOT",
+                "base_path": "api_app.analyzers_manager.observable_analyzers",
+            },
+            "name": "modules",
+            "type": "list",
+            "description": "",
+            "is_secret": False,
+            "required": False,
+        },
+        "analyzer_config": "BBOT",
+        "connector_config": None,
+        "visualizer_config": None,
+        "ingestor_config": None,
+        "pivot_config": None,
+        "for_organization": False,
+        "value": [],
+        "updated_at": "2025-03-08T13:26:48.196551Z",
+        "owner": None,
+    },
+    {
+        "parameter": {
+            "python_module": {
+                "module": "bbot.BBOT",
+                "base_path": "api_app.analyzers_manager.observable_analyzers",
+            },
+            "name": "presets",
+            "type": "list",
+            "description": "",
+            "is_secret": False,
+            "required": False,
+        },
+        "analyzer_config": "BBOT",
+        "connector_config": None,
+        "visualizer_config": None,
+        "ingestor_config": None,
+        "pivot_config": None,
+        "for_organization": False,
+        "value": ["web-basic"],
+        "updated_at": "2025-03-08T13:26:48.116399Z",
+        "owner": None,
+    },
+]
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [
+            ForwardManyToOneDescriptor,
+            ReverseManyToOneDescriptor,
+            ReverseOneToOneDescriptor,
+            ForwardOneToOneDescriptor,
+        ]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0071_delete_last_elastic_report"),
+        ("analyzers_manager", "0153_alter_spamhaus_drop_supported_observable"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/observable_analyzers/bbot.py

@@ -0,0 +1,102 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+import logging
+from urllib.parse import urlparse
+
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer
+from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from api_app.choices import Classification
+from api_app.models import PythonConfig
+from tests.mock_utils import MockUpResponse
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+class BBOT(ObservableAnalyzer, DockerBasedAnalyzer):
+    """
+    BBOT Docker-based analyzer for IntelOwl.
+    """
+
+    name: str = "BBOT_Analyzer"
+    url: str = "http://bbot_analyzer:5001/run"
+    max_tries: int = 25
+    poll_distance: int = 5
+
+    def __init__(self, config: PythonConfig, **kwargs):
+        super().__init__(config, **kwargs)
+        self.args: list[str] = []
+
+    def config(self, runtime_configuration: dict):
+        super().config(runtime_configuration)
+        target = self.observable_name
+
+        if self.observable_classification == Classification.URL:
+            logger.debug(f"Extracting hostname from URL: {target}")
+            target = urlparse(target).hostname
+
+        self.args.append(f"-t {target}")
+        self.args.extend([f"-p {preset}" for preset in self.presets])
+        self.args.extend([f"-m {module}" for module in self.modules])
+
+    def run(self):
+        """
+        Executes BBOT inside the Docker container via HTTP API.
+        """
+        req_data = {
+            "target": self.observable_name,
+            "presets": self.presets,
+            "modules": self.modules,
+        }
+
+        logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}")
+
+        try:
+            report = self._docker_run(req_data, analyzer_name=self.name)
+            logger.info(f"BBOT scan completed successfully with report: {report}")
+            return report
+        except Exception as e:
+            raise AnalyzerRunException(f"BBOT analyzer failed: {str(e)}")
+
+    def update(self):
+        pass
+
+    @staticmethod
+    def mocked_docker_analyzer_post(*args, **kwargs):
+        mock_response = {
+            "success": True,
+            "report": {
+                "events": [
+                    {
+                        "id": "SCAN:7804fe5d0d26eec716926da9a4002d4ceb171300",
+                        "name": "melodramatic_todd",
+                        "preset": {
+                            "flags": ["iis-shortnames", "web-basic"],
+                            "config": {
+                                "modules": {"iis_shortnames": {"detect_only": False}}
+                            },
+                            "description": "melodramatic_todd",
+                            "output_modules": ["json"],
+                        },
+                        "status": "FINISHED",
+                        "target": {
+                            "hash": "a2d3b5795582da7a4edc56ef63ae6d6866a70d9c",
+                            "seeds": ["test.com"],
+                            "blacklist": [],
+                            "seed_hash": "1f26e4e291bfa260f77d2411c88906aee99786c5",
+                            "whitelist": ["test.com"],
+                            "scope_hash": "86df039469ae73720ac0d8cdd7cf92c3953659b4",
+                            "strict_scope": False,
+                            "blacklist_hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                            "whitelist_hash": "1f26e4e291bfa260f77d2411c88906aee99786c5",
+                        },
+                        "duration": "52 seconds",
+                        "started_at": "2025-03-18T14:30:59.131139",
+                        "finished_at": "2025-03-18T14:31:51.854936",
+                        "duration_seconds": 52.723797,
+                    }
+                ],
+                "json_output": [],
+            },
+        }
+        return MockUpResponse(mock_response, 200)

integrations/bbot/Dockerfile

@@ -0,0 +1,41 @@
+FROM python:3.12-slim
+
+# Environment variables
+ENV PROJECT_PATH=/opt/deploy/bbot
+ENV USER=bbot-user
+ENV HOME=${PROJECT_PATH}
+ENV BBOT_HOME=${PROJECT_PATH}
+
+# Create a non-root user
+RUN useradd -ms /bin/bash ${USER}
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    debianutils build-essential libssl-dev libffi-dev cargo openssl \
+    libpq-dev curl unzip git make bash tar p7zip-full p7zip && \
+    apt-get clean && apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /usr/share/doc/* /usr/share/man/*
+
+
+# Upgrade pip and install Python packages
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir fastapi uvicorn[standard] bbot
+
+# Pre-install BBOT dependencies
+RUN bbot --install-all-deps -y --force
+
+# Set up project directory
+WORKDIR ${PROJECT_PATH}
+
+# Copy application files
+COPY --chown=${USER}:${USER} app.py ./
+
+# Make scripts executable
+RUN chmod u+x app.py
+
+# Expose port
+EXPOSE 5001
+
+# Entrypoint
+# ENTRYPOINT ["./entrypoint.sh"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5001", "--log-level", "debug"]