2.2.0 (#1069)

* local files, nginx

* api test code

* vt/maxmind error, update notebook

* python3.12 - go1.22 - ws updates

* #1030 update - #947 method 1

* update libs

* upgrade tree view

* fix download dump error

* keep pip for volatility for shared install path

* improve plugin install, regipy plugins

* fix admin upload plugins

* #1068

* update changelog

---------

.env.sample

@@ -0,0 +1 @@
+VERSION=latest

.envs/.local/.django

@@ -7,6 +7,14 @@ IPYTHONDIR=/app/.ipython
 # ------------------------------------------------------------------------------
 ALLOWED_HOSTS=*
 
+# Smtp
+# ------------------------------------------------------------------------------
+EMAIL_HOST=mailpit
+EMAIL_PORT=1025
+EMAIL_USE_TLS=False
+EMAIL_HOST_USER=
+EMAIL_HOST_PASSWORD=
+
 # Redis
 # ------------------------------------------------------------------------------
 REDIS_URL=redis://redis:6379/0
@@ -29,6 +37,21 @@ AWESOME_PATH=https://raw.githubusercontent.com/InQuest/awesome-yara/master/READM
 LOCAL_YARA_PATH=/yara
 DEFAULT_YARA_RULE_PATH=/yara/default.yara
 VOLATILITY_SYMBOL_DOWNLOAD_PATH=https://downloads.volatilityfoundation.org/volatility3/symbols
+LOCAL_UPLOAD_PATH=/uploads
+# $ from regipy.plugins.plugin import PLUGINS
+# $ [plugin.NAME for plugin in PLUGINS]
+# ['amcache', 'terminal_services_history', 'bootkey', 'last_logon_plugin', 'winscp_saved_sessions',
+# 'ras_tracing', 'active_control_set', 'timezone_data', 'safeboot_configuration', 'uac_plugin',
+# 'user_assist', 'word_wheel_query', 'print_demon_plugin', 'shimcache', 'usbstor_plugin', 'wdigest',
+# 'installed_programs_ntuser', 'software_classes_installer', 'installed_programs_software', 'background_activity_moderator',
+# 'ntuser_classes_installer', 'typed_urls', 'usrclass_shellbag_plugin', 'network_data', 'image_file_execution_options',
+# 'domain_sid', 'routes', 'boot_entry_list', 'ntuser_shellbag_plugin', 'typed_paths', 'services', 'local_sid',
+# 'host_domain_name', 'profilelist_plugin', 'winrar_plugin', 'ntuser_persistence', 'computer_name', 'network_drives_plugin', 'software_plugin']
+REGIPY_PLUGINS=computer_name,host_domain_name,network_drives_plugin
+
+# Debug
+# ------------------------------------------------------------------------------
+DEBUG_LEVEL=WARNING
 
 # Ldap
 # ------------------------------------------------------------------------------
@@ -47,4 +70,4 @@ no_proxy=
 
 # CSRF_TRUSTED_ORIGINS
 # ------------------------------------------------------------------------------
-CSRF_TRUSTED_ORIGINS=
+CSRF_TRUSTED_ORIGINS=https://orochi.dev,https://localhost

.gitignore

@@ -271,7 +271,12 @@ compose/local/clamav/freshclam.conf
 
 # History
 .history
+
+# Temporary test files
 orochi/utils/test.py
+docker-compose__tmp.yml
+
+# Maxmind
 compose/local/maxmind/GeoLite2-ASN.mmdb
 compose/local/maxmind/GeoLite2-City.mmdb
-compose/local/maxmind/GeoLite2-Country.mmdb
+compose/local/maxmind/GeoLite2-Country.mmdb

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
       - id: check-yaml
 
   - repo: https://github.com/psf/black
-    rev: 24.1.1
+    rev: 24.3.0
     hooks:
       - id: black
 

CHANGELOG.md

@@ -1,10 +1,23 @@
 ## Changelog
 
-
 <details open>
-  <summary><b>OROCHI 2.2.0</b></summary>
+  <summary><b>OROCHI 2.2.0 [2024/03/26]</b></summary>
+
+  * Upload ntoskrnl.exe and generate symbol [[#1020](https://github.com/LDO-CERT/orochi/issues/1020)]
+  * evaluate possibility to switch from daphne to uvicorn for asgi [[#982](https://github.com/LDO-CERT/orochi/issues/982)]
+  * Improve path flexibility for local import [[#451](https://github.com/LDO-CERT/orochi/issues/451)]
+  * uv for installing requirements [[#1030](https://github.com/LDO-CERT/orochi/issues/1030)]
+  * Read only users for educational. [[#947](https://github.com/LDO-CERT/orochi/issues/947)]
+  * Add use case example with API. [[#248](https://github.com/LDO-CERT/orochi/issues/248)]
+  * put custom plugins under volatility3 /plugins/ [[#1068](https://github.com/LDO-CERT/orochi/issues/1068)]
+  * Improve tree rendered plugins
+</details>
+
+<details>
+  <summary><b>OROCHI 2.1.1 [2024/02/13]</b></summary>
 
   * ADD more info on foreign addr in netstat [[#494](https://github.com/LDO-CERT/orochi/issues/494)]
+  * Expand/Collapse folders [[#1006](https://github.com/LDO-CERT/orochi/issues/1006)]
 </details>
 
 <details open>

compose/local/dask/Dockerfile

@@ -1,6 +1,8 @@
-FROM daskdev/dask:2024.1.1-py3.11
+FROM daskdev/dask:2024.3.1-py3.12
 ENV DEBIAN_FRONTEND noninteractive
 
+ARG local_folder=/uploads
+
 RUN apt-get --allow-releaseinfo-change update \
   # dependencies for building Python packages
   && apt-get install -y --no-install-recommends build-essential \
@@ -45,14 +47,18 @@ RUN python setup.py build \
 # Workers should have similar reqs as django
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install -r /requirements/base.txt --ignore-installed ruamel.yaml
+RUN pip install uv==0.1.21 -e git+https://github.com/volatilityfoundation/volatility3.git@55dd39f2ba60ffdd2126b7ea011940f0df42815a#egg=volatility3 \
+  && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh
 RUN chmod +x /usr/bin/prepare.sh
 
 COPY ./compose/local/dask/daskworkerinit.py /opt/daskworkerinit.py
 COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py
 
+RUN mkdir -p $local_folder /maxmind
+COPY ./compose/local/maxmind /maxmind
+
 WORKDIR /app
 COPY . .
 ENV PYTHONPATH "${PYTHONPATH}:/app"

compose/local/django/Dockerfile

@@ -1,9 +1,11 @@
-FROM python:3.11-slim-buster as common-base
+FROM python:3.12-slim-bookworm as common-base
 
 ENV DJANGO_SETTINGS_MODULE config.settings.local
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 
+ARG local_folder=/uploads
+
 RUN apt-get update \
   # dependencies for building Python packages
   && apt-get install -y --no-install-recommends build-essential \
@@ -54,22 +56,21 @@ RUN python setup.py build
 
 FROM base-builder as go-builder
 WORKDIR /
-RUN curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz --output go1.15.2.linux-amd64.tar.gz \
-  && tar -C /usr/local -xzf go1.15.2.linux-amd64.tar.gz
-RUN curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz --output go1.15.2.linux-amd64.tar.gz \
-  && tar -C /usr/local -xzf go1.15.2.linux-amd64.tar.gz
+RUN curl https://dl.google.com/go/go1.22.0.linux-amd64.tar.gz --output go1.22.0.linux-amd64.tar.gz \
+  && tar -C /usr/local -xzf go1.22.0.linux-amd64.tar.gz
 RUN git clone https://github.com/volatilityfoundation/dwarf2json.git
 WORKDIR /dwarf2json
 RUN /usr/local/go/bin/go build
 
 FROM common-base
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install -r /requirements/local.txt --ignore-installed ruamel.yaml
+RUN pip install uv==0.1.21 -e git+https://github.com/volatilityfoundation/volatility3.git@55dd39f2ba60ffdd2126b7ea011940f0df42815a#egg=volatility3 \
+  && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py
 
-RUN mkdir /maxmind
+RUN mkdir -p $local_folder /maxmind
 COPY ./compose/local/maxmind /maxmind
 
 WORKDIR /app

compose/local/django/start

@@ -7,4 +7,4 @@ set -o nounset
 python manage.py makemigrations
 python manage.py migrate
 python manage.py collectstatic --skip-checks --noinput --clear
-daphne -b 0.0.0.0 -p 8000 config.asgi:application
+gunicorn config.wsgi:application --workers=2 --timeout 1800 -b :8000

compose/local/nginx/Dockerfile

@@ -0,0 +1,19 @@
+FROM alpine:latest as alpine
+ARG DOMAIN_NAME=orochi.dev
+ARG DAYS_VALID=365
+
+RUN apk add --no-cache openssl
+RUN echo "Creating self-signed certificate valid for ${DAYS_VALID} days for domain ${DOMAIN_NAME}" && \
+  openssl \
+  req -x509 \
+  -nodes \
+  -subj "/CN=${DOMAIN_NAME}" \
+  -addext "subjectAltName=DNS:${DOMAIN_NAME}" \
+  -days ${DAYS_VALID} \
+  -newkey rsa:2048 -keyout /tmp/self-signed.key \
+  -out /tmp/self-signed.crt
+
+# Prepare an NGINX-based image with the certificate created above.
+FROM nginx:latest as nginx
+COPY --from=alpine /tmp/self-signed.key /etc/ssl/private
+COPY --from=alpine /tmp/self-signed.crt /etc/ssl/certs

compose/local/nginx/conf/default.conf

@@ -0,0 +1,57 @@
+upstream django_wsgi {
+  server  django_wsgi:8000;
+}
+
+upstream django_asgi {
+    server django_asgi:9000;
+}
+
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+    '' close;
+}
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+
+    proxy_connect_timeout 50000;
+    proxy_read_timeout 50000;
+    proxy_send_timeout 50000;
+
+    server_name orochi.dev;
+
+    location = /favicon.ico { access_log off; log_not_found off; }
+
+    ssl_certificate /etc/ssl/certs/self-signed.crt;
+    ssl_certificate_key /etc/ssl/private/self-signed.key;
+
+    location / {
+      proxy_pass http://django_wsgi;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header Host $host;
+      proxy_redirect off;
+
+      client_max_body_size 1000M;
+    }
+
+    location /ws/notify/ {
+      proxy_pass http://django_asgi;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+
+      proxy_redirect     off;
+      proxy_set_header   Host $host;
+      proxy_set_header   X-Real-IP $remote_addr;
+      proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header   X-Forwarded-Host $server_name;
+    }
+}

config/routing.py

@@ -1,13 +1,15 @@
-import orochi.website.routing
-from django.core.asgi import get_asgi_application
 from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
+from channels.security.websocket import AllowedHostsOriginValidator
+from django.core.asgi import get_asgi_application
+
+import orochi.website.routing
 
 application = ProtocolTypeRouter(
     {
         "http": get_asgi_application(),
-        "websocket": AuthMiddlewareStack(
-            URLRouter(orochi.website.routing.websocket_urlpatterns)
+        "websocket": AllowedHostsOriginValidator(
+            AuthMiddlewareStack(URLRouter(orochi.website.routing.websocket_urlpatterns))
         ),
     }
 )

config/settings/base.py

@@ -228,6 +228,7 @@
 
 # LOGGING
 # ------------------------------------------------------------------------------
+DEBUG_LEVEL = env("DEBUG_LEVEL", default="WARNING")
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -239,14 +240,14 @@
     },
     "handlers": {
         "console": {
-            "level": "INFO",
+            "level": DEBUG_LEVEL,
             "class": "logging.StreamHandler",
             "formatter": "verbose",
         }
     },
-    "root": {"level": "INFO", "handlers": ["console"]},
+    "root": {"level": DEBUG_LEVEL, "handlers": ["console"]},
     "loggers": {
-        "distributed": {"level": "INFO", "handlers": ["console"]},
+        "distributed": {"level": DEBUG_LEVEL, "handlers": ["console"]},
     },
 }
 
@@ -275,7 +276,7 @@
 ASGI_APPLICATION = "config.routing.application"
 CHANNEL_LAYERS = {
     "default": {
-        "BACKEND": "channels_redis.pubsub.RedisPubSubChannelLayer",
+        "BACKEND": "channels_redis.core.RedisChannelLayer",
         "CONFIG": {
             "hosts": [(env("REDIS_SERVER"), env("REDIS_PORT"))],
         },
@@ -329,8 +330,12 @@
 RULES_ANALYSIS_INDEX = "rules_analysis"
 # local path of volatility folder
 VOLATILITY_SYMBOL_PATH = "/src/volatility3/volatility3/symbols"
-VOLATILITY_PLUGIN_PATH = "/src/volatility3/volatility3/framework/plugins/custom"
+VOLATILITY_PLUGIN_PATH = "/src/volatility3/volatility3/plugins/custom"
 # local path of dwarg2json executable
 DWARF2JSON = "/dwarf2json/./dwarf2json"
 # online path of volatility symbols
 VOLATILITY_SYMBOL_DOWNLOAD_PATH = env("VOLATILITY_SYMBOL_DOWNLOAD_PATH")
+# path of a remote folder with already uploaded files
+LOCAL_UPLOAD_PATH = env("LOCAL_UPLOAD_PATH")
+# Regipy plugins
+REGIPY_PLUGINS = env.list("REGIPY_PLUGINS")

config/settings/local.py

@@ -27,7 +27,7 @@
 
 # EMAIL
 # ------------------------------------------------------------------------------
-EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
+EMAIL_HOST = env("EMAIL_HOST", default="mailpit")
 EMAIL_PORT = 1025
 EMAIL_USE_TLS = env("EMAIL_USE_TLS", default=True)
 EMAIL_HOST_USER = env("EMAIL_HOST_USER", default=None)

docker-compose-swarm.yml

@@ -68,7 +68,7 @@ services:
     volumes:
       - media_path:/media
       - symbols_path:/src/volatility3/volatility3/symbols
-      - plugin_path:/src/volatility3/volatility3/framework/plugins/custom
+      - plugin_path:/src/volatility3/volatility3/plugins/custom
       - yara_path:/yara
       - cache_path:/root/.cache/volatility3
       - ./compose/local/__init__.py:/src/volatility3/volatility3/framework/constants/__init__.py
@@ -178,7 +178,7 @@ services:
     volumes:
       - media_path:/media
       - symbols_path:/src/volatility3/volatility3/symbols
-      - plugin_path:/src/volatility3/volatility3/framework/plugins/custom
+      - plugin_path:/src/volatility3/volatility3/plugins/custom
       - yara_path:/yara
       - cache_path:/root/.cache/volatility3
 
@@ -213,7 +213,7 @@ services:
     volumes:
       - media_path:/media
       - symbols_path:/src/volatility3/volatility3/symbols
-      - plugin_path:/src/volatility3/volatility3/framework/plugins/custom
+      - plugin_path:/src/volatility3/volatility3/plugins/custom
       - yara_path:/yara
       - cache_path:/root/.cache/volatility3
       - ./compose/local/__init__.py:/src/volatility3/volatility3/framework/constants/__init__.py