Merge branch 'certtools:develop' into develop

.github/workflows/build-docs.yml

@@ -1,39 +1,61 @@
-#Github Workflow to run test documentation built
-#
-#SPDX-FileCopyrightText: 2020 IntelMQ Team <[email protected]>
-#SPDX-License-Identifier: AGPL-3.0-or-later
-#
-name: "Build the documentation with sphinx"
+# SPDX-FileCopyrightText: 2023 Filip Pokorný
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: "Build and publish documentation"
+
 on:
   push:
-    branches: [develop, maintenance, master]
-    paths-ignore:
-      - '.github/**'
+    branches:
+      - develop
+      - maintenance
+      - mkdocs
+
   pull_request:
-    branches: [develop, maintenance]
-    paths-ignore:
-      - '.github/**'
+    branches:
+      - develop
+      - maintenance
+      - mkdocs
+
+  release:
+    types:
+      - published
+
+permissions:
+  contents: write
 
 jobs:
-  documentationbuild:
+  build:
     runs-on: ubuntu-latest
-    name: Build the documentation
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: "Checkout repository"
+        uses: actions/checkout@v3
+
+      - name: "Setup python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+
+      - name: "Install build dependencies"
+        run: |
+          pip install mkdocs-material mike lunr pygments mkdocstrings[python] mkdocs-material mkdocs-glightbox mkdocs-redirects mkdocs-minify-plugin
+
+      - name: "Prepare git"
+        run: |
+          git fetch origin gh-pages --depth=1
+          git config user.name intelmq-bot
+          git config user.email intelmq-bot
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
+      - name: "Build docs without publishing"
+        if: github.event_name == 'pull_request'
+        run: |
+          mkdocs build
 
-    - name: Install documentation dependencies
-      run: pip install -r docs/requirements.txt
+      - name: "Build docs with version tag and publish"
+        if: github.event_name == 'release'
+        run: |
+          mike deploy --push --update-aliases ${{ github.ref_name }} latest
 
-    - name: Build documentation
-      run: make -C docs html
+      - name: "Build docs with branch tag and publish"
+        if: github.event_name == 'push'
+        run: |
+          mike deploy --push ${{ github.ref_name }}

.github/workflows/codespell.excludelines

@@ -26,14 +26,8 @@ The correct name for the parameter "delimeter" is "delimiter". Please fix your c
 # intelmq/tests/bots/collectors/fireeye/first_request.json "attch"
 {"alert": [{"explanation": {"malwareDetected": {"malware": [{"md5Sum": "21232f297a57a5a743894a0e4a801fc3", "sha256": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", "name": "Downloader.Emotet"}]}, "osChanges": []}, "src": {"smtpMailFrom": "[email protected]"}, "alertUrl": "https://127.0.0.1/emps/eanalysis?e_id=161862&type=attch", "action": "blocked", "occurred": "2021-01-05 14:19:06 +0100", "dst": {"smtpTo": "[email protected]"}, "smtpMessage": {"subject": "Online Streaming am 30.06.2020 1800 Uhr REMINDER"}, "applianceId": "16EV1C1A6K94", "id": 1454270, "rootInfection": 7113664, "sensorIp": "127.0.0.1", "name": "MALWARE_OBJECT", "severity": "MAJR", "uuid": "1591de22-4926-4124-b3ed-ffff96766295", "ack": "no", "product": "EMAIL_MPS", "sensor": "mail", "vlan": 0, "malicious": "yes", "scVersion": "0000.000"}], "appliance": "CMS", "version": "CMS (CMS) 0.0.0.000000", "msg": "concise", "alertsCount": 1}
 
-# ./docs/user/universe.rst:55: bund ==> bind, bound
-Developed and maintained by `Intevation <https://intevation.de>`_, initially funded by `BSI <http://bsi.bund.de/>`_.
-
-# ./docs/dev/harmonization-fields.rst:27: compromized ==> compromised
-Destination                 destination.local_hostname          :ref:`string`                 Some sources report a internal hostname within a NAT related to the name configured for a compromized system
-
-# ./docs/dev/harmonization-fields.rst:28: compromized ==> compromised
-Destination                 destination.local_ip                :ref:`ipaddress`              Some sources report a internal (NATed) IP address related a compromized system. N.B. RFC1918 IPs are OK here.
+# ./docs/overview.md:60: bund ==> bind, bound
+Developed and maintained by [Intevation](https://intevation.de), initially funded by [BSI](https://bsi.bund.de/).
 
 # ./intelmq/tests/bots/parsers/shodan/test_parser.py:36: ALLO ==> ALLOW
                          '   ALLO   MLST   MLSD   SITE   P@SW   STRU   CLNT   MFMT\n'

.gitignore

@@ -11,7 +11,7 @@
 *.profile
 .vscode/
 .profile
-intelmq.egg-info
+*.egg-info
 build
 dist
 *.old
@@ -27,13 +27,11 @@ src/
 .eggs
 .secrets
 .venv/
+venv/
 .env
 
-# sphinx
-docs/source
-docs/_build
-docs/user/feeds.rst
-docs/dev/harmonization-fields.rst
+# mkdocs
+docs_build
 
 # Debian build filed
 debian/files

CHANGELOG.md

@@ -3,8 +3,8 @@
    SPDX-License-Identifier: AGPL-3.0-or-later
 -->
 
-CHANGELOG
-==========
+# CHANGELOG
+
 
 
 3.2.2 (unreleased)
@@ -19,26 +19,37 @@ CHANGELOG
     if `auth_by_ssl_client_certificate` is *false*);
   - `password` (STOMP authentication passcode, default: "guest"; to be used only
     if `auth_by_ssl_client_certificate` is *false*).
+- Add the possibility to set the `ssl_ca_certificate` configuration parameter for
+  `intelmq.bots.collectors.stomp.collector` and/or `intelmq.bots.outputs.stomp.output`
+  to an empty string - which means that the SSL machinery used for STOMP communication
+  will attempt to load the system’s default CA certificates (PR#2414 by Jan Kaliszewski).
 
 ### Core
 - `intelmq.lib.message`: For invalid message keys, add a hint on the failure to the exception: not allowed by configuration or not matching regular expression (PR#2398 by Sebastian Wagner).
 - `intelmq.lib.exceptions.InvalidKey`: Add optional parameter `additional_text` (PR#2398 by Sebastian Wagner).
+- Change the way we discover bots to allow easy extending based on the entry point name. (PR#2413 by Kamil Mankowski)
 - `intelmq.lib.mixins`: Add a new class, `StompMixin` (defined in a new submodule: `stomp`),
   which provides certain common STOMP-bot-specific operations, factored out from
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
-  (PR#2408 by Jan Kaliszewski).
+  (PR#2408 and PR#2414 by Jan Kaliszewski).
 
 ### Development
+- Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).
 
 ### Data Format
 
 ### Bots
 #### Collectors
-- `intelmq.bots.collectors.stomp.collector` (PR#2408 by Jan Kaliszewski):
-  - Add support for authentication based on STOMP login and passcode,
-    introducing 3 new configuration parameters (see above: *Configuration*).
+- `intelmq.bots.collectors.stomp.collector` (PR#2408 and PR#2414 by Jan Kaliszewski):
+  - Drop support for versions of `stomp.py` older than `4.1.12`.
   - Update the code to support new versions of `stomp.py`, including the latest (`8.1.0`);
     fixes [#2342](https://github.com/certtools/intelmq/issues/2342).
+  - Add support for authentication based on STOMP login and passcode, introducing three
+    new configuration parameters (see above: *Configuration*).
+  - Add support for loading the system’s default CA certificates, as an alternative to
+    specifying the CA certificate(s) file path explicitly (see above: *Configuration*).
+  - Fix (by carefully targeted monkey patching) certain security problems caused by
+    SSL-related weaknesses that some versions of `stomp.py` suffer from.
   - Fix the reconnection behavior: do not attempt to reconnect after `shutdown`. Also,
     never attempt to reconnect if the version of `stomp.py` is older than `4.1.21` (it
     did not work properly anyway).
@@ -52,26 +63,40 @@ CHANGELOG
 #### Parsers
 
 #### Experts
+- `intelmq.bots.experts.jinja` (PR#2417 by Mikk Margus Möll):
+  - Add optional `socket_perms` and `socket_group` parameters to change
+    file permissions on socket file, if it is in use.
 
 #### Outputs
-- `intelmq.bots.outputs.stomp.output` (PR#2408 by Jan Kaliszewski):
-  - Add support for authentication based on STOMP login and passcode,
-    introducing 3 new configuration parameters (see above: *Configuration*).
+- `intelmq.bots.outputs.stomp.output` (PR#2408 and PR#2414 by Jan Kaliszewski):
+  - Drop support for versions of `stomp.py` older than `4.1.12`.
   - Update the code to support new versions of `stomp.py`, including the latest (`8.1.0`).
+  - Add support for authentication based on STOMP login and passcode, introducing three
+    new configuration parameters (see above: *Configuration*).
+  - Add support for loading the system’s default CA certificates, as an alternative to
+    specifying the CA certificate(s) file path explicitly (see above: *Configuration*).
+  - Fix (by carefully targeted monkey patching) certain security problems caused by
+    SSL-related weaknesses that some versions of `stomp.py` suffer from.
   - Fix `AttributeError` caused by attempts to get unset attributes of `StompOutputBot`
     (`ssl_ca_cert` et consortes).
   - Add coercion of the `port` config parameter to `int`.
   - Add implementation of the `check` hook (verifying, in particular, accessibility
     of necessary file(s)).
-  - Add `stomp.py` version check (raise `MissingDependencyError` if not `>=4.1.8`).
+  - Add `stomp.py` version check (raise `MissingDependencyError` if not `>=4.1.12`).
   - Minor fixes/improvements and some refactoring (see also above: *Core*...).
 
 ### Documentation
 - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner).
+- Add a guide of developing extensions packages (PR#2413 by Kamil Mankowski)
 - Update/fix/improve the stuff related to the STOMP bots and integration with the *n6*'s
-  Stream API (PR#2408 by Jan Kaliszewski).
+  Stream API (PR#2408 and PR#2414 by Jan Kaliszewski).
+- Complete documentation overhaul. Change to markdown format. Uses the mkdocs-material (PR#2419 by Filip Pokorný).
 
 ### Packaging
+- Add `pendulum` to suggested packages, as it is required for the sieve bot (PR#2424 by Sebastian Wagner).
+- `debian/control`: in `Suggests` field, replace ``python3-stomp.py (>= 4.1.9)`` with
+  ``python3-stomp (>= 4.1.12)``, i.e., fix the package name by removing the `.py`
+  suffix and bump the minimum version to `4.1.12` (PR#2414 by Jan Kaliszewski).
 
 ### Tests
 

Makefile

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2023 Filip Pokorný
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+.PHONY: coverage docs clean
+
+coverage:
+	python -m pytest --cov=intelmq -v
+
+docs: mkdocs.yml docs/* intelmq/etc/feeds.yaml intelmq/etc/harmonization.conf intelmq/lib/harmonization.py
+	python3 scripts/generate-feeds-docs.py
+	python3 scripts/generate-event-docs.py
+	mkdocs build
+
+clean:
+	rm -rf docs_build .mypy_cache .coverage .pytest_cache dist
+
+codespell:
+	codespell -x .github/workflows/codespell.excludelines
+
+test:
+	pytest --no-cov -v intelmq/tests/ && echo "Success!"

README.md

@@ -0,0 +1 @@
+docs/index.md