Merge pull request #407 from opencybersecurityalliance/develop

v1.7.6

CHANGELOG.rst

@@ -9,6 +9,21 @@ The format is based on `Keep a Changelog`_.
 Unreleased
 ==========
 
+1.7.6 (2023-09-25)
+==================
+
+Added
+-----
+
+- ``DESCRIBE`` command to get insight of attributes
+- ``ikestrel`` interactive shell (command-line utility)
+- Custom stix-shifter connector support #402
+
+Fixed
+-----
+
+- Command-line utility tests failed without install
+
 1.7.5 (2023-09-07)
 ==================
 

bin/ikestrel

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+# Executing a hunt in an interactive CLI
+# Usage: `ikestrel [-v] [--debug]`
+
+import argparse
+import cmd
+import logging
+
+from tabulate import tabulate
+
+from kestrel.session import Session
+from kestrel.codegen.display import DisplayBlockSummary, DisplayDataframe
+from kestrel.exceptions import KestrelException
+from kestrel.utils import add_logging_handler, clear_logging_handlers
+
+
+CMDS = [  # command_no_result from kestrel.lark
+    "APPLY",
+    "DISP",
+    "INFO",
+    "SAVE",
+]
+
+
+def display_outputs(outputs):
+    for i in outputs:
+        if isinstance(i, DisplayBlockSummary):
+            print(i.to_string())
+        elif isinstance(i, DisplayDataframe):
+            data = i.to_dict()["data"]
+            print(tabulate(data, headers="keys"))
+        else:
+            print(i.to_string())
+
+
+class Cli(cmd.Cmd):
+    prompt = "> "
+
+    def __init__(self, session: Session):
+        self.session = session
+        self.buf = ""
+        super().__init__()
+
+    def default(self, line: str):
+        try:
+            outputs = self.session.execute(line)
+            display_outputs(outputs)
+        except KestrelException as e:
+            print(e)
+
+    def completenames(self, text, *ignored):
+        code, _start, _end = ignored
+        if code.isupper():
+            # Probably a command?
+            results = [i for i in CMDS if i.startswith(code)]
+        else:
+            # Try all commands and vars
+            results = [i for i in CMDS if i.lower().startswith(code)]
+            results += [
+                i for i in self.session.get_variable_names() if i.startswith(code)
+            ]
+        return results
+
+    def completedefault(self, *ignored):
+        _, code, start, end = ignored
+        results = self.session.do_complete(code, end)
+        stub = code[start:]
+        return [stub + suffix for suffix in results]
+
+    def do_EOF(self, _line: str):
+        print()
+        return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Kestrel Interpreter")
+    parser.add_argument(
+        "-v", "--verbose", help="print verbose log", action="store_true"
+    )
+    parser.add_argument(
+        "--debug", help="debug level log (default is info level)", action="store_true"
+    )
+    args = parser.parse_args()
+
+    clear_logging_handlers()
+    if args.verbose:
+        add_logging_handler(logging.StreamHandler(), args.debug)
+
+    with Session(debug_mode=args.debug) as s:
+        cli = Cli(s)
+        cli.cmdloop()

bin/stix-shifter-diag

@@ -3,7 +3,7 @@
 import argparse
 import datetime
 from kestrel_datasource_stixshifter.diagnosis import Diagnosis
-from kestrel_datasource_stixshifter.connector import check_module_availability
+from kestrel_datasource_stixshifter.connector import setup_connector_module
 from firepit.timestamp import timefmt
 
 
@@ -83,7 +83,7 @@ if __name__ == "__main__":
     diag.diagnose_config()
 
     # 2. setup connector and ping
-    check_module_availability(diag.connector_name, args.ignore_cert)
+    setup_connector_module(diag.connector_name, diag.allow_dev_connector, args.ignore_cert)
 
     # 3. query translation test
     diag.diagnose_translate_query(patterns[0])

docs/language/commands.rst

@@ -7,7 +7,7 @@ A Kestrel command describes a :ref:`language/tac:hunt step` in one of the five c
 #. Retrieval: ``GET``, ``FIND``, ``NEW``.
 #. Transformation: ``SORT``, ``GROUP``.
 #. Enrichment: ``APPLY``.
-#. Inspection: ``INFO``, ``DISP``.
+#. Inspection: ``INFO``, ``DISP``, ``DESCRIBE``.
 #. Flow-control: ``SAVE``, ``LOAD``, ``ASSIGN``, ``MERGE``, ``JOIN``.
 
 To achieve :ref:`language/tac:composable hunt flow` and allow threat hunters to compose hunt
@@ -47,6 +47,8 @@ object, or both a variable and a display object.
 +---------+----------------+---------------+----------------+---------------+
 | DISP    | yes            | maybe         | no             | yes           |
 +---------+----------------+---------------+----------------+---------------+
+| DESCRIBE | yes           | no            | no             | yes           |
++---------+----------------+---------------+----------------+---------------+
 | SORT    | yes            | yes           | yes            | no            |
 +---------+----------------+---------------+----------------+---------------+
 | GROUP   | yes            | yes           | yes            | no            |
@@ -674,6 +676,41 @@ Examples
     # display the timestamps from observations of those processes:
     DISP TIMESTAMPED(procs) ATTR pid, name, command_line
 
+DESCRIBE
+--------
+
+The command ``DESCRIBE`` is an *inspection* hunt step to show
+descriptive statistics of a Kestrel variable attribute.
+
+Syntax
+^^^^^^
+::
+
+    DESCRIBE varx.attr
+
+The command shows the following information of an numeric attribute:
+
+- count: the number of non-NULL values
+- mean: the average value
+- min: the minimum value
+- max: the maximum value
+
+The command shows the following information of other attributes:
+
+- count: the number of non-NULL values
+- unique: the number of unique values
+- top: the most freqently occurring value
+- freq: the number of occurrences of the top value
+
+Examples
+^^^^^^^^
+
+.. code-block:: coffeescript
+
+    # showing information like unique count of src_port
+    nt = GET network-traffic FROM stixshifter://idsX WHERE dst_port = 80
+    DESCRIBE nt.src_port
+
 SORT
 ----
 

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools >= 56.0.0", "wheel"]
+requires = ["setuptools >= 68.2.2", "wheel"]
 build-backend = "setuptools.build_meta"

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kestrel-lang
-version = 1.7.5
+version = 1.7.6
 description = Kestrel Threat Hunting Language
 long_description = file:README.rst
 long_description_content_type = text/x-rst
@@ -26,21 +26,22 @@ package_dir =
     =src
 scripts =
     bin/kestrel
+    bin/ikestrel
     bin/stix-shifter-diag
 python_requires = >= 3.8
 install_requires =
-    typeguard>=4.1.3
+    typeguard>=4.1.5
     pyyaml>=6.0.1
     lxml>=4.9.3
     lark>=1.1.7
-    pandas>=2.0.0
+    pandas>=2.0.3
     pyarrow>=13.0.0
     docker>=6.1.3
     requests>=2.31.0
-    nest-asyncio>=1.5.7
+    nest-asyncio>=1.5.8
     stix-shifter==6.2.1
     stix-shifter-utils==6.2.1
-    firepit>=2.3.27
+    firepit>=2.3.29
 tests_require =
     pytest
 

src/kestrel/codegen/commands.py

@@ -25,7 +25,18 @@
 
 from firepit.deref import auto_deref
 from firepit.exceptions import InvalidAttr
-from firepit.query import Limit, Offset, Order, Predicate, Projection, Query
+from firepit.query import (
+    Aggregation,
+    Column,
+    Limit,
+    Group,
+    Offset,
+    Order,
+    Predicate,
+    Projection,
+    Query,
+    Table,
+)
 from firepit.stix20 import summarize_pattern
 
 from kestrel.utils import remove_empty_dicts, dedup_ordered_dicts
@@ -223,6 +234,57 @@ def disp(stmt, session):
     return None, DisplayDataframe(dedup_ordered_dicts(remove_empty_dicts(content)))
 
 
+@_debug_logger
+@_skip_command_if_empty_input
+def describe(stmt, session):
+    entity_table = session.symtable[stmt["input"]].entity_table
+    attribute = stmt["attribute"]
+    schema = {i["name"]: i["type"] for i in session.store.schema(entity_table)}
+    attr_type = schema[attribute].lower()
+
+    result = OrderedDict()
+
+    qry = Query(entity_table)
+    if attr_type in ("integer", "bigint", "numeric"):
+        qry.append(
+            Aggregation(
+                [
+                    ("COUNT", attribute, "count"),
+                    ("AVG", attribute, "mean"),
+                    ("MIN", attribute, "min"),
+                    ("MAX", attribute, "max"),
+                ]
+            )
+        )
+    else:
+        qry.append(
+            Aggregation(
+                [("COUNT", attribute, "count"), ("NUNIQUE", attribute, "unique")]
+            )
+        )
+        cursor = session.store.run_query(qry)
+        content = cursor.fetchall()[0]
+        result.update(content)
+
+        # Need second query for top and freq
+        qry = Query(
+            [
+                Table(entity_table),
+                Group([Column(attribute, alias="top")]),
+                Aggregation([("COUNT", "*", "freq")]),
+                Order([("freq", Order.DESC)]),
+                Limit(1),
+            ]
+        )
+
+    cursor = session.store.run_query(qry)
+    content = cursor.fetchall()[0]
+
+    result.update(content)
+
+    return None, DisplayDict(result)
+
+
 @_debug_logger
 @_default_output
 @_skip_command_if_empty_input

src/kestrel/syntax/kestrel.lark

@@ -31,6 +31,7 @@ assignment: VARIABLE "=" expression
                   | disp
                   | info
                   | save
+                  | describe
 
 //
 // All commands
@@ -58,6 +59,8 @@ info: "INFO"i VARIABLE
 
 save: "SAVE"i VARIABLE "TO"i stdpath
 
+describe: "DESCRIBE"i var_attr
+
 //
 // Variable definition
 //
@@ -255,6 +258,8 @@ literal_list: "(" literal ("," literal)* ")"
 
 reference_or_simple_string: ECNAME ("." ATTRIBUTE)?
 
+var_attr: ECNAME "." ATTRIBUTE
+
 ?string: advanced_string
 
 number: NUMBER

src/kestrel/syntax/parser.py

@@ -100,6 +100,11 @@ def disp(self, args):
             packet["attrs"] = "*"
         return packet
 
+    def describe(self, args):
+        packet = {"command": "describe"}
+        packet.update(args[0])
+        return packet
+
     def get(self, args):
         packet = {
             "command": "get",
@@ -272,6 +277,9 @@ def literal_list(self, args):
     def literal(self, args):
         return args[0]
 
+    def var_attr(self, args):
+        return {"input": _first(args), "attribute": _second(args)}
+
     def reference_or_simple_string(self, args):
         if len(args) > 1:
             variable = _first(args)