From 6beee679bb2b074cee3d37ab0c1eff1f0eff0142 Mon Sep 17 00:00:00 2001 From: Ming Fang Date: Wed, 4 Dec 2024 00:09:23 -0500 Subject: [PATCH 1/4] Initial changes to support SQLite backend. --- .../database_backend_base.py | 3 ++ .../database_backends/sqlite_backend.py | 44 +++++++++++++++++++ .../relational_db/relational_db_testing.py | 4 +- 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 stix2/datastore/relational_db/database_backends/sqlite_backend.py diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py index e5082451..4cf46f1b 100644 --- a/stix2/datastore/relational_db/database_backends/database_backend_base.py +++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py @@ -18,6 +18,9 @@ def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any) self.database_connection = create_engine(database_connection_url) + def _fk_pragma_on_connect(self): + self.database_connnection.execute('pragma foreign_keys=ON') + def _create_database(self): if self.database_exists: drop_database(self.database_connection_url) diff --git a/stix2/datastore/relational_db/database_backends/sqlite_backend.py b/stix2/datastore/relational_db/database_backends/sqlite_backend.py new file mode 100644 index 00000000..ae5473bf --- /dev/null +++ b/stix2/datastore/relational_db/database_backends/sqlite_backend.py @@ -0,0 +1,44 @@ +import os +from typing import Any + +from sqlalchemy import TIMESTAMP, LargeBinary, Text +from sqlalchemy import event + +from stix2.base import ( + _DomainObject, _MetaObject, _Observable, _RelationshipObject, +) +from stix2.datastore.relational_db.utils import schema_for + +from .database_backend_base import DatabaseBackend + + +class SQLiteBackend(DatabaseBackend): + default_database_connection_url = f"sqlite:///stix-data-sink.db" + + def __init__(self, database_connection_url=default_database_connection_url, force_recreate=False, **kwargs: Any): + super().__init__(database_connection_url, force_recreate=force_recreate, **kwargs) + + event.listen(self.database_connection, 'connect', self._fk_pragma_on_connect) + + # ========================================================================= + # sql type methods (overrides) + + @staticmethod + def determine_sql_type_for_binary_property(): # noqa: F811 + return SQLiteBackend.determine_sql_type_for_string_property() + + @staticmethod + def determine_sql_type_for_hex_property(): # noqa: F811 + # return LargeBinary + return SQLiteBackend.determine_sql_type_for_string_property() + + @staticmethod + def determine_sql_type_for_timestamp_property(): # noqa: F811 + return TIMESTAMP(timezone=True) + + # ========================================================================= + # Other methods + + @staticmethod + def array_allowed(): + return False diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py index 54bf0aae..acd0727a 100644 --- a/stix2/datastore/relational_db/relational_db_testing.py +++ b/stix2/datastore/relational_db/relational_db_testing.py @@ -1,6 +1,7 @@ import datetime as dt from database_backends.postgres_backend import PostgresBackend +from database_backends.sqlite_backend import SQLiteBackend import pytz import stix2 @@ -287,7 +288,8 @@ def test_dictionary(): def main(): store = RelationalDBStore( - PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True), + #PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True), + SQLiteBackend("sqlite:///stix-data-sink.db", force_recreate=True), True, None, True, From 28603d73f901e339f5f63cc74eb048dc37d6323a Mon Sep 17 00:00:00 2001 From: Ming Fang Date: Wed, 4 Dec 2024 17:37:02 -0500 Subject: [PATCH 2/4] Enable foreign key constraint for SQLite directly from the SQLiteBackend subclass. --- .../database_backends/database_backend_base.py | 3 --- .../relational_db/database_backends/sqlite_backend.py | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py index 4cf46f1b..e5082451 100644 --- a/stix2/datastore/relational_db/database_backends/database_backend_base.py +++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py @@ -18,9 +18,6 @@ def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any) self.database_connection = create_engine(database_connection_url) - def _fk_pragma_on_connect(self): - self.database_connnection.execute('pragma foreign_keys=ON') - def _create_database(self): if self.database_exists: drop_database(self.database_connection_url) diff --git a/stix2/datastore/relational_db/database_backends/sqlite_backend.py b/stix2/datastore/relational_db/database_backends/sqlite_backend.py index ae5473bf..f8094a15 100644 --- a/stix2/datastore/relational_db/database_backends/sqlite_backend.py +++ b/stix2/datastore/relational_db/database_backends/sqlite_backend.py @@ -2,6 +2,7 @@ from typing import Any from sqlalchemy import TIMESTAMP, LargeBinary, Text +from sqlalchemy.engine import Engine from sqlalchemy import event from stix2.base import ( @@ -18,7 +19,11 @@ class SQLiteBackend(DatabaseBackend): def __init__(self, database_connection_url=default_database_connection_url, force_recreate=False, **kwargs: Any): super().__init__(database_connection_url, force_recreate=force_recreate, **kwargs) - event.listen(self.database_connection, 'connect', self._fk_pragma_on_connect) + set_sqlite_pragma(self) + + @event.listens_for(Engine, "connect") + def set_sqlite_pragma(self): + self.database_connection.execute("PRAGMA foreign_keys=ON") # ========================================================================= # sql type methods (overrides) From 2ac14aec75b61791a9a4244fb69b69ac7543084c Mon Sep 17 00:00:00 2001 From: Ming Fang Date: Thu, 5 Dec 2024 12:04:56 -0500 Subject: [PATCH 3/4] Use SQLAlchemy's event.listens_for() to ensure that foreign key constraint is enforced on new connections before use. --- .../database_backends/sqlite_backend.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/stix2/datastore/relational_db/database_backends/sqlite_backend.py b/stix2/datastore/relational_db/database_backends/sqlite_backend.py index f8094a15..545f928b 100644 --- a/stix2/datastore/relational_db/database_backends/sqlite_backend.py +++ b/stix2/datastore/relational_db/database_backends/sqlite_backend.py @@ -2,7 +2,6 @@ from typing import Any from sqlalchemy import TIMESTAMP, LargeBinary, Text -from sqlalchemy.engine import Engine from sqlalchemy import event from stix2.base import ( @@ -19,11 +18,14 @@ class SQLiteBackend(DatabaseBackend): def __init__(self, database_connection_url=default_database_connection_url, force_recreate=False, **kwargs: Any): super().__init__(database_connection_url, force_recreate=force_recreate, **kwargs) - set_sqlite_pragma(self) - - @event.listens_for(Engine, "connect") - def set_sqlite_pragma(self): - self.database_connection.execute("PRAGMA foreign_keys=ON") + @event.listens_for(self.database_connection, "connect") + def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + result = cursor.execute("PRAGMA foreign_keys") + for row in result: + print('PRAGMA foreign_keys:', row) + cursor.close() # ========================================================================= # sql type methods (overrides) From 9183447a83446765a6d3e640408b92b59174109e Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Mon, 9 Dec 2024 15:27:38 -0500 Subject: [PATCH 4/4] constraitns for reg-ex db dependent --- .../database_backend_base.py | 4 ++ .../database_backends/postgres_backend.py | 4 ++ .../datastore/relational_db/table_creation.py | 64 ++++++------------- 3 files changed, 26 insertions(+), 46 deletions(-) diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py index e5082451..7e5ad4af 100644 --- a/stix2/datastore/relational_db/database_backends/database_backend_base.py +++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py @@ -120,6 +120,10 @@ def determine_stix_type(stix_object): def array_allowed(): return False + @staticmethod + def create_regex_constraint_expression(column, pattern): + pass + def process_value_for_insert(self, stix_type, value): sql_type = stix_type.determine_sql_type(self) if sql_type == self.determine_sql_type_for_string_property(): diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py index 931636e2..c90f5256 100644 --- a/stix2/datastore/relational_db/database_backends/postgres_backend.py +++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py @@ -73,3 +73,7 @@ def determine_sql_type_for_timestamp_property(): # noqa: F811 @staticmethod def array_allowed(): return True + + @staticmethod + def create_regex_constraint_expression(column_name, pattern): + return f"{column_name} ~ {pattern}" diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py index f57ff856..ea71b947 100644 --- a/stix2/datastore/relational_db/table_creation.py +++ b/stix2/datastore/relational_db/table_creation.py @@ -162,6 +162,7 @@ def create_kill_chain_phases_table(name, metadata, db_backend, schema_name, tabl def create_granular_markings_table(metadata, db_backend, sco_or_sdo): schema_name = db_backend.schema_for_core() tables = list() + reg_ex = f"'^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'" # noqa: E131 columns = [ Column( "id", @@ -173,10 +174,7 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo): Column( "marking_ref", db_backend.determine_sql_type_for_reference_property(), - CheckConstraint( - "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", - # noqa: E131 - ), + CheckConstraint(db_backend.create_regex_constraint_expression("marking_ref", reg_ex)), ), ] if db_backend.array_allowed(): @@ -230,15 +228,13 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo): def create_external_references_tables(metadata, db_backend): + reg_ex = "'^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'" # noqa: E131 columns = [ Column( "id", db_backend.determine_sql_type_for_key_as_id(), ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"), - CheckConstraint( - "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", # noqa: E131 - ), - ), + CheckConstraint(db_backend.create_regex_constraint_expression("id", reg_ex))), Column("source_name", db_backend.determine_sql_type_for_string_property()), Column("description", db_backend.determine_sql_type_for_string_property()), Column("url", db_backend.determine_sql_type_for_string_property()), @@ -255,26 +251,23 @@ def create_external_references_tables(metadata, db_backend): def create_core_table(metadata, db_backend, stix_type_name): tables = list() table_name = "core_" + stix_type_name + reg_ex = "'^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'" # noqa: E131 columns = [ Column( "id", db_backend.determine_sql_type_for_key_as_id(), - CheckConstraint( - "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", # noqa: E131 - ), + CheckConstraint(db_backend.create_regex_constraint_expression("id", reg_ex)), primary_key=True, ), Column("spec_version", db_backend.determine_sql_type_for_string_property(), default="2.1"), ] if stix_type_name == "sdo": + reg_ex = "'^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'" # noqa: E131 sdo_columns = [ Column( "created_by_ref", db_backend.determine_sql_type_for_reference_property(), - CheckConstraint( - "created_by_ref ~ '^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", # noqa: E131 - ), - ), + CheckConstraint(db_backend.create_regex_constraint_expression("created_by_ref", reg_ex))), Column("created", db_backend.determine_sql_type_for_timestamp_property()), Column("modified", db_backend.determine_sql_type_for_timestamp_property()), Column("revoked", db_backend.determine_sql_type_for_boolean_property()), @@ -405,9 +398,8 @@ def generate_table_information(self, name, db_backend, **kwargs): # noqa: F811 return Column( name, self.determine_sql_type(db_backend), - CheckConstraint( - # this regular expression might accept or reject some legal base64 strings - f"{name} ~ " + "'^[-A-Za-z0-9+/]*={0,3}$'", + # this regular expression might accept or reject some legal base64 strings + CheckConstraint(db_backend.create_regex_constraint_expression(name, "'^[-A-Za-z0-9+/]*={0,3}$'") ), nullable=not self.required, ) @@ -534,9 +526,7 @@ def generate_table_information(self, name, db_backend, **kwargs): # noqa: F811 return Column( name, self.determine_sql_type(db_backend), - CheckConstraint( - f"{name} ~ '^{enum_re}$'", - ), + CheckConstraint(db_backend.create_regex_constraint_expression(name, f"'^{enum_re}$'")), nullable=not self.required, ) @@ -609,18 +599,7 @@ def generate_table_information(self, name, db_backend, **kwargs): # noqa: F811 schema_name = kwargs.get('schema_name') table_name = kwargs.get("table_name") core_table = kwargs.get("core_table") - # if schema_name == "common": - # return Column( - # name, - # Text, - # CheckConstraint( - # f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", - # # noqa: E131 - # ), - # primary_key=True, - # nullable=not (self.required), - # ) - # else: + id_req_exp = f"'^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'" # noqa: E131 if schema_name: foreign_key_column = f"common.core_{core_table}.id" else: @@ -630,8 +609,7 @@ def generate_table_information(self, name, db_backend, **kwargs): # noqa: F811 db_backend.determine_sql_type_for_key_as_id(), ForeignKey(foreign_key_column, ondelete="CASCADE"), CheckConstraint( - f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", - # noqa: E131 + db_backend.create_regex_constraint_expression(name, id_req_exp) ), primary_key=True, nullable=not (self.required), @@ -743,17 +721,14 @@ def ref_column(name, specifics, db_backend, auth_type=0): if specifics: types = "|".join(specifics) if auth_type == 0: + reg_ex = f"'^({types})" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'" # noqa: F811 constraint = \ - CheckConstraint( - f"{name} ~ '^({types})" + - "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'", - ) + CheckConstraint(db_backend.create_regex_constraint_expression(name, reg_ex)) else: + reg_ex = "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')" constraint = \ - CheckConstraint( - f"(NOT({name} ~ '^({types})')) AND ({name} ~ " + - "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')", - ) + CheckConstraint(db_backend.create_regex_constraint_expression(f"NOT({name}", f"'^({types})'") + " AND " + + db_backend.create_regex_constraint_expression(name, reg_ex)) return Column(name, db_backend.determine_sql_type_for_reference_property(), constraint) else: return Column( @@ -789,9 +764,6 @@ def generate_table_information(self, name, db_backend, **kwargs): # noqa: F811 return Column( name, self.determine_sql_type(db_backend), - # CheckConstraint( - # f"{name} ~ '^{enum_re}$'" - # ), nullable=not (self.required), )