From fff68bd217c066d2d79c50a029c64aac63391f28 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 00:48:49 +0000 Subject: [PATCH 001/145] ORM for _allocations set on Batch --- orm.py | 40 ++++++++++++++++++++++++++---------- test_orm.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 14 deletions(-) diff --git a/orm.py b/orm.py index 6a3e728e..3e6b4a70 100644 --- a/orm.py +++ b/orm.py @@ -1,5 +1,5 @@ -from sqlalchemy import Table, MetaData, Column, Integer, String, Date -from sqlalchemy.orm import mapper +from sqlalchemy import Table, MetaData, Column, Integer, String, Date, ForeignKey +from sqlalchemy.orm import mapper, relationship import model @@ -9,21 +9,39 @@ order_lines = Table( "order_lines", metadata, - Column("orderid", String(255), primary_key=True), - Column("sku", String(255), primary_key=True), - Column("qty", Integer), + Column("id", Integer, primary_key=True, autoincrement=True), + Column("sku", String(255)), + Column("qty", Integer, nullable=False), + Column("orderid", String(255)), ) batches = Table( "batches", metadata, - Column("reference", String(255), primary_key=True), - Column("sku", String(255), primary_key=True), - Column("_purchased_qty", Integer), - Column("eta", Date), + Column("id", Integer, primary_key=True, autoincrement=True), + Column("reference", String(255)), + Column("sku", String(255)), + Column("_purchased_quantity", Integer, nullable=False), + Column("eta", Date, nullable=True), +) + +allocations = Table( + "allocations", + metadata, + Column("id", Integer, primary_key=True, autoincrement=True), + Column("orderline_id", ForeignKey("order_lines.id")), + Column("batch_id", ForeignKey("batches.id")), ) def start_mappers(): - mapper(model.OrderLine, order_lines) - mapper(model.Batch, batches) + lines_mapper = mapper(model.OrderLine, order_lines) + mapper( + model.Batch, + batches, + properties={ + "_allocations": relationship( + lines_mapper, secondary=allocations, collection_class=set, + ) + }, + ) diff --git a/test_orm.py b/test_orm.py index 9dc98719..6626a13c 100644 --- a/test_orm.py +++ b/test_orm.py @@ -26,10 +26,14 @@ def test_orderline_mapper_can_save_lines(session): assert rows == [("order1", "DECORATIVE-WIDGET", 12)] -def test_batches(session): - session.execute('INSERT INTO "batches" VALUES ("batch1", "sku1", 100, null)') +def test_retrieving_batches(session): session.execute( - 'INSERT INTO "batches" VALUES ("batch2", "sku2", 200, "2011-04-11")' + "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" + ' VALUES ("batch1", "sku1", 100, null)' + ) + session.execute( + "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" + ' VALUES ("batch2", "sku2", 200, "2011-04-11")' ) expected = [ model.Batch("batch1", "sku1", 100, eta=None), @@ -37,3 +41,51 @@ def test_batches(session): ] assert session.query(model.Batch).all() == expected + + +def test_saving_batches(session): + batch = model.Batch("batch1", "sku1", 100, eta=None) + session.add(batch) + session.commit() + rows = list( + session.execute( + 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' + ) + ) + assert rows == [("batch1", "sku1", 100, None)] + + +def test_saving_allocations(session): + batch = model.Batch("batch1", "sku1", 100, eta=None) + line = model.OrderLine("order1", "sku1", 10) + batch.allocate(line) + session.add(batch) + session.commit() + rows = list(session.execute('SELECT orderline_id, batch_id FROM "allocations"')) + assert rows == [(batch.id, line.id)] + + +def test_retrieving_allocations(session): + session.execute( + 'INSERT INTO order_lines (orderid, sku, qty) VALUES ("order1", "sku1", 12)' + ) + [[olid]] = session.execute( + "SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku", + dict(orderid="order1", sku="sku1"), + ) + session.execute( + "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" + ' VALUES ("batch1", "sku1", 100, null)' + ) + [[bid]] = session.execute( + "SELECT id FROM batches WHERE reference=:ref AND sku=:sku", + dict(ref="batch1", sku="sku1"), + ) + session.execute( + "INSERT INTO allocations (orderline_id, batch_id) VALUES (:olid, :bid)", + dict(olid=olid, bid=bid), + ) + + batch = session.query(model.Batch).one() + + assert batch._allocations == {model.OrderLine("order1", "sku1", 12)} From 8dae12639c6247185877d199e3055d0cacb126ea Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 01:10:30 +0000 Subject: [PATCH 002/145] repository tests --- test_orm.py | 8 +++--- test_repository.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 test_repository.py diff --git a/test_orm.py b/test_orm.py index 6626a13c..07a63491 100644 --- a/test_orm.py +++ b/test_orm.py @@ -47,12 +47,10 @@ def test_saving_batches(session): batch = model.Batch("batch1", "sku1", 100, eta=None) session.add(batch) session.commit() - rows = list( - session.execute( - 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' - ) + rows = session.execute( + 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' ) - assert rows == [("batch1", "sku1", 100, None)] + assert list(rows) == [("batch1", "sku1", 100, None)] def test_saving_allocations(session): diff --git a/test_repository.py b/test_repository.py new file mode 100644 index 00000000..7c668a39 --- /dev/null +++ b/test_repository.py @@ -0,0 +1,67 @@ +# pylint: disable=protected-access +import model +import repository + + +def test_repository_can_save_a_batch(session): + batch = model.Batch("batch1", "RUSTY-SOAPDISH", 100, eta=None) + + repo = repository.SqlAlchemyRepository(session) + repo.add(batch) + session.commit() + + rows = session.execute( + 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' + ) + assert list(rows) == [("batch1", "RUSTY-SOAPDISH", 100, None)] + + +def insert_order_line(session): + session.execute( + "INSERT INTO order_lines (orderid, sku, qty)" + ' VALUES ("order1", "GENERIC-SOFA", 12)' + ) + [[orderline_id]] = session.execute( + "SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku", + dict(orderid="order1", sku="GENERIC-SOFA"), + ) + return orderline_id + + +def insert_batch(session, batch_id): + session.execute( + "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" + ' VALUES (:batch_id, "GENERIC-SOFA", 100, null)', + dict(batch_id=batch_id), + ) + [[batch_id]] = session.execute( + 'SELECT id FROM batches WHERE reference=:batch_id AND sku="GENERIC-SOFA"', + dict(batch_id=batch_id), + ) + return batch_id + + +def insert_allocation(session, orderline_id, batch_id): + session.execute( + "INSERT INTO allocations (orderline_id, batch_id)" + " VALUES (:orderline_id, :batch_id)", + dict(orderline_id=orderline_id, batch_id=batch_id), + ) + + +def test_repository_can_retrieve_a_batch_with_allocations(session): + orderline_id = insert_order_line(session) + batch1_id = insert_batch(session, "batch1") + insert_batch(session, "batch2") + insert_allocation(session, orderline_id, batch1_id) + + repo = repository.SqlAlchemyRepository(session) + retrieved = repo.get("batch1") + + expected = model.Batch("batch1", "GENERIC-SOFA", 100, eta=None) + assert retrieved == expected # Batch.__eq__ only compares reference + assert retrieved.sku == expected.sku + assert retrieved._purchased_quantity == expected._purchased_quantity + assert retrieved._allocations == { + model.OrderLine("order1", "GENERIC-SOFA", 12), + } From b635a154d8ddf58a3bbf40a1d5c96c65430bc49d Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 01:10:39 +0000 Subject: [PATCH 003/145] repository for batches [chapter_02_repository_ends] --- repository.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 repository.py diff --git a/repository.py b/repository.py new file mode 100644 index 00000000..73b85e31 --- /dev/null +++ b/repository.py @@ -0,0 +1,26 @@ +import abc +import model + + +class AbstractRepository(abc.ABC): + @abc.abstractmethod + def add(self, batch: model.Batch): + raise NotImplementedError + + @abc.abstractmethod + def get(self, reference) -> model.Batch: + raise NotImplementedError + + +class SqlAlchemyRepository(AbstractRepository): + def __init__(self, session): + self.session = session + + def add(self, batch): + self.session.add(batch) + + def get(self, reference): + return self.session.query(model.Batch).filter_by(reference=reference).one() + + def list(self): + return self.session.query(model.Batch).all() From fda077db691e09e69eb54e7fbf29feb732da37b5 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 04:10:52 +0000 Subject: [PATCH 004/145] first api tests [first_api_test] --- config.py | 15 +++++++++++++ conftest.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++ test_api.py | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 config.py create mode 100644 test_api.py diff --git a/config.py b/config.py new file mode 100644 index 00000000..f3b55cc9 --- /dev/null +++ b/config.py @@ -0,0 +1,15 @@ +import os + + +def get_postgres_uri(): + host = os.environ.get("DB_HOST", "localhost") + port = 54321 if host == "localhost" else 5432 + password = os.environ.get("DB_PASSWORD", "abc123") + user, db_name = "allocation", "allocation" + return f"postgresql://{user}:{password}@{host}:{port}/{db_name}" + + +def get_api_url(): + host = os.environ.get("API_HOST", "localhost") + port = 5005 if host == "localhost" else 80 + return f"http://{host}:{port}" diff --git a/conftest.py b/conftest.py index 9f7b74b0..f43dd114 100644 --- a/conftest.py +++ b/conftest.py @@ -1,8 +1,13 @@ +# pylint: disable=redefined-outer-name +import time +from pathlib import Path + import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers from orm import metadata, start_mappers +from config import get_postgres_uri @pytest.fixture @@ -17,3 +22,60 @@ def session(in_memory_db): start_mappers() yield sessionmaker(bind=in_memory_db)() clear_mappers() + + +@pytest.fixture(scope="session") +def postgres_db(): + engine = create_engine(get_postgres_uri()) + metadata.create_all(engine) + return engine + + +@pytest.fixture +def postgres_session(postgres_db): + start_mappers() + yield sessionmaker(bind=postgres_db)() + clear_mappers() + + +@pytest.fixture +def add_stock(postgres_session): + batches_added = set() + skus_added = set() + + def _add_stock(lines): + for ref, sku, qty, eta in lines: + postgres_session.execute( + "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" + " VALUES (:ref, :sku, :qty, :eta)", + dict(ref=ref, sku=sku, qty=qty, eta=eta), + ) + [[batch_id]] = postgres_session.execute( + "SELECT id FROM batches WHERE reference=:ref AND sku=:sku", + dict(ref=ref, sku=sku), + ) + batches_added.add(batch_id) + skus_added.add(sku) + postgres_session.commit() + + yield _add_stock + + for batch_id in batches_added: + postgres_session.execute( + "DELETE FROM allocations WHERE batch_id=:batch_id", + dict(batch_id=batch_id), + ) + postgres_session.execute( + "DELETE FROM batches WHERE id=:batch_id", dict(batch_id=batch_id), + ) + for sku in skus_added: + postgres_session.execute( + "DELETE FROM order_lines WHERE sku=:sku", dict(sku=sku), + ) + postgres_session.commit() + + +@pytest.fixture +def restart_api(): + (Path(__file__).parent / "flask_app.py").touch() + time.sleep(0.3) diff --git a/test_api.py b/test_api.py new file mode 100644 index 00000000..04baaf43 --- /dev/null +++ b/test_api.py @@ -0,0 +1,43 @@ +import uuid +import pytest +import requests + +import config + + +def random_suffix(): + return uuid.uuid4().hex[:6] + + +def random_sku(name=""): + return f"sku-{name}-{random_suffix()}" + + +def random_batchref(name=""): + return f"batch-{name}-{random_suffix()}" + + +def random_orderid(name=""): + return f"order-{name}-{random_suffix()}" + + +@pytest.mark.usefixtures("restart_api") +def test_api_returns_allocation(add_stock): + sku, othersku = random_sku(), random_sku("other") + earlybatch = random_batchref(1) + laterbatch = random_batchref(2) + otherbatch = random_batchref(3) + add_stock( + [ + (laterbatch, sku, 100, "2011-01-02"), + (earlybatch, sku, 100, "2011-01-01"), + (otherbatch, othersku, 100, None), + ] + ) + data = {"orderid": random_orderid(), "sku": sku, "qty": 3} + url = config.get_api_url() + + r = requests.post(f"{url}/allocate", json=data) + + assert r.status_code == 201 + assert r.json()["batchref"] == earlybatch From 15902a6f69b914015c5d5a4b90cc4d04c5e046d7 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 11:30:31 +0000 Subject: [PATCH 005/145] all the dockerfile gubbins --- Dockerfile | 10 ++++++++++ Makefile | 21 ++++++++++++++++++--- docker-compose.yml | 26 ++++++++++++++++++++++++++ requirements.txt | 5 +++++ 4 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..901a93fc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.9-slim-buster + +COPY requirements.txt /tmp +RUN pip install -r /tmp/requirements.txt + +RUN mkdir -p /code +COPY *.py /code/ +WORKDIR /code +ENV FLASK_APP=flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 +CMD flask run --host=0.0.0.0 --port=80 diff --git a/Makefile b/Makefile index 77cbd229..1e248187 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,23 @@ +# these will speed up builds, for docker-compose >= 1.25 +export COMPOSE_DOCKER_CLI_BUILD=1 +export DOCKER_BUILDKIT=1 + +all: down build up test + +build: + docker-compose build + +up: + docker-compose up -d app + +down: + docker-compose down + +logs: + docker-compose logs app | tail -100 + test: pytest --tb=short -watch-tests: - ls *.py | entr pytest --tb=short - black: black -l 86 $$(find * -name '*.py') diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..313cf946 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3" +services: + + app: + build: + context: . + dockerfile: Dockerfile + depends_on: + - postgres + environment: + - DB_HOST=postgres + - DB_PASSWORD=abc123 + volumes: + - ./:/code + ports: + - "5005:80" + + + postgres: + image: postgres:9.6 + environment: + - POSTGRES_USER=allocation + - POSTGRES_PASSWORD=abc123 + ports: + - "54321:5432" + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..01a974f5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pytest +sqlalchemy +flask +psycopg2-binary +requests From 8fc086ef836fd5fcf726d37e285e0e9d4ace49f4 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 04:21:51 +0000 Subject: [PATCH 006/145] first cut of flask app [first_cut_flask_app] --- flask_app.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 flask_app.py diff --git a/flask_app.py b/flask_app.py new file mode 100644 index 00000000..21be2902 --- /dev/null +++ b/flask_app.py @@ -0,0 +1,26 @@ +from flask import Flask, request +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +import config +import model +import orm +import repository + + +orm.start_mappers() +get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) +app = Flask(__name__) + + +@app.route("/allocate", methods=["POST"]) +def allocate_endpoint(): + session = get_session() + batches = repository.SqlAlchemyRepository(session).list() + line = model.OrderLine( + request.json["orderid"], request.json["sku"], request.json["qty"], + ) + + batchref = model.allocate(line, batches) + + return {"batchref": batchref}, 201 From 158e760fed9a7d530d34b3acf4e4159477eac73d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 11:58:59 +0000 Subject: [PATCH 007/145] test persistence by double-allocating. [second_api_test] --- test_api.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_api.py b/test_api.py index 04baaf43..9fd51a34 100644 --- a/test_api.py +++ b/test_api.py @@ -41,3 +41,26 @@ def test_api_returns_allocation(add_stock): assert r.status_code == 201 assert r.json()["batchref"] == earlybatch + + +@pytest.mark.usefixtures("restart_api") +def test_allocations_are_persisted(add_stock): + sku = random_sku() + batch1, batch2 = random_batchref(1), random_batchref(2) + order1, order2 = random_orderid(1), random_orderid(2) + add_stock( + [(batch1, sku, 10, "2011-01-01"), (batch2, sku, 10, "2011-01-02"),] + ) + line1 = {"orderid": order1, "sku": sku, "qty": 10} + line2 = {"orderid": order2, "sku": sku, "qty": 10} + url = config.get_api_url() + + # first order uses up all stock in batch 1 + r = requests.post(f"{url}/allocate", json=line1) + assert r.status_code == 201 + assert r.json()["batchref"] == batch1 + + # second order should go to batch 2 + r = requests.post(f"{url}/allocate", json=line2) + assert r.status_code == 201 + assert r.json()["batchref"] == batch2 From d0ee7ada3dbb034dbe5fce7a06e209486e88cccc Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 11:59:20 +0000 Subject: [PATCH 008/145] need to commit [flask_commit] --- flask_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flask_app.py b/flask_app.py index 21be2902..5ae4320e 100644 --- a/flask_app.py +++ b/flask_app.py @@ -23,4 +23,5 @@ def allocate_endpoint(): batchref = model.allocate(line, batches) + session.commit() return {"batchref": batchref}, 201 From f20c4792525ee9b138bd6586eb88359d1d70a8db Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 12:10:34 +0000 Subject: [PATCH 009/145] test some 400 error cases [test_error_cases] --- test_api.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_api.py b/test_api.py index 9fd51a34..2061119d 100644 --- a/test_api.py +++ b/test_api.py @@ -64,3 +64,26 @@ def test_allocations_are_persisted(add_stock): r = requests.post(f"{url}/allocate", json=line2) assert r.status_code == 201 assert r.json()["batchref"] == batch2 + + +@pytest.mark.usefixtures("restart_api") +def test_400_message_for_out_of_stock(add_stock): + sku, smalL_batch, large_order = random_sku(), random_batchref(), random_orderid() + add_stock( + [(smalL_batch, sku, 10, "2011-01-01"),] + ) + data = {"orderid": large_order, "sku": sku, "qty": 20} + url = config.get_api_url() + r = requests.post(f"{url}/allocate", json=data) + assert r.status_code == 400 + assert r.json()["message"] == f"Out of stock for sku {sku}" + + +@pytest.mark.usefixtures("restart_api") +def test_400_message_for_invalid_sku(): + unknown_sku, orderid = random_sku(), random_orderid() + data = {"orderid": orderid, "sku": unknown_sku, "qty": 20} + url = config.get_api_url() + r = requests.post(f"{url}/allocate", json=data) + assert r.status_code == 400 + assert r.json()["message"] == f"Invalid sku {unknown_sku}" From 80dece9d1736a3abf2ae605af25606c85b39bccb Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 12:17:15 +0000 Subject: [PATCH 010/145] flask now does error handling [flask_error_handling] --- flask_app.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flask_app.py b/flask_app.py index 5ae4320e..3ffa1f4f 100644 --- a/flask_app.py +++ b/flask_app.py @@ -13,6 +13,10 @@ app = Flask(__name__) +def is_valid_sku(sku, batches): + return sku in {b.sku for b in batches} + + @app.route("/allocate", methods=["POST"]) def allocate_endpoint(): session = get_session() @@ -21,7 +25,13 @@ def allocate_endpoint(): request.json["orderid"], request.json["sku"], request.json["qty"], ) - batchref = model.allocate(line, batches) + if not is_valid_sku(line.sku, batches): + return {"message": f"Invalid sku {line.sku}"}, 400 + + try: + batchref = model.allocate(line, batches) + except model.OutOfStock as e: + return {"message": str(e)}, 400 session.commit() return {"batchref": batchref}, 201 From a1903fad959426a0d7bee8c186b0bfb73a2755d2 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 13:44:27 +0000 Subject: [PATCH 011/145] first tests for the services layer [first_services_tests] --- test_services.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 test_services.py diff --git a/test_services.py b/test_services.py new file mode 100644 index 00000000..92dcd517 --- /dev/null +++ b/test_services.py @@ -0,0 +1,22 @@ +import pytest + +import model +import services + + +def test_returns_allocation(): + line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) + batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) + repo = FakeRepository([batch]) + + result = services.allocate(line, repo, FakeSession()) + assert result == "b1" + + +def test_error_for_invalid_sku(): + line = model.OrderLine("o1", "NONEXISTENTSKU", 10) + batch = model.Batch("b1", "AREALSKU", 100, eta=None) + repo = FakeRepository([batch]) + + with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): + services.allocate(line, repo, FakeSession()) From 9fe9a3de7ea0afabf652bacf035e86b39ac02d6c Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 00:54:06 +0000 Subject: [PATCH 012/145] FakeRepository [fake_repo] --- test_services.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test_services.py b/test_services.py index 92dcd517..fb15f428 100644 --- a/test_services.py +++ b/test_services.py @@ -1,9 +1,23 @@ import pytest - import model +import repository import services +class FakeRepository(repository.AbstractRepository): + def __init__(self, batches): + self._batches = set(batches) + + def add(self, batch): + self._batches.add(batch) + + def get(self, reference): + return next(b for b in self._batches if b.reference == reference) + + def list(self): + return list(self._batches) + + def test_returns_allocation(): line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) From 15b9bf79d134384a1cac645eba4b055f9bf4ac1d Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 00:54:48 +0000 Subject: [PATCH 013/145] FakeSession [fake_session] --- test_services.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_services.py b/test_services.py index fb15f428..380f46c9 100644 --- a/test_services.py +++ b/test_services.py @@ -18,6 +18,13 @@ def list(self): return list(self._batches) +class FakeSession: + committed = False + + def commit(self): + self.committed = True + + def test_returns_allocation(): line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) From eded7ebf6cc5f0fb05dc547af6daee6549e171ca Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:52:08 +0100 Subject: [PATCH 014/145] test commmits [second_services_test] --- test_services.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_services.py b/test_services.py index 380f46c9..abb13a1e 100644 --- a/test_services.py +++ b/test_services.py @@ -41,3 +41,13 @@ def test_error_for_invalid_sku(): with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): services.allocate(line, repo, FakeSession()) + + +def test_commits(): + line = model.OrderLine("o1", "OMINOUS-MIRROR", 10) + batch = model.Batch("b1", "OMINOUS-MIRROR", 100, eta=None) + repo = FakeRepository([batch]) + session = FakeSession() + + services.allocate(line, repo, session) + assert session.committed is True From d1e2e6e59fb8fe4020bec8425bdbd30b9648ff08 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 13:44:44 +0000 Subject: [PATCH 015/145] services layer with valid-sku check [service_function] --- services.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 services.py diff --git a/services.py b/services.py new file mode 100644 index 00000000..cb007393 --- /dev/null +++ b/services.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import model +from model import OrderLine +from repository import AbstractRepository + + +class InvalidSku(Exception): + pass + + +def is_valid_sku(sku, batches): + return sku in {b.sku for b in batches} + + +def allocate(line: OrderLine, repo: AbstractRepository, session) -> str: + batches = repo.list() + if not is_valid_sku(line.sku, batches): + raise InvalidSku(f"Invalid sku {line.sku}") + batchref = model.allocate(line, batches) + session.commit() + return batchref From cf2f52ba9696b7fa2336b46a55c6f11f180fa203 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 14:33:19 +0000 Subject: [PATCH 016/145] modify flask app to use service layer [flask_app_using_service_layer] --- flask_app.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/flask_app.py b/flask_app.py index 3ffa1f4f..3412cbe9 100644 --- a/flask_app.py +++ b/flask_app.py @@ -6,6 +6,7 @@ import model import orm import repository +import services orm.start_mappers() @@ -13,25 +14,17 @@ app = Flask(__name__) -def is_valid_sku(sku, batches): - return sku in {b.sku for b in batches} - - @app.route("/allocate", methods=["POST"]) def allocate_endpoint(): session = get_session() - batches = repository.SqlAlchemyRepository(session).list() + repo = repository.SqlAlchemyRepository(session) line = model.OrderLine( request.json["orderid"], request.json["sku"], request.json["qty"], ) - if not is_valid_sku(line.sku, batches): - return {"message": f"Invalid sku {line.sku}"}, 400 - try: - batchref = model.allocate(line, batches) - except model.OutOfStock as e: + batchref = services.allocate(line, repo, session) + except (model.OutOfStock, services.InvalidSku) as e: return {"message": str(e)}, 400 - session.commit() return {"batchref": batchref}, 201 From b537364dcb7b22e0348fda4b66219e3bfe4d259b Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 14:48:36 +0000 Subject: [PATCH 017/145] strip out unecessary tests from e2e layer [fewer_e2e_tests] --- test_api.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/test_api.py b/test_api.py index 2061119d..5386ab88 100644 --- a/test_api.py +++ b/test_api.py @@ -22,7 +22,7 @@ def random_orderid(name=""): @pytest.mark.usefixtures("restart_api") -def test_api_returns_allocation(add_stock): +def test_happy_path_returns_201_and_allocated_batch(add_stock): sku, othersku = random_sku(), random_sku("other") earlybatch = random_batchref(1) laterbatch = random_batchref(2) @@ -44,43 +44,7 @@ def test_api_returns_allocation(add_stock): @pytest.mark.usefixtures("restart_api") -def test_allocations_are_persisted(add_stock): - sku = random_sku() - batch1, batch2 = random_batchref(1), random_batchref(2) - order1, order2 = random_orderid(1), random_orderid(2) - add_stock( - [(batch1, sku, 10, "2011-01-01"), (batch2, sku, 10, "2011-01-02"),] - ) - line1 = {"orderid": order1, "sku": sku, "qty": 10} - line2 = {"orderid": order2, "sku": sku, "qty": 10} - url = config.get_api_url() - - # first order uses up all stock in batch 1 - r = requests.post(f"{url}/allocate", json=line1) - assert r.status_code == 201 - assert r.json()["batchref"] == batch1 - - # second order should go to batch 2 - r = requests.post(f"{url}/allocate", json=line2) - assert r.status_code == 201 - assert r.json()["batchref"] == batch2 - - -@pytest.mark.usefixtures("restart_api") -def test_400_message_for_out_of_stock(add_stock): - sku, smalL_batch, large_order = random_sku(), random_batchref(), random_orderid() - add_stock( - [(smalL_batch, sku, 10, "2011-01-01"),] - ) - data = {"orderid": large_order, "sku": sku, "qty": 20} - url = config.get_api_url() - r = requests.post(f"{url}/allocate", json=data) - assert r.status_code == 400 - assert r.json()["message"] == f"Out of stock for sku {sku}" - - -@pytest.mark.usefixtures("restart_api") -def test_400_message_for_invalid_sku(): +def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() data = {"orderid": orderid, "sku": unknown_sku, "qty": 20} url = config.get_api_url() From 952a3d2f53ec56ca320b8415097350799f38de10 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Mar 2019 21:21:17 +0000 Subject: [PATCH 018/145] fix conftest waits and travis config [chapter_04_service_layer_ends] --- .travis.yml | 4 ++-- conftest.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a74f94b..f601680f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,10 @@ language: python python: 3.8 install: -- pip3 install sqlalchemy +- pip3 install -r requirements.txt script: -- make test +- make all branches: except: diff --git a/conftest.py b/conftest.py index f43dd114..da82d88a 100644 --- a/conftest.py +++ b/conftest.py @@ -3,11 +3,14 @@ from pathlib import Path import pytest +import requests +from requests.exceptions import ConnectionError +from sqlalchemy.exc import OperationalError from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers from orm import metadata, start_mappers -from config import get_postgres_uri +import config @pytest.fixture @@ -24,9 +27,31 @@ def session(in_memory_db): clear_mappers() +def wait_for_postgres_to_come_up(engine): + deadline = time.time() + 10 + while time.time() < deadline: + try: + return engine.connect() + except OperationalError: + time.sleep(0.5) + pytest.fail("Postgres never came up") + + +def wait_for_webapp_to_come_up(): + deadline = time.time() + 10 + url = config.get_api_url() + while time.time() < deadline: + try: + return requests.get(url) + except ConnectionError: + time.sleep(0.5) + pytest.fail("API never came up") + + @pytest.fixture(scope="session") def postgres_db(): - engine = create_engine(get_postgres_uri()) + engine = create_engine(config.get_postgres_uri()) + wait_for_postgres_to_come_up(engine) metadata.create_all(engine) return engine @@ -78,4 +103,5 @@ def _add_stock(lines): @pytest.fixture def restart_api(): (Path(__file__).parent / "flask_app.py").touch() - time.sleep(0.3) + time.sleep(0.5) + wait_for_webapp_to_come_up() From bdf8fe94baffb69097e727135b0c6ce77ac87674 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 23 Dec 2019 16:37:06 +0000 Subject: [PATCH 019/145] move to a more nested folder structure --- orm.py => adapters/orm.py | 0 repository.py => adapters/repository.py | 0 model.py => domain/model.py | 0 flask_app.py => entrypoints/flask_app.py | 0 services.py => service_layer/services.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename orm.py => adapters/orm.py (100%) rename repository.py => adapters/repository.py (100%) rename model.py => domain/model.py (100%) rename flask_app.py => entrypoints/flask_app.py (100%) rename services.py => service_layer/services.py (100%) diff --git a/orm.py b/adapters/orm.py similarity index 100% rename from orm.py rename to adapters/orm.py diff --git a/repository.py b/adapters/repository.py similarity index 100% rename from repository.py rename to adapters/repository.py diff --git a/model.py b/domain/model.py similarity index 100% rename from model.py rename to domain/model.py diff --git a/flask_app.py b/entrypoints/flask_app.py similarity index 100% rename from flask_app.py rename to entrypoints/flask_app.py diff --git a/services.py b/service_layer/services.py similarity index 100% rename from services.py rename to service_layer/services.py From 1bb572af7f8fce0a5236fc005c4d5bc305464c94 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 31 Dec 2019 13:18:04 +0000 Subject: [PATCH 020/145] nest the tests too --- test_api.py => tests/e2e/test_api.py | 0 test_orm.py => tests/integration/test_orm.py | 0 test_repository.py => tests/integration/test_repository.py | 0 test_allocate.py => tests/unit/test_allocate.py | 0 test_batches.py => tests/unit/test_batches.py | 0 test_services.py => tests/unit/test_services.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename test_api.py => tests/e2e/test_api.py (100%) rename test_orm.py => tests/integration/test_orm.py (100%) rename test_repository.py => tests/integration/test_repository.py (100%) rename test_allocate.py => tests/unit/test_allocate.py (100%) rename test_batches.py => tests/unit/test_batches.py (100%) rename test_services.py => tests/unit/test_services.py (100%) diff --git a/test_api.py b/tests/e2e/test_api.py similarity index 100% rename from test_api.py rename to tests/e2e/test_api.py diff --git a/test_orm.py b/tests/integration/test_orm.py similarity index 100% rename from test_orm.py rename to tests/integration/test_orm.py diff --git a/test_repository.py b/tests/integration/test_repository.py similarity index 100% rename from test_repository.py rename to tests/integration/test_repository.py diff --git a/test_allocate.py b/tests/unit/test_allocate.py similarity index 100% rename from test_allocate.py rename to tests/unit/test_allocate.py diff --git a/test_batches.py b/tests/unit/test_batches.py similarity index 100% rename from test_batches.py rename to tests/unit/test_batches.py diff --git a/test_services.py b/tests/unit/test_services.py similarity index 100% rename from test_services.py rename to tests/unit/test_services.py From db8921818a5aaeb3495f0405fa089648ade4b306 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 31 Dec 2019 12:44:27 +0000 Subject: [PATCH 021/145] get all tests passing --- Dockerfile | 2 +- adapters/__init__.py | 0 adapters/orm.py | 2 +- adapters/repository.py | 2 +- domain/__init__.py | 0 entrypoints/__init__.py | 0 entrypoints/flask_app.py | 7 +++---- service_layer/__init__.py | 0 service_layer/services.py | 6 +++--- tests/__init__.py | 0 conftest.py => tests/conftest.py | 4 ++-- tests/integration/test_orm.py | 2 +- tests/integration/test_repository.py | 4 ++-- tests/unit/test_allocate.py | 2 +- tests/unit/test_batches.py | 2 +- tests/unit/test_services.py | 6 +++--- 16 files changed, 19 insertions(+), 20 deletions(-) create mode 100644 adapters/__init__.py create mode 100644 domain/__init__.py create mode 100644 entrypoints/__init__.py create mode 100644 service_layer/__init__.py create mode 100644 tests/__init__.py rename conftest.py => tests/conftest.py (95%) diff --git a/Dockerfile b/Dockerfile index 901a93fc..d4803afb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,5 +6,5 @@ RUN pip install -r /tmp/requirements.txt RUN mkdir -p /code COPY *.py /code/ WORKDIR /code -ENV FLASK_APP=flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 +ENV FLASK_APP=entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 CMD flask run --host=0.0.0.0 --port=80 diff --git a/adapters/__init__.py b/adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/adapters/orm.py b/adapters/orm.py index 3e6b4a70..8499c2dc 100644 --- a/adapters/orm.py +++ b/adapters/orm.py @@ -1,7 +1,7 @@ from sqlalchemy import Table, MetaData, Column, Integer, String, Date, ForeignKey from sqlalchemy.orm import mapper, relationship -import model +from domain import model metadata = MetaData() diff --git a/adapters/repository.py b/adapters/repository.py index 73b85e31..7e7434b9 100644 --- a/adapters/repository.py +++ b/adapters/repository.py @@ -1,5 +1,5 @@ import abc -import model +from domain import model class AbstractRepository(abc.ABC): diff --git a/domain/__init__.py b/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/entrypoints/__init__.py b/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/entrypoints/flask_app.py b/entrypoints/flask_app.py index 3412cbe9..62e40d44 100644 --- a/entrypoints/flask_app.py +++ b/entrypoints/flask_app.py @@ -3,10 +3,9 @@ from sqlalchemy.orm import sessionmaker import config -import model -import orm -import repository -import services +from domain import model +from adapters import orm, repository +from service_layer import services orm.start_mappers() diff --git a/service_layer/__init__.py b/service_layer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/service_layer/services.py b/service_layer/services.py index cb007393..7c7ff8f1 100644 --- a/service_layer/services.py +++ b/service_layer/services.py @@ -1,8 +1,8 @@ from __future__ import annotations -import model -from model import OrderLine -from repository import AbstractRepository +from domain import model +from domain.model import OrderLine +from adapters.repository import AbstractRepository class InvalidSku(Exception): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/conftest.py b/tests/conftest.py similarity index 95% rename from conftest.py rename to tests/conftest.py index da82d88a..11e66b35 100644 --- a/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers -from orm import metadata, start_mappers +from adapters.orm import metadata, start_mappers import config @@ -102,6 +102,6 @@ def _add_stock(lines): @pytest.fixture def restart_api(): - (Path(__file__).parent / "flask_app.py").touch() + (Path(__file__).parent / "../entrypoints/flask_app.py").touch() time.sleep(0.5) wait_for_webapp_to_come_up() diff --git a/tests/integration/test_orm.py b/tests/integration/test_orm.py index 07a63491..3a551977 100644 --- a/tests/integration/test_orm.py +++ b/tests/integration/test_orm.py @@ -1,4 +1,4 @@ -import model +from domain import model from datetime import date diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index 7c668a39..3fa66d9f 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,6 +1,6 @@ # pylint: disable=protected-access -import model -import repository +from domain import model +from adapters import repository def test_repository_can_save_a_batch(session): diff --git a/tests/unit/test_allocate.py b/tests/unit/test_allocate.py index 7e189307..4e434e28 100644 --- a/tests/unit/test_allocate.py +++ b/tests/unit/test_allocate.py @@ -1,6 +1,6 @@ from datetime import date, timedelta import pytest -from model import allocate, OrderLine, Batch, OutOfStock +from domain.model import allocate, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) diff --git a/tests/unit/test_batches.py b/tests/unit/test_batches.py index 62288330..7feda3d7 100644 --- a/tests/unit/test_batches.py +++ b/tests/unit/test_batches.py @@ -1,5 +1,5 @@ from datetime import date -from model import Batch, OrderLine +from domain.model import Batch, OrderLine def test_allocating_to_a_batch_reduces_the_available_quantity(): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index abb13a1e..64ef3158 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,7 +1,7 @@ import pytest -import model -import repository -import services +from domain import model +from adapters import repository +from service_layer import services class FakeRepository(repository.AbstractRepository): From c5822aa764b767efabc4975060b9533cef094fbf Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:11:47 +0100 Subject: [PATCH 022/145] rewrite service layer to take primitives [service_takes_primitives] --- service_layer/services.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/service_layer/services.py b/service_layer/services.py index 7c7ff8f1..464cdb7c 100644 --- a/service_layer/services.py +++ b/service_layer/services.py @@ -13,7 +13,11 @@ def is_valid_sku(sku, batches): return sku in {b.sku for b in batches} -def allocate(line: OrderLine, repo: AbstractRepository, session) -> str: +def allocate( + orderid: str, sku: str, qty: int, + repo: AbstractRepository, session +) -> str: + line = OrderLine(orderid, sku, qty) batches = repo.list() if not is_valid_sku(line.sku, batches): raise InvalidSku(f"Invalid sku {line.sku}") From f341bee916afc28fbc23d50be2d1979dadc9978d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:12:16 +0100 Subject: [PATCH 023/145] services tests partially converted to primitives [tests_call_with_primitives] --- tests/unit/test_services.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 64ef3158..9c114ed3 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -26,28 +26,25 @@ def commit(self): def test_returns_allocation(): - line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) - batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) + batch = model.Batch("batch1", "COMPLICATED-LAMP", 100, eta=None) repo = FakeRepository([batch]) - result = services.allocate(line, repo, FakeSession()) - assert result == "b1" + result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) + assert result == "batch1" def test_error_for_invalid_sku(): - line = model.OrderLine("o1", "NONEXISTENTSKU", 10) batch = model.Batch("b1", "AREALSKU", 100, eta=None) repo = FakeRepository([batch]) with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate(line, repo, FakeSession()) + services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) def test_commits(): - line = model.OrderLine("o1", "OMINOUS-MIRROR", 10) batch = model.Batch("b1", "OMINOUS-MIRROR", 100, eta=None) repo = FakeRepository([batch]) session = FakeSession() - services.allocate(line, repo, session) + services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) assert session.committed is True From 1e1f2381e348b29e04a9812114feb8ca6f92bbc9 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:59:52 +0100 Subject: [PATCH 024/145] fixture function for batches [services_factory_function] --- tests/unit/test_services.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 9c114ed3..0d404f4a 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -5,6 +5,10 @@ class FakeRepository(repository.AbstractRepository): + @staticmethod + def for_batch(ref, sku, qty, eta=None): + return FakeRepository([model.Batch(ref, sku, qty, eta),]) + def __init__(self, batches): self._batches = set(batches) @@ -26,25 +30,19 @@ def commit(self): def test_returns_allocation(): - batch = model.Batch("batch1", "COMPLICATED-LAMP", 100, eta=None) - repo = FakeRepository([batch]) - + repo = FakeRepository.for_batch("batch1", "COMPLICATED-LAMP", 100, eta=None) result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) assert result == "batch1" def test_error_for_invalid_sku(): - batch = model.Batch("b1", "AREALSKU", 100, eta=None) - repo = FakeRepository([batch]) - + repo = FakeRepository.for_batch("b1", "AREALSKU", 100, eta=None) with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) def test_commits(): - batch = model.Batch("b1", "OMINOUS-MIRROR", 100, eta=None) - repo = FakeRepository([batch]) + repo = FakeRepository.for_batch("b1", "OMINOUS-MIRROR", 100, eta=None) session = FakeSession() - services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) assert session.committed is True From 5c953dada5e1a136ec21f8eaf15a588e32d4b858 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:37:17 +0100 Subject: [PATCH 025/145] new service to add a batch [add_batch_service] --- service_layer/services.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/service_layer/services.py b/service_layer/services.py index 464cdb7c..36c7bd9a 100644 --- a/service_layer/services.py +++ b/service_layer/services.py @@ -1,4 +1,6 @@ from __future__ import annotations +from typing import Optional +from datetime import date from domain import model from domain.model import OrderLine @@ -13,6 +15,14 @@ def is_valid_sku(sku, batches): return sku in {b.sku for b in batches} +def add_batch( + ref: str, sku: str, qty: int, eta: Optional[date], + repo: AbstractRepository, session, +) -> None: + repo.add(model.Batch(ref, sku, qty, eta)) + session.commit() + + def allocate( orderid: str, sku: str, qty: int, repo: AbstractRepository, session From 262eec06146958ed6b8a47d700f9c32fd92638ce Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:37:39 +0100 Subject: [PATCH 026/145] service-layer test for add batch [test_add_batch] --- tests/unit/test_services.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 0d404f4a..f5574ef8 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -29,6 +29,13 @@ def commit(self): self.committed = True +def test_add_batch(): + repo, session = FakeRepository([]), FakeSession() + services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, repo, session) + assert repo.get("b1") is not None + assert session.committed + + def test_returns_allocation(): repo = FakeRepository.for_batch("batch1", "COMPLICATED-LAMP", 100, eta=None) result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) From c8fbb60a7bbc918e5ea9a3f7ea70d35118c41620 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:37:57 +0100 Subject: [PATCH 027/145] all service-layer tests now services [services_tests_all_services] --- tests/unit/test_services.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index f5574ef8..7d9d8738 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,14 +1,9 @@ import pytest -from domain import model from adapters import repository from service_layer import services class FakeRepository(repository.AbstractRepository): - @staticmethod - def for_batch(ref, sku, qty, eta=None): - return FakeRepository([model.Batch(ref, sku, qty, eta),]) - def __init__(self, batches): self._batches = set(batches) @@ -36,20 +31,24 @@ def test_add_batch(): assert session.committed -def test_returns_allocation(): - repo = FakeRepository.for_batch("batch1", "COMPLICATED-LAMP", 100, eta=None) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) +def test_allocate_returns_allocation(): + repo, session = FakeRepository([]), FakeSession() + services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, repo, session) + result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, session) assert result == "batch1" -def test_error_for_invalid_sku(): - repo = FakeRepository.for_batch("b1", "AREALSKU", 100, eta=None) +def test_allocate_errors_for_invalid_sku(): + repo, session = FakeRepository([]), FakeSession() + services.add_batch("b1", "AREALSKU", 100, None, repo, session) + with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) def test_commits(): - repo = FakeRepository.for_batch("b1", "OMINOUS-MIRROR", 100, eta=None) + repo, session = FakeRepository([]), FakeSession() session = FakeSession() + services.add_batch("b1", "OMINOUS-MIRROR", 100, None, repo, session) services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) assert session.committed is True From 96301d2ca4aeb0f884906a500d2044821839dab5 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:44:09 +0100 Subject: [PATCH 028/145] modify flask app to use new service layer api [api_uses_modified_service] --- entrypoints/flask_app.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/entrypoints/flask_app.py b/entrypoints/flask_app.py index 62e40d44..aace66e7 100644 --- a/entrypoints/flask_app.py +++ b/entrypoints/flask_app.py @@ -17,12 +17,14 @@ def allocate_endpoint(): session = get_session() repo = repository.SqlAlchemyRepository(session) - line = model.OrderLine( - request.json["orderid"], request.json["sku"], request.json["qty"], - ) - try: - batchref = services.allocate(line, repo, session) + batchref = services.allocate( + request.json["orderid"], + request.json["sku"], + request.json["qty"], + repo, + session, + ) except (model.OutOfStock, services.InvalidSku) as e: return {"message": str(e)}, 400 From 6b404b52471786fccd439875e311985cecf35ea8 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 24 Apr 2019 12:21:59 +0100 Subject: [PATCH 029/145] add api endpoint for add_batch [api_for_add_batch] --- entrypoints/flask_app.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/entrypoints/flask_app.py b/entrypoints/flask_app.py index aace66e7..15ba414b 100644 --- a/entrypoints/flask_app.py +++ b/entrypoints/flask_app.py @@ -1,3 +1,4 @@ +from datetime import datetime from flask import Flask, request from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -7,12 +8,29 @@ from adapters import orm, repository from service_layer import services - orm.start_mappers() get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) app = Flask(__name__) +@app.route("/add_batch", methods=["POST"]) +def add_batch(): + session = get_session() + repo = repository.SqlAlchemyRepository(session) + eta = request.json["eta"] + if eta is not None: + eta = datetime.fromisoformat(eta).date() + services.add_batch( + request.json["ref"], + request.json["sku"], + request.json["qty"], + eta, + repo, + session, + ) + return "OK", 201 + + @app.route("/allocate", methods=["POST"]) def allocate_endpoint(): session = get_session() From fd45a6ff9e0ef20fc4dc698d169139f4f830c775 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 24 Apr 2019 12:22:34 +0100 Subject: [PATCH 030/145] api tests no longer need hardcoded sql fixture [chapter_05_high_gear_low_gear_ends] --- tests/conftest.py | 37 ------------------------------------- tests/e2e/test_api.py | 24 +++++++++++++++--------- 2 files changed, 15 insertions(+), 46 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 11e66b35..447164f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,43 +63,6 @@ def postgres_session(postgres_db): clear_mappers() -@pytest.fixture -def add_stock(postgres_session): - batches_added = set() - skus_added = set() - - def _add_stock(lines): - for ref, sku, qty, eta in lines: - postgres_session.execute( - "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" - " VALUES (:ref, :sku, :qty, :eta)", - dict(ref=ref, sku=sku, qty=qty, eta=eta), - ) - [[batch_id]] = postgres_session.execute( - "SELECT id FROM batches WHERE reference=:ref AND sku=:sku", - dict(ref=ref, sku=sku), - ) - batches_added.add(batch_id) - skus_added.add(sku) - postgres_session.commit() - - yield _add_stock - - for batch_id in batches_added: - postgres_session.execute( - "DELETE FROM allocations WHERE batch_id=:batch_id", - dict(batch_id=batch_id), - ) - postgres_session.execute( - "DELETE FROM batches WHERE id=:batch_id", dict(batch_id=batch_id), - ) - for sku in skus_added: - postgres_session.execute( - "DELETE FROM order_lines WHERE sku=:sku", dict(sku=sku), - ) - postgres_session.commit() - - @pytest.fixture def restart_api(): (Path(__file__).parent / "../entrypoints/flask_app.py").touch() diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 5386ab88..991db8db 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -21,28 +21,34 @@ def random_orderid(name=""): return f"order-{name}-{random_suffix()}" +def post_to_add_batch(ref, sku, qty, eta): + url = config.get_api_url() + r = requests.post( + f"{url}/add_batch", json={"ref": ref, "sku": sku, "qty": qty, "eta": eta} + ) + assert r.status_code == 201 + + +@pytest.mark.usefixtures("postgres_db") @pytest.mark.usefixtures("restart_api") -def test_happy_path_returns_201_and_allocated_batch(add_stock): +def test_happy_path_returns_201_and_allocated_batch(): sku, othersku = random_sku(), random_sku("other") earlybatch = random_batchref(1) laterbatch = random_batchref(2) otherbatch = random_batchref(3) - add_stock( - [ - (laterbatch, sku, 100, "2011-01-02"), - (earlybatch, sku, 100, "2011-01-01"), - (otherbatch, othersku, 100, None), - ] - ) + post_to_add_batch(laterbatch, sku, 100, "2011-01-02") + post_to_add_batch(earlybatch, sku, 100, "2011-01-01") + post_to_add_batch(otherbatch, othersku, 100, None) data = {"orderid": random_orderid(), "sku": sku, "qty": 3} - url = config.get_api_url() + url = config.get_api_url() r = requests.post(f"{url}/allocate", json=data) assert r.status_code == 201 assert r.json()["batchref"] == earlybatch +@pytest.mark.usefixtures("postgres_db") @pytest.mark.usefixtures("restart_api") def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() From d9c340c26624145c4114f7b2b6f7bf963999885f Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 27 Feb 2019 06:13:15 +0000 Subject: [PATCH 031/145] start moving files into src folder and add setup.py --- Dockerfile | 15 ++++++++++----- {adapters => src/allocation}/__init__.py | 0 {domain => src/allocation/adapters}/__init__.py | 0 {adapters => src/allocation/adapters}/orm.py | 0 .../allocation/adapters}/repository.py | 0 config.py => src/allocation/config.py | 0 .../allocation/domain}/__init__.py | 0 {domain => src/allocation/domain}/model.py | 0 .../allocation/entrypoints}/__init__.py | 0 .../allocation/entrypoints}/flask_app.py | 0 src/allocation/service_layer/__init__.py | 0 .../allocation/service_layer}/services.py | 11 +++++++---- src/setup.py | 5 +++++ 13 files changed, 22 insertions(+), 9 deletions(-) rename {adapters => src/allocation}/__init__.py (100%) rename {domain => src/allocation/adapters}/__init__.py (100%) rename {adapters => src/allocation/adapters}/orm.py (100%) rename {adapters => src/allocation/adapters}/repository.py (100%) rename config.py => src/allocation/config.py (100%) rename {entrypoints => src/allocation/domain}/__init__.py (100%) rename {domain => src/allocation/domain}/model.py (100%) rename {service_layer => src/allocation/entrypoints}/__init__.py (100%) rename {entrypoints => src/allocation/entrypoints}/flask_app.py (100%) create mode 100644 src/allocation/service_layer/__init__.py rename {service_layer => src/allocation/service_layer}/services.py (79%) create mode 100644 src/setup.py diff --git a/Dockerfile b/Dockerfile index d4803afb..73024d18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,15 @@ FROM python:3.9-slim-buster -COPY requirements.txt /tmp +# RUN apt install gcc libpq (no longer needed bc we use psycopg2-binary) + +COPY requirements.txt /tmp/ RUN pip install -r /tmp/requirements.txt -RUN mkdir -p /code -COPY *.py /code/ -WORKDIR /code -ENV FLASK_APP=entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 +RUN mkdir -p /src +COPY src/ /src/ +RUN pip install -e /src +COPY tests/ /tests/ + +WORKDIR /src +ENV FLASK_APP=allocation/entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 CMD flask run --host=0.0.0.0 --port=80 diff --git a/adapters/__init__.py b/src/allocation/__init__.py similarity index 100% rename from adapters/__init__.py rename to src/allocation/__init__.py diff --git a/domain/__init__.py b/src/allocation/adapters/__init__.py similarity index 100% rename from domain/__init__.py rename to src/allocation/adapters/__init__.py diff --git a/adapters/orm.py b/src/allocation/adapters/orm.py similarity index 100% rename from adapters/orm.py rename to src/allocation/adapters/orm.py diff --git a/adapters/repository.py b/src/allocation/adapters/repository.py similarity index 100% rename from adapters/repository.py rename to src/allocation/adapters/repository.py diff --git a/config.py b/src/allocation/config.py similarity index 100% rename from config.py rename to src/allocation/config.py diff --git a/entrypoints/__init__.py b/src/allocation/domain/__init__.py similarity index 100% rename from entrypoints/__init__.py rename to src/allocation/domain/__init__.py diff --git a/domain/model.py b/src/allocation/domain/model.py similarity index 100% rename from domain/model.py rename to src/allocation/domain/model.py diff --git a/service_layer/__init__.py b/src/allocation/entrypoints/__init__.py similarity index 100% rename from service_layer/__init__.py rename to src/allocation/entrypoints/__init__.py diff --git a/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py similarity index 100% rename from entrypoints/flask_app.py rename to src/allocation/entrypoints/flask_app.py diff --git a/src/allocation/service_layer/__init__.py b/src/allocation/service_layer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/service_layer/services.py b/src/allocation/service_layer/services.py similarity index 79% rename from service_layer/services.py rename to src/allocation/service_layer/services.py index 36c7bd9a..d9e1108f 100644 --- a/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -16,16 +16,19 @@ def is_valid_sku(sku, batches): def add_batch( - ref: str, sku: str, qty: int, eta: Optional[date], - repo: AbstractRepository, session, + ref: str, + sku: str, + qty: int, + eta: Optional[date], + repo: AbstractRepository, + session, ) -> None: repo.add(model.Batch(ref, sku, qty, eta)) session.commit() def allocate( - orderid: str, sku: str, qty: int, - repo: AbstractRepository, session + orderid: str, sku: str, qty: int, repo: AbstractRepository, session ) -> str: line = OrderLine(orderid, sku, qty) batches = repo.list() diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 00000000..731007b0 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup( + name="allocation", version="0.1", packages=["allocation"], +) From c843a10031e471cfd019bbf6f090afa95a251085 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 27 Feb 2019 06:29:22 +0000 Subject: [PATCH 032/145] fix all the imports, get it all working --- docker-compose.yml | 2 +- mypy.ini | 7 ++----- src/allocation/adapters/orm.py | 2 +- src/allocation/adapters/repository.py | 2 +- src/allocation/entrypoints/flask_app.py | 8 ++++---- src/allocation/service_layer/services.py | 6 +++--- tests/conftest.py | 6 +++--- tests/e2e/test_api.py | 2 +- tests/integration/test_orm.py | 2 +- tests/integration/test_repository.py | 4 ++-- tests/mypy.ini | 6 ++++++ tests/unit/test_allocate.py | 2 +- tests/unit/test_batches.py | 2 +- tests/unit/test_services.py | 4 ++-- 14 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 tests/mypy.ini diff --git a/docker-compose.yml b/docker-compose.yml index 313cf946..cc8341f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - DB_HOST=postgres - DB_PASSWORD=abc123 volumes: - - ./:/code + - ./src:/src ports: - "5005:80" diff --git a/mypy.ini b/mypy.ini index ead5ef09..65ab939b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,9 +1,6 @@ [mypy] ignore_missing_imports = False +mypy_path = ./src -[mypy-pytest.*] +[mypy-pytest.*,sqlalchemy.*] ignore_missing_imports = True - -[mypy-sqlalchemy.*] -ignore_missing_imports = True - diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 8499c2dc..444a8cbb 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -1,7 +1,7 @@ from sqlalchemy import Table, MetaData, Column, Integer, String, Date, ForeignKey from sqlalchemy.orm import mapper, relationship -from domain import model +from allocation.domain import model metadata = MetaData() diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 7e7434b9..85e46076 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,5 +1,5 @@ import abc -from domain import model +from allocation.domain import model class AbstractRepository(abc.ABC): diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 15ba414b..2164c9b6 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -3,10 +3,10 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import config -from domain import model -from adapters import orm, repository -from service_layer import services +from allocation import config +from allocation.domain import model +from allocation.adapters import orm, repository +from allocation.service_layer import services orm.start_mappers() get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index d9e1108f..845dec39 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -2,9 +2,9 @@ from typing import Optional from datetime import date -from domain import model -from domain.model import OrderLine -from adapters.repository import AbstractRepository +from allocation.domain import model +from allocation.domain.model import OrderLine +from allocation.adapters.repository import AbstractRepository class InvalidSku(Exception): diff --git a/tests/conftest.py b/tests/conftest.py index 447164f4..bcf0fa4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,8 +9,8 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers -from adapters.orm import metadata, start_mappers -import config +from allocation.adapters.orm import metadata, start_mappers +from allocation import config @pytest.fixture @@ -65,6 +65,6 @@ def postgres_session(postgres_db): @pytest.fixture def restart_api(): - (Path(__file__).parent / "../entrypoints/flask_app.py").touch() + (Path(__file__).parent / "../src/allocation/entrypoints/flask_app.py").touch() time.sleep(0.5) wait_for_webapp_to_come_up() diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 991db8db..29b85761 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -2,7 +2,7 @@ import pytest import requests -import config +from allocation import config def random_suffix(): diff --git a/tests/integration/test_orm.py b/tests/integration/test_orm.py index 3a551977..db3a7a68 100644 --- a/tests/integration/test_orm.py +++ b/tests/integration/test_orm.py @@ -1,4 +1,4 @@ -from domain import model +from allocation.domain import model from datetime import date diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index 3fa66d9f..0fa69ab3 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,6 +1,6 @@ # pylint: disable=protected-access -from domain import model -from adapters import repository +from allocation.domain import model +from allocation.adapters import repository def test_repository_can_save_a_batch(session): diff --git a/tests/mypy.ini b/tests/mypy.ini new file mode 100644 index 00000000..39fadd90 --- /dev/null +++ b/tests/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +ignore_missing_imports = False +mypy_path = ../src + +[mypy-pytest.*,sqlalchemy.*] +ignore_missing_imports = True diff --git a/tests/unit/test_allocate.py b/tests/unit/test_allocate.py index 4e434e28..48dcfe5c 100644 --- a/tests/unit/test_allocate.py +++ b/tests/unit/test_allocate.py @@ -1,6 +1,6 @@ from datetime import date, timedelta import pytest -from domain.model import allocate, OrderLine, Batch, OutOfStock +from allocation.domain.model import allocate, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) diff --git a/tests/unit/test_batches.py b/tests/unit/test_batches.py index 7feda3d7..8735f41e 100644 --- a/tests/unit/test_batches.py +++ b/tests/unit/test_batches.py @@ -1,5 +1,5 @@ from datetime import date -from domain.model import Batch, OrderLine +from allocation.domain.model import Batch, OrderLine def test_allocating_to_a_batch_reduces_the_available_quantity(): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 7d9d8738..af0ae807 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,6 +1,6 @@ import pytest -from adapters import repository -from service_layer import services +from allocation.adapters import repository +from allocation.service_layer import services class FakeRepository(repository.AbstractRepository): From 12d2bc2ca5f9ef644ad762eb022c6bb3a9bb6d0d Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 27 Feb 2019 06:44:34 +0000 Subject: [PATCH 033/145] get tests working in docker container --- .travis.yml | 3 --- Makefile | 17 +++++++++++++---- docker-compose.yml | 3 +++ tests/__init__.py | 0 tests/mypy.ini | 6 ------ tests/pytest.ini | 2 ++ 6 files changed, 18 insertions(+), 13 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/mypy.ini create mode 100644 tests/pytest.ini diff --git a/.travis.yml b/.travis.yml index f601680f..fcd3ceea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,6 @@ dist: xenial language: python python: 3.8 -install: -- pip3 install -r requirements.txt - script: - make all diff --git a/Makefile b/Makefile index 1e248187..6409e955 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,22 @@ up: docker-compose up -d app down: - docker-compose down + docker-compose down --remove-orphans + +test: up + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit /tests/integration /tests/e2e + +unit-tests: + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit + +integration-tests: up + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/integration + +e2e-tests: up + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/e2e logs: docker-compose logs app | tail -100 -test: - pytest --tb=short - black: black -l 86 $$(find * -name '*.py') diff --git a/docker-compose.yml b/docker-compose.yml index cc8341f9..039400e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,11 @@ services: environment: - DB_HOST=postgres - DB_PASSWORD=abc123 + - API_HOST=app + - PYTHONDONTWRITEBYTECODE=1 volumes: - ./src:/src + - ./tests:/tests ports: - "5005:80" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/mypy.ini b/tests/mypy.ini deleted file mode 100644 index 39fadd90..00000000 --- a/tests/mypy.ini +++ /dev/null @@ -1,6 +0,0 @@ -[mypy] -ignore_missing_imports = False -mypy_path = ../src - -[mypy-pytest.*,sqlalchemy.*] -ignore_missing_imports = True diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000..bbd083ac --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --tb=short From 28afa9a64c7b109837475ad683a99fef1f9101a2 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 24 May 2019 00:10:48 +0100 Subject: [PATCH 034/145] make mypy slightly stricter --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index 65ab939b..62194f35 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,7 @@ [mypy] ignore_missing_imports = False mypy_path = ./src +check_untyped_defs = True [mypy-pytest.*,sqlalchemy.*] ignore_missing_imports = True From 53ad7987bfc8b15d029c45a143743903a973215e Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 23:08:10 +0000 Subject: [PATCH 035/145] better requirements.txt [appendix_project_structure_ends] --- requirements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 01a974f5..8c779254 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,10 @@ -pytest +# app sqlalchemy flask psycopg2-binary + +# tests +pytest +pytest-icdiff +mypy requests From 833d48bedf2599eaaf645def0ee419864bb68fe0 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 23:11:44 +0000 Subject: [PATCH 036/145] basic uow test, uow and conftest.py changes --- src/allocation/service_layer/services.py | 11 ++---- src/allocation/unit_of_work.py | 50 ++++++++++++++++++++++++ tests/conftest.py | 9 ++++- tests/integration/test_uow.py | 40 +++++++++++++++++++ 4 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 src/allocation/unit_of_work.py create mode 100644 tests/integration/test_uow.py diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 845dec39..051de589 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -16,19 +16,16 @@ def is_valid_sku(sku, batches): def add_batch( - ref: str, - sku: str, - qty: int, - eta: Optional[date], - repo: AbstractRepository, - session, + ref: str, sku: str, qty: int, eta: Optional[date], + repo: AbstractRepository, session, ) -> None: repo.add(model.Batch(ref, sku, qty, eta)) session.commit() def allocate( - orderid: str, sku: str, qty: int, repo: AbstractRepository, session + orderid: str, sku: str, qty: int, + repo: AbstractRepository, session, ) -> str: line = OrderLine(orderid, sku, qty) batches = repo.list() diff --git a/src/allocation/unit_of_work.py b/src/allocation/unit_of_work.py new file mode 100644 index 00000000..d89e1138 --- /dev/null +++ b/src/allocation/unit_of_work.py @@ -0,0 +1,50 @@ +# pylint: disable=attribute-defined-outside-init +from __future__ import annotations +import abc +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm.session import Session + +from allocation import config +from allocation import repository + + +class AbstractUnitOfWork(abc.ABC): + batches: repository.AbstractRepository + + def __enter__(self) -> AbstractUnitOfWork: + return self + + def __exit__(self, *args): + self.rollback() + + @abc.abstractmethod + def commit(self): + raise NotImplementedError + + @abc.abstractmethod + def rollback(self): + raise NotImplementedError + + +DEFAULT_SESSION_FACTORY = sessionmaker(bind=create_engine(config.get_postgres_uri(),)) + + +class SqlAlchemyUnitOfWork(AbstractUnitOfWork): + def __init__(self, session_factory=DEFAULT_SESSION_FACTORY): + self.session_factory = session_factory + + def __enter__(self): + self.session = self.session_factory() # type: Session + self.batches = repository.SqlAlchemyRepository(self.session) + return super().__enter__() + + def __exit__(self, *args): + super().__exit__(*args) + self.session.close() + + def commit(self): + self.session.commit() + + def rollback(self): + self.session.rollback() diff --git a/tests/conftest.py b/tests/conftest.py index bcf0fa4e..3dff0b83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,12 +21,17 @@ def in_memory_db(): @pytest.fixture -def session(in_memory_db): +def session_factory(in_memory_db): start_mappers() - yield sessionmaker(bind=in_memory_db)() + yield sessionmaker(bind=in_memory_db) clear_mappers() +@pytest.fixture +def session(session_factory): + return session_factory() + + def wait_for_postgres_to_come_up(engine): deadline = time.time() + 10 while time.time() < deadline: diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py new file mode 100644 index 00000000..5ee11051 --- /dev/null +++ b/tests/integration/test_uow.py @@ -0,0 +1,40 @@ +import pytest +from allocation import model +from allocation import unit_of_work + + +def insert_batch(session, ref, sku, qty, eta): + session.execute( + "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" + " VALUES (:ref, :sku, :qty, :eta)", + dict(ref=ref, sku=sku, qty=qty, eta=eta), + ) + + +def get_allocated_batch_ref(session, orderid, sku): + [[orderlineid]] = session.execute( + "SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku", + dict(orderid=orderid, sku=sku), + ) + [[batchref]] = session.execute( + "SELECT b.reference FROM allocations JOIN batches AS b ON batch_id = b.id" + " WHERE orderline_id=:orderlineid", + dict(orderlineid=orderlineid), + ) + return batchref + + +def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): + session = session_factory() + insert_batch(session, "batch1", "HIPSTER-WORKBENCH", 100, None) + session.commit() + + uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + with uow: + batch = uow.batches.get(reference="batch1") + line = model.OrderLine("o1", "HIPSTER-WORKBENCH", 10) + batch.allocate(line) + uow.commit() + + batchref = get_allocated_batch_ref(session, "o1", "HIPSTER-WORKBENCH") + assert batchref == "batch1" From 658e61acd072c7f2bdec8fb0e9b50a9adc0edfdb Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 20 Mar 2019 10:53:14 +0000 Subject: [PATCH 037/145] use uow in services, flask app --- src/allocation/adapters/orm.py | 4 +- src/allocation/entrypoints/flask_app.py | 18 +++----- src/allocation/service_layer/services.py | 24 ++++++----- .../{ => service_layer}/unit_of_work.py | 8 +++- src/setup.py | 4 +- tests/integration/test_uow.py | 5 +-- tests/unit/test_services.py | 42 ++++++++++--------- 7 files changed, 55 insertions(+), 50 deletions(-) rename src/allocation/{ => service_layer}/unit_of_work.py (88%) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 444a8cbb..c189a739 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -41,7 +41,9 @@ def start_mappers(): batches, properties={ "_allocations": relationship( - lines_mapper, secondary=allocations, collection_class=set, + lines_mapper, + secondary=allocations, + collection_class=set, ) }, ) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 2164c9b6..602a09c1 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -3,20 +3,16 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from allocation import config from allocation.domain import model -from allocation.adapters import orm, repository -from allocation.service_layer import services +from allocation.adapters import orm +from allocation.service_layer import services, unit_of_work -orm.start_mappers() -get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) app = Flask(__name__) +orm.start_mappers() @app.route("/add_batch", methods=["POST"]) def add_batch(): - session = get_session() - repo = repository.SqlAlchemyRepository(session) eta = request.json["eta"] if eta is not None: eta = datetime.fromisoformat(eta).date() @@ -25,23 +21,19 @@ def add_batch(): request.json["sku"], request.json["qty"], eta, - repo, - session, + unit_of_work.SqlAlchemyUnitOfWork(), ) return "OK", 201 @app.route("/allocate", methods=["POST"]) def allocate_endpoint(): - session = get_session() - repo = repository.SqlAlchemyRepository(session) try: batchref = services.allocate( request.json["orderid"], request.json["sku"], request.json["qty"], - repo, - session, + unit_of_work.SqlAlchemyUnitOfWork(), ) except (model.OutOfStock, services.InvalidSku) as e: return {"message": str(e)}, 400 diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 051de589..c5d46772 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -4,7 +4,7 @@ from allocation.domain import model from allocation.domain.model import OrderLine -from allocation.adapters.repository import AbstractRepository +from allocation.service_layer import unit_of_work class InvalidSku(Exception): @@ -17,20 +17,22 @@ def is_valid_sku(sku, batches): def add_batch( ref: str, sku: str, qty: int, eta: Optional[date], - repo: AbstractRepository, session, -) -> None: - repo.add(model.Batch(ref, sku, qty, eta)) - session.commit() + uow: unit_of_work.AbstractUnitOfWork, +): + with uow: + uow.batches.add(model.Batch(ref, sku, qty, eta)) + uow.commit() def allocate( orderid: str, sku: str, qty: int, - repo: AbstractRepository, session, + uow: unit_of_work.AbstractUnitOfWork, ) -> str: line = OrderLine(orderid, sku, qty) - batches = repo.list() - if not is_valid_sku(line.sku, batches): - raise InvalidSku(f"Invalid sku {line.sku}") - batchref = model.allocate(line, batches) - session.commit() + with uow: + batches = uow.batches.list() + if not is_valid_sku(line.sku, batches): + raise InvalidSku(f"Invalid sku {line.sku}") + batchref = model.allocate(line, batches) + uow.commit() return batchref diff --git a/src/allocation/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py similarity index 88% rename from src/allocation/unit_of_work.py rename to src/allocation/service_layer/unit_of_work.py index d89e1138..bf9196c8 100644 --- a/src/allocation/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -6,7 +6,7 @@ from sqlalchemy.orm.session import Session from allocation import config -from allocation import repository +from allocation.adapters import repository class AbstractUnitOfWork(abc.ABC): @@ -27,7 +27,11 @@ def rollback(self): raise NotImplementedError -DEFAULT_SESSION_FACTORY = sessionmaker(bind=create_engine(config.get_postgres_uri(),)) +DEFAULT_SESSION_FACTORY = sessionmaker( + bind=create_engine( + config.get_postgres_uri(), + ) +) class SqlAlchemyUnitOfWork(AbstractUnitOfWork): diff --git a/src/setup.py b/src/setup.py index 731007b0..b2b0839a 100644 --- a/src/setup.py +++ b/src/setup.py @@ -1,5 +1,7 @@ from setuptools import setup setup( - name="allocation", version="0.1", packages=["allocation"], + name="allocation", + version="0.1", + packages=["allocation"], ) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 5ee11051..23f27619 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,6 +1,5 @@ -import pytest -from allocation import model -from allocation import unit_of_work +from allocation.domain import model +from allocation.service_layer import unit_of_work def insert_batch(session, ref, sku, qty, eta): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index af0ae807..091dbb2c 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,6 +1,6 @@ import pytest from allocation.adapters import repository -from allocation.service_layer import services +from allocation.service_layer import services, unit_of_work class FakeRepository(repository.AbstractRepository): @@ -17,38 +17,42 @@ def list(self): return list(self._batches) -class FakeSession: - committed = False +class FakeUnitOfWork(unit_of_work.AbstractUnitOfWork): + def __init__(self): + self.batches = FakeRepository([]) + self.committed = False def commit(self): self.committed = True + def rollback(self): + pass + def test_add_batch(): - repo, session = FakeRepository([]), FakeSession() - services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, repo, session) - assert repo.get("b1") is not None - assert session.committed + uow = FakeUnitOfWork() + services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) + assert uow.batches.get("b1") is not None + assert uow.committed def test_allocate_returns_allocation(): - repo, session = FakeRepository([]), FakeSession() - services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, repo, session) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, session) + uow = FakeUnitOfWork() + services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) + result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) assert result == "batch1" def test_allocate_errors_for_invalid_sku(): - repo, session = FakeRepository([]), FakeSession() - services.add_batch("b1", "AREALSKU", 100, None, repo, session) + uow = FakeUnitOfWork() + services.add_batch("b1", "AREALSKU", 100, None, uow) with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) + services.allocate("o1", "NONEXISTENTSKU", 10, uow) -def test_commits(): - repo, session = FakeRepository([]), FakeSession() - session = FakeSession() - services.add_batch("b1", "OMINOUS-MIRROR", 100, None, repo, session) - services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) - assert session.committed is True +def test_allocate_commits(): + uow = FakeUnitOfWork() + services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) + services.allocate("o1", "OMINOUS-MIRROR", 10, uow) + assert uow.committed From 7526014be1a288f967393ed7bb544655c87a2817 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 12 Mar 2019 11:52:13 +0000 Subject: [PATCH 038/145] two more tests for rollback behaviour [chapter_06_uow_ends] --- tests/integration/test_uow.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 23f27619..3887e3ca 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,3 +1,4 @@ +import pytest from allocation.domain import model from allocation.service_layer import unit_of_work @@ -37,3 +38,28 @@ def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): batchref = get_allocated_batch_ref(session, "o1", "HIPSTER-WORKBENCH") assert batchref == "batch1" + + +def test_rolls_back_uncommitted_work_by_default(session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + with uow: + insert_batch(uow.session, "batch1", "MEDIUM-PLINTH", 100, None) + + new_session = session_factory() + rows = list(new_session.execute('SELECT * FROM "batches"')) + assert rows == [] + + +def test_rolls_back_on_error(session_factory): + class MyException(Exception): + pass + + uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + with pytest.raises(MyException): + with uow: + insert_batch(uow.session, "batch1", "LARGE-FORK", 100, None) + raise MyException() + + new_session = session_factory() + rows = list(new_session.execute('SELECT * FROM "batches"')) + assert rows == [] From c50ab2c54cdc00d829eb60c6776a2cc2e93ac2e9 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 10:26:16 +0000 Subject: [PATCH 039/145] start on a Product model with an allocate fn [product_aggregate] --- src/allocation/adapters/orm.py | 14 ++- src/allocation/adapters/repository.py | 15 ++-- src/allocation/domain/model.py | 19 ++-- src/allocation/service_layer/services.py | 16 ++-- src/allocation/service_layer/unit_of_work.py | 4 +- tests/integration/test_orm.py | 89 ------------------- tests/integration/test_repository.py | 67 -------------- tests/integration/test_uow.py | 8 +- .../{test_allocate.py => test_product.py} | 18 ++-- tests/unit/test_services.py | 28 +++--- 10 files changed, 73 insertions(+), 205 deletions(-) delete mode 100644 tests/integration/test_orm.py delete mode 100644 tests/integration/test_repository.py rename tests/unit/{test_allocate.py => test_product.py} (68%) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index c189a739..2af008a2 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -15,12 +15,19 @@ Column("orderid", String(255)), ) +products = Table( + "products", + metadata, + Column("sku", String(255), primary_key=True), + # Column('version_number', Integer, nullable=False, default=0), +) + batches = Table( "batches", metadata, Column("id", Integer, primary_key=True, autoincrement=True), Column("reference", String(255)), - Column("sku", String(255)), + Column("sku", ForeignKey("products.sku")), Column("_purchased_quantity", Integer, nullable=False), Column("eta", Date, nullable=True), ) @@ -36,7 +43,7 @@ def start_mappers(): lines_mapper = mapper(model.OrderLine, order_lines) - mapper( + batches_mapper = mapper( model.Batch, batches, properties={ @@ -47,3 +54,6 @@ def start_mappers(): ) }, ) + mapper( + model.Product, products, properties={"batches": relationship(batches_mapper)} + ) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 85e46076..7e18dbed 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -4,11 +4,11 @@ class AbstractRepository(abc.ABC): @abc.abstractmethod - def add(self, batch: model.Batch): + def add(self, product: model.Product): raise NotImplementedError @abc.abstractmethod - def get(self, reference) -> model.Batch: + def get(self, sku) -> model.Product: raise NotImplementedError @@ -16,11 +16,8 @@ class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): self.session = session - def add(self, batch): - self.session.add(batch) + def add(self, product): + self.session.add(product) - def get(self, reference): - return self.session.query(model.Batch).filter_by(reference=reference).one() - - def list(self): - return self.session.query(model.Batch).all() + def get(self, sku): + return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index b3203325..c432783c 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -8,13 +8,18 @@ class OutOfStock(Exception): pass -def allocate(line: OrderLine, batches: List[Batch]) -> str: - try: - batch = next(b for b in sorted(batches) if b.can_allocate(line)) - batch.allocate(line) - return batch.reference - except StopIteration: - raise OutOfStock(f"Out of stock for sku {line.sku}") +class Product: + def __init__(self, sku: str, batches: List[Batch]): + self.sku = sku + self.batches = batches + + def allocate(self, line: OrderLine) -> str: + try: + batch = next(b for b in sorted(self.batches) if b.can_allocate(line)) + batch.allocate(line) + return batch.reference + except StopIteration: + raise OutOfStock(f"Out of stock for sku {line.sku}") @dataclass(unsafe_hash=True) diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index c5d46772..5311ca6e 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -11,16 +11,16 @@ class InvalidSku(Exception): pass -def is_valid_sku(sku, batches): - return sku in {b.sku for b in batches} - - def add_batch( ref: str, sku: str, qty: int, eta: Optional[date], uow: unit_of_work.AbstractUnitOfWork, ): with uow: - uow.batches.add(model.Batch(ref, sku, qty, eta)) + product = uow.products.get(sku=sku) + if product is None: + product = model.Product(sku, batches=[]) + uow.products.add(product) + product.batches.append(model.Batch(ref, sku, qty, eta)) uow.commit() @@ -30,9 +30,9 @@ def allocate( ) -> str: line = OrderLine(orderid, sku, qty) with uow: - batches = uow.batches.list() - if not is_valid_sku(line.sku, batches): + product = uow.products.get(sku=line.sku) + if product is None: raise InvalidSku(f"Invalid sku {line.sku}") - batchref = model.allocate(line, batches) + batchref = product.allocate(line) uow.commit() return batchref diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index bf9196c8..2b41b7a9 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -10,7 +10,7 @@ class AbstractUnitOfWork(abc.ABC): - batches: repository.AbstractRepository + products: repository.AbstractRepository def __enter__(self) -> AbstractUnitOfWork: return self @@ -40,7 +40,7 @@ def __init__(self, session_factory=DEFAULT_SESSION_FACTORY): def __enter__(self): self.session = self.session_factory() # type: Session - self.batches = repository.SqlAlchemyRepository(self.session) + self.products = repository.SqlAlchemyRepository(self.session) return super().__enter__() def __exit__(self, *args): diff --git a/tests/integration/test_orm.py b/tests/integration/test_orm.py deleted file mode 100644 index db3a7a68..00000000 --- a/tests/integration/test_orm.py +++ /dev/null @@ -1,89 +0,0 @@ -from allocation.domain import model -from datetime import date - - -def test_orderline_mapper_can_load_lines(session): - session.execute( - "INSERT INTO order_lines (orderid, sku, qty) VALUES " - '("order1", "RED-CHAIR", 12),' - '("order1", "RED-TABLE", 13),' - '("order2", "BLUE-LIPSTICK", 14)' - ) - expected = [ - model.OrderLine("order1", "RED-CHAIR", 12), - model.OrderLine("order1", "RED-TABLE", 13), - model.OrderLine("order2", "BLUE-LIPSTICK", 14), - ] - assert session.query(model.OrderLine).all() == expected - - -def test_orderline_mapper_can_save_lines(session): - new_line = model.OrderLine("order1", "DECORATIVE-WIDGET", 12) - session.add(new_line) - session.commit() - - rows = list(session.execute('SELECT orderid, sku, qty FROM "order_lines"')) - assert rows == [("order1", "DECORATIVE-WIDGET", 12)] - - -def test_retrieving_batches(session): - session.execute( - "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" - ' VALUES ("batch1", "sku1", 100, null)' - ) - session.execute( - "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" - ' VALUES ("batch2", "sku2", 200, "2011-04-11")' - ) - expected = [ - model.Batch("batch1", "sku1", 100, eta=None), - model.Batch("batch2", "sku2", 200, eta=date(2011, 4, 11)), - ] - - assert session.query(model.Batch).all() == expected - - -def test_saving_batches(session): - batch = model.Batch("batch1", "sku1", 100, eta=None) - session.add(batch) - session.commit() - rows = session.execute( - 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' - ) - assert list(rows) == [("batch1", "sku1", 100, None)] - - -def test_saving_allocations(session): - batch = model.Batch("batch1", "sku1", 100, eta=None) - line = model.OrderLine("order1", "sku1", 10) - batch.allocate(line) - session.add(batch) - session.commit() - rows = list(session.execute('SELECT orderline_id, batch_id FROM "allocations"')) - assert rows == [(batch.id, line.id)] - - -def test_retrieving_allocations(session): - session.execute( - 'INSERT INTO order_lines (orderid, sku, qty) VALUES ("order1", "sku1", 12)' - ) - [[olid]] = session.execute( - "SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku", - dict(orderid="order1", sku="sku1"), - ) - session.execute( - "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" - ' VALUES ("batch1", "sku1", 100, null)' - ) - [[bid]] = session.execute( - "SELECT id FROM batches WHERE reference=:ref AND sku=:sku", - dict(ref="batch1", sku="sku1"), - ) - session.execute( - "INSERT INTO allocations (orderline_id, batch_id) VALUES (:olid, :bid)", - dict(olid=olid, bid=bid), - ) - - batch = session.query(model.Batch).one() - - assert batch._allocations == {model.OrderLine("order1", "sku1", 12)} diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py deleted file mode 100644 index 0fa69ab3..00000000 --- a/tests/integration/test_repository.py +++ /dev/null @@ -1,67 +0,0 @@ -# pylint: disable=protected-access -from allocation.domain import model -from allocation.adapters import repository - - -def test_repository_can_save_a_batch(session): - batch = model.Batch("batch1", "RUSTY-SOAPDISH", 100, eta=None) - - repo = repository.SqlAlchemyRepository(session) - repo.add(batch) - session.commit() - - rows = session.execute( - 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' - ) - assert list(rows) == [("batch1", "RUSTY-SOAPDISH", 100, None)] - - -def insert_order_line(session): - session.execute( - "INSERT INTO order_lines (orderid, sku, qty)" - ' VALUES ("order1", "GENERIC-SOFA", 12)' - ) - [[orderline_id]] = session.execute( - "SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku", - dict(orderid="order1", sku="GENERIC-SOFA"), - ) - return orderline_id - - -def insert_batch(session, batch_id): - session.execute( - "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" - ' VALUES (:batch_id, "GENERIC-SOFA", 100, null)', - dict(batch_id=batch_id), - ) - [[batch_id]] = session.execute( - 'SELECT id FROM batches WHERE reference=:batch_id AND sku="GENERIC-SOFA"', - dict(batch_id=batch_id), - ) - return batch_id - - -def insert_allocation(session, orderline_id, batch_id): - session.execute( - "INSERT INTO allocations (orderline_id, batch_id)" - " VALUES (:orderline_id, :batch_id)", - dict(orderline_id=orderline_id, batch_id=batch_id), - ) - - -def test_repository_can_retrieve_a_batch_with_allocations(session): - orderline_id = insert_order_line(session) - batch1_id = insert_batch(session, "batch1") - insert_batch(session, "batch2") - insert_allocation(session, orderline_id, batch1_id) - - repo = repository.SqlAlchemyRepository(session) - retrieved = repo.get("batch1") - - expected = model.Batch("batch1", "GENERIC-SOFA", 100, eta=None) - assert retrieved == expected # Batch.__eq__ only compares reference - assert retrieved.sku == expected.sku - assert retrieved._purchased_quantity == expected._purchased_quantity - assert retrieved._allocations == { - model.OrderLine("order1", "GENERIC-SOFA", 12), - } diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 3887e3ca..8bf0c69d 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -4,6 +4,10 @@ def insert_batch(session, ref, sku, qty, eta): + session.execute( + "INSERT INTO products (sku) VALUES (:sku)", + dict(sku=sku), + ) session.execute( "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" " VALUES (:ref, :sku, :qty, :eta)", @@ -31,9 +35,9 @@ def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) with uow: - batch = uow.batches.get(reference="batch1") + product = uow.products.get(sku="HIPSTER-WORKBENCH") line = model.OrderLine("o1", "HIPSTER-WORKBENCH", 10) - batch.allocate(line) + product.allocate(line) uow.commit() batchref = get_allocated_batch_ref(session, "o1", "HIPSTER-WORKBENCH") diff --git a/tests/unit/test_allocate.py b/tests/unit/test_product.py similarity index 68% rename from tests/unit/test_allocate.py rename to tests/unit/test_product.py index 48dcfe5c..366611d6 100644 --- a/tests/unit/test_allocate.py +++ b/tests/unit/test_product.py @@ -1,18 +1,19 @@ from datetime import date, timedelta import pytest -from allocation.domain.model import allocate, OrderLine, Batch, OutOfStock +from allocation.domain.model import Product, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) later = tomorrow + timedelta(days=10) -def test_prefers_current_stock_batches_to_shipments(): +def test_prefers_warehouse_batches_to_shipments(): in_stock_batch = Batch("in-stock-batch", "RETRO-CLOCK", 100, eta=None) shipment_batch = Batch("shipment-batch", "RETRO-CLOCK", 100, eta=tomorrow) + product = Product(sku="RETRO-CLOCK", batches=[in_stock_batch, shipment_batch]) line = OrderLine("oref", "RETRO-CLOCK", 10) - allocate(line, [in_stock_batch, shipment_batch]) + product.allocate(line) assert in_stock_batch.available_quantity == 90 assert shipment_batch.available_quantity == 100 @@ -22,9 +23,10 @@ def test_prefers_earlier_batches(): earliest = Batch("speedy-batch", "MINIMALIST-SPOON", 100, eta=today) medium = Batch("normal-batch", "MINIMALIST-SPOON", 100, eta=tomorrow) latest = Batch("slow-batch", "MINIMALIST-SPOON", 100, eta=later) + product = Product(sku="MINIMALIST-SPOON", batches=[medium, earliest, latest]) line = OrderLine("order1", "MINIMALIST-SPOON", 10) - allocate(line, [medium, earliest, latest]) + product.allocate(line) assert earliest.available_quantity == 90 assert medium.available_quantity == 100 @@ -35,13 +37,15 @@ def test_returns_allocated_batch_ref(): in_stock_batch = Batch("in-stock-batch-ref", "HIGHBROW-POSTER", 100, eta=None) shipment_batch = Batch("shipment-batch-ref", "HIGHBROW-POSTER", 100, eta=tomorrow) line = OrderLine("oref", "HIGHBROW-POSTER", 10) - allocation = allocate(line, [in_stock_batch, shipment_batch]) + product = Product(sku="HIGHBROW-POSTER", batches=[in_stock_batch, shipment_batch]) + allocation = product.allocate(line) assert allocation == in_stock_batch.reference def test_raises_out_of_stock_exception_if_cannot_allocate(): batch = Batch("batch1", "SMALL-FORK", 10, eta=today) - allocate(OrderLine("order1", "SMALL-FORK", 10), [batch]) + product = Product(sku="SMALL-FORK", batches=[batch]) + product.allocate(OrderLine("order1", "SMALL-FORK", 10)) with pytest.raises(OutOfStock, match="SMALL-FORK"): - allocate(OrderLine("order2", "SMALL-FORK", 1), [batch]) + product.allocate(OrderLine("order2", "SMALL-FORK", 1)) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 091dbb2c..9055d03a 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -4,22 +4,19 @@ class FakeRepository(repository.AbstractRepository): - def __init__(self, batches): - self._batches = set(batches) + def __init__(self, products): + self._products = set(products) - def add(self, batch): - self._batches.add(batch) + def add(self, product): + self._products.add(product) - def get(self, reference): - return next(b for b in self._batches if b.reference == reference) - - def list(self): - return list(self._batches) + def get(self, sku): + return next((p for p in self._products if p.sku == sku), None) class FakeUnitOfWork(unit_of_work.AbstractUnitOfWork): def __init__(self): - self.batches = FakeRepository([]) + self.products = FakeRepository([]) self.committed = False def commit(self): @@ -29,13 +26,20 @@ def rollback(self): pass -def test_add_batch(): +def test_add_batch_for_new_product(): uow = FakeUnitOfWork() services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) - assert uow.batches.get("b1") is not None + assert uow.products.get("CRUNCHY-ARMCHAIR") is not None assert uow.committed +def test_add_batch_for_existing_product(): + uow = FakeUnitOfWork() + services.add_batch("b1", "GARISH-RUG", 100, None, uow) + services.add_batch("b2", "GARISH-RUG", 99, None, uow) + assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] + + def test_allocate_returns_allocation(): uow = FakeUnitOfWork() services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) From 9d6eed1b45c97ec9136a9cf9d4b07273e27e3d81 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 22:59:22 +0000 Subject: [PATCH 040/145] first cut of a product with a version number --- src/allocation/domain/model.py | 4 +++- tests/unit/test_product.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index c432783c..a48e1f69 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -9,14 +9,16 @@ class OutOfStock(Exception): class Product: - def __init__(self, sku: str, batches: List[Batch]): + def __init__(self, sku: str, batches: List[Batch], version_number: int = 0): self.sku = sku self.batches = batches + self.version_number = version_number def allocate(self, line: OrderLine) -> str: try: batch = next(b for b in sorted(self.batches) if b.can_allocate(line)) batch.allocate(line) + self.version_number += 1 return batch.reference except StopIteration: raise OutOfStock(f"Out of stock for sku {line.sku}") diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index 366611d6..05d75355 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -49,3 +49,13 @@ def test_raises_out_of_stock_exception_if_cannot_allocate(): with pytest.raises(OutOfStock, match="SMALL-FORK"): product.allocate(OrderLine("order2", "SMALL-FORK", 1)) + + +def test_increments_version_number(): + line = OrderLine("oref", "SCANDI-PEN", 10) + product = Product( + sku="SCANDI-PEN", batches=[Batch("b1", "SCANDI-PEN", 100, eta=None)] + ) + product.version_number = 7 + product.allocate(line) + assert product.version_number == 8 From 16109430b6fa180f349a7a09f537718ec6911caf Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 23:39:49 +0000 Subject: [PATCH 041/145] start adding tests for consistency on version_number [data_integrity_test] --- src/allocation/adapters/orm.py | 2 +- src/allocation/adapters/repository.py | 1 + tests/__init__.py | 0 tests/conftest.py | 9 +++- tests/e2e/__init__.py | 0 tests/e2e/test_api.py | 18 +------- tests/integration/__init__.py | 0 tests/integration/test_uow.py | 62 +++++++++++++++++++++++++-- tests/random_refs.py | 17 ++++++++ 9 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/random_refs.py diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 2af008a2..ddd1f3ab 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -19,7 +19,7 @@ "products", metadata, Column("sku", String(255), primary_key=True), - # Column('version_number', Integer, nullable=False, default=0), + Column("version_number", Integer, nullable=False, server_default="0"), ) batches = Table( diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 7e18dbed..e25220cb 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -20,4 +20,5 @@ def add(self, product): self.session.add(product) def get(self, sku): + print(sku, type(sku)) return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py index 3dff0b83..d51c20f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,12 +62,17 @@ def postgres_db(): @pytest.fixture -def postgres_session(postgres_db): +def postgres_session_factory(postgres_db): start_mappers() - yield sessionmaker(bind=postgres_db)() + yield sessionmaker(bind=postgres_db) clear_mappers() +@pytest.fixture +def postgres_session(postgres_session_factory): + return postgres_session_factory() + + @pytest.fixture def restart_api(): (Path(__file__).parent / "../src/allocation/entrypoints/flask_app.py").touch() diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 29b85761..79345fce 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -1,24 +1,8 @@ -import uuid import pytest import requests from allocation import config - - -def random_suffix(): - return uuid.uuid4().hex[:6] - - -def random_sku(name=""): - return f"sku-{name}-{random_suffix()}" - - -def random_batchref(name=""): - return f"batch-{name}-{random_suffix()}" - - -def random_orderid(name=""): - return f"order-{name}-{random_suffix()}" +from ..random_refs import random_sku, random_batchref, random_orderid def post_to_add_batch(ref, sku, qty, eta): diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 8bf0c69d..a95907cf 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,12 +1,18 @@ +# pylint: disable=broad-except +import threading +import time +import traceback +from typing import List import pytest from allocation.domain import model from allocation.service_layer import unit_of_work +from ..random_refs import random_sku, random_batchref, random_orderid -def insert_batch(session, ref, sku, qty, eta): +def insert_batch(session, ref, sku, qty, eta, product_version=1): session.execute( - "INSERT INTO products (sku) VALUES (:sku)", - dict(sku=sku), + "INSERT INTO products (sku, version_number) VALUES (:sku, :version)", + dict(sku=sku, version=product_version), ) session.execute( "INSERT INTO batches (reference, sku, _purchased_quantity, eta)" @@ -67,3 +73,53 @@ class MyException(Exception): new_session = session_factory() rows = list(new_session.execute('SELECT * FROM "batches"')) assert rows == [] + + +def try_to_allocate(orderid, sku, exceptions): + line = model.OrderLine(orderid, sku, 10) + try: + with unit_of_work.SqlAlchemyUnitOfWork() as uow: + product = uow.products.get(sku=sku) + product.allocate(line) + time.sleep(0.2) + uow.commit() + except Exception as e: + print(traceback.format_exc()) + exceptions.append(e) + + +def test_concurrent_updates_to_version_are_not_allowed(postgres_session_factory): + sku, batch = random_sku(), random_batchref() + session = postgres_session_factory() + insert_batch(session, batch, sku, 100, eta=None, product_version=1) + session.commit() + + order1, order2 = random_orderid(1), random_orderid(2) + exceptions = [] # type: List[Exception] + try_to_allocate_order1 = lambda: try_to_allocate(order1, sku, exceptions) + try_to_allocate_order2 = lambda: try_to_allocate(order2, sku, exceptions) + thread1 = threading.Thread(target=try_to_allocate_order1) + thread2 = threading.Thread(target=try_to_allocate_order2) + thread1.start() + thread2.start() + thread1.join() + thread2.join() + + [[version]] = session.execute( + "SELECT version_number FROM products WHERE sku=:sku", + dict(sku=sku), + ) + assert version == 2 + [exception] = exceptions + assert "could not serialize access due to concurrent update" in str(exception) + + orders = session.execute( + "SELECT orderid FROM allocations" + " JOIN batches ON allocations.batch_id = batches.id" + " JOIN order_lines ON allocations.orderline_id = order_lines.id" + " WHERE order_lines.sku=:sku", + dict(sku=sku), + ) + assert orders.rowcount == 1 + with unit_of_work.SqlAlchemyUnitOfWork() as uow: + uow.session.execute("select 1") diff --git a/tests/random_refs.py b/tests/random_refs.py new file mode 100644 index 00000000..096dd519 --- /dev/null +++ b/tests/random_refs.py @@ -0,0 +1,17 @@ +import uuid + + +def random_suffix(): + return uuid.uuid4().hex[:6] + + +def random_sku(name=""): + return f"sku-{name}-{random_suffix()}" + + +def random_batchref(name=""): + return f"batch-{name}-{random_suffix()}" + + +def random_orderid(name=""): + return f"order-{name}-{random_suffix()}" From bdd3d906711039eafa3e43f6456f2715fbf23805 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 23:40:06 +0000 Subject: [PATCH 042/145] select for update is one approach [with_for_update] --- src/allocation/adapters/repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index e25220cb..e08b4915 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -20,5 +20,9 @@ def add(self, product): self.session.add(product) def get(self, sku): - print(sku, type(sku)) - return self.session.query(model.Product).filter_by(sku=sku).first() + return ( + self.session.query(model.Product) + .filter_by(sku=sku) + .with_for_update() + .first() + ) From dfaa46ac4c6f1b0cef307d09d22dec9f71d6edb6 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 23:53:32 +0000 Subject: [PATCH 043/145] alternative: isolation=SERIALIZABLE [chapter_07_aggregate_ends] --- src/allocation/adapters/repository.py | 7 +------ src/allocation/service_layer/unit_of_work.py | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index e08b4915..7e18dbed 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -20,9 +20,4 @@ def add(self, product): self.session.add(product) def get(self, sku): - return ( - self.session.query(model.Product) - .filter_by(sku=sku) - .with_for_update() - .first() - ) + return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index 2b41b7a9..0c9a17e4 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -30,6 +30,7 @@ def rollback(self): DEFAULT_SESSION_FACTORY = sessionmaker( bind=create_engine( config.get_postgres_uri(), + isolation_level="REPEATABLE READ", ) ) From 5d5f194c33d4475e2be4dbb7722f28c34ff07bb9 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 10:59:34 +0100 Subject: [PATCH 044/145] Mocky test for email [mocky_test_for_send_email] --- src/allocation/adapters/email.py | 2 ++ tests/unit/test_services.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/allocation/adapters/email.py diff --git a/src/allocation/adapters/email.py b/src/allocation/adapters/email.py new file mode 100644 index 00000000..18b17d60 --- /dev/null +++ b/src/allocation/adapters/email.py @@ -0,0 +1,2 @@ +def send_mail(*args): + print("SENDING EMAIL:", *args) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 9055d03a..a5a07a29 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,5 +1,7 @@ +from unittest import mock import pytest from allocation.adapters import repository +from allocation.domain.model import OutOfStock from allocation.service_layer import services, unit_of_work @@ -60,3 +62,16 @@ def test_allocate_commits(): services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) services.allocate("o1", "OMINOUS-MIRROR", 10, uow) assert uow.committed + + +def test_sends_email_on_out_of_stock_error(): + uow = FakeUnitOfWork() + services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + + with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: + with pytest.raises(OutOfStock): + services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + assert mock_send_mail.call_args == mock.call( + "stock@made.com", + f"Out of stock for POPULAR-CURTAINS", + ) From 62aff873ead08c898b6404d7a3417ecb89658918 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 11:13:48 +0100 Subject: [PATCH 045/145] email in model [email_in_model] --- src/allocation/domain/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index a48e1f69..a9f03fd0 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set +from allocation.adapters import email class OutOfStock(Exception): @@ -21,6 +22,7 @@ def allocate(self, line: OrderLine) -> str: self.version_number += 1 return batch.reference except StopIteration: + email.send_mail("stock@made.com", f"Out of stock for {line.sku}") raise OutOfStock(f"Out of stock for sku {line.sku}") From 1a3b48d2b352ea6e2270e368ad62d446a91f8563 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 11:23:24 +0100 Subject: [PATCH 046/145] putting it in the services layer isn't lovely either [email_in_services] --- src/allocation/domain/model.py | 2 -- src/allocation/service_layer/services.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index a9f03fd0..a48e1f69 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,7 +2,6 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set -from allocation.adapters import email class OutOfStock(Exception): @@ -22,7 +21,6 @@ def allocate(self, line: OrderLine) -> str: self.version_number += 1 return batch.reference except StopIteration: - email.send_mail("stock@made.com", f"Out of stock for {line.sku}") raise OutOfStock(f"Out of stock for sku {line.sku}") diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 5311ca6e..622c8679 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -2,6 +2,7 @@ from typing import Optional from datetime import date +from allocation.adapters import email from allocation.domain import model from allocation.domain.model import OrderLine from allocation.service_layer import unit_of_work @@ -33,6 +34,10 @@ def allocate( product = uow.products.get(sku=line.sku) if product is None: raise InvalidSku(f"Invalid sku {line.sku}") - batchref = product.allocate(line) - uow.commit() - return batchref + try: + batchref = product.allocate(line) + uow.commit() + return batchref + except model.OutOfStock: + email.send_mail("stock@made.com", f"Out of stock for {line.sku}") + raise From 05f7256fc1993c266f1cb0c7e284b8001e0621ba Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 28 Mar 2019 15:45:20 +0000 Subject: [PATCH 047/145] first cut of out of stock event [domain_event] --- src/allocation/domain/events.py | 10 ++++++++++ src/allocation/domain/model.py | 6 +++++- tests/unit/test_product.py | 12 ++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 src/allocation/domain/events.py diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py new file mode 100644 index 00000000..332de931 --- /dev/null +++ b/src/allocation/domain/events.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +class Event: + pass + + +@dataclass +class OutOfStock(Event): + sku: str diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index a48e1f69..20c8238c 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set +from . import events class OutOfStock(Exception): @@ -13,6 +14,7 @@ def __init__(self, sku: str, batches: List[Batch], version_number: int = 0): self.sku = sku self.batches = batches self.version_number = version_number + self.events = [] # type: List[events.Event] def allocate(self, line: OrderLine) -> str: try: @@ -21,7 +23,9 @@ def allocate(self, line: OrderLine) -> str: self.version_number += 1 return batch.reference except StopIteration: - raise OutOfStock(f"Out of stock for sku {line.sku}") + self.events.append(events.OutOfStock(line.sku)) + # raise OutOfStock(f"Out of stock for sku {line.sku}") + return None @dataclass(unsafe_hash=True) diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index 05d75355..8fcd7ca3 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -1,5 +1,7 @@ from datetime import date, timedelta import pytest + +from allocation.domain import events from allocation.domain.model import Product, OrderLine, Batch, OutOfStock today = date.today() @@ -51,6 +53,16 @@ def test_raises_out_of_stock_exception_if_cannot_allocate(): product.allocate(OrderLine("order2", "SMALL-FORK", 1)) +def test_records_out_of_stock_event_if_cannot_allocate(): + sku1_batch = Batch("batch1", "sku1", 100, eta=today) + sku2_line = OrderLine("oref", "sku2", 10) + product = Product(sku="sku1", batches=[sku1_batch]) + + with pytest.raises(OutOfStock): + product.allocate(sku2_line) + assert product.events[-1] == events.OutOfStock(sku="sku2") + + def test_increments_version_number(): line = OrderLine("oref", "SCANDI-PEN", 10) product = Product( From ab4080ca3ed689b31a5c1dbb1bc9d8e821404593 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 10:14:18 +0100 Subject: [PATCH 048/145] remove commented-out line from model --- src/allocation/domain/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 20c8238c..159e2c75 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -24,7 +24,6 @@ def allocate(self, line: OrderLine) -> str: return batch.reference except StopIteration: self.events.append(events.OutOfStock(line.sku)) - # raise OutOfStock(f"Out of stock for sku {line.sku}") return None From 845286e3f655fc6eb55a4db49b40b0ebfeb3dc58 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 10:14:28 +0100 Subject: [PATCH 049/145] Adjust unit tests now we're no longer raising out of stock exception --- tests/unit/test_product.py | 22 ++++++---------------- tests/unit/test_services.py | 4 +--- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index 8fcd7ca3..1a1482b6 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -1,8 +1,7 @@ from datetime import date, timedelta -import pytest - from allocation.domain import events -from allocation.domain.model import Product, OrderLine, Batch, OutOfStock +from allocation.domain.model import Product, OrderLine, Batch + today = date.today() tomorrow = today + timedelta(days=1) @@ -44,23 +43,14 @@ def test_returns_allocated_batch_ref(): assert allocation == in_stock_batch.reference -def test_raises_out_of_stock_exception_if_cannot_allocate(): +def test_records_out_of_stock_event_if_cannot_allocate(): batch = Batch("batch1", "SMALL-FORK", 10, eta=today) product = Product(sku="SMALL-FORK", batches=[batch]) product.allocate(OrderLine("order1", "SMALL-FORK", 10)) - with pytest.raises(OutOfStock, match="SMALL-FORK"): - product.allocate(OrderLine("order2", "SMALL-FORK", 1)) - - -def test_records_out_of_stock_event_if_cannot_allocate(): - sku1_batch = Batch("batch1", "sku1", 100, eta=today) - sku2_line = OrderLine("oref", "sku2", 10) - product = Product(sku="sku1", batches=[sku1_batch]) - - with pytest.raises(OutOfStock): - product.allocate(sku2_line) - assert product.events[-1] == events.OutOfStock(sku="sku2") + allocation = product.allocate(OrderLine("order2", "SMALL-FORK", 1)) + assert product.events[-1] == events.OutOfStock(sku="SMALL-FORK") + assert allocation is None def test_increments_version_number(): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index a5a07a29..97db3230 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,7 +1,6 @@ from unittest import mock import pytest from allocation.adapters import repository -from allocation.domain.model import OutOfStock from allocation.service_layer import services, unit_of_work @@ -69,8 +68,7 @@ def test_sends_email_on_out_of_stock_error(): services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: - with pytest.raises(OutOfStock): - services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + services.allocate("o1", "POPULAR-CURTAINS", 10, uow) assert mock_send_mail.call_args == mock.call( "stock@made.com", f"Out of stock for POPULAR-CURTAINS", From 290914b7c1b3c20f79139aa3a9f4bb549ce467af Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 02:01:07 +0000 Subject: [PATCH 050/145] first cut of message bus [service_talks_to_messagebus] --- src/allocation/service_layer/messagebus.py | 20 ++++++++++++++++++++ src/allocation/service_layer/services.py | 13 +++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 src/allocation/service_layer/messagebus.py diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py new file mode 100644 index 00000000..497d555f --- /dev/null +++ b/src/allocation/service_layer/messagebus.py @@ -0,0 +1,20 @@ +from typing import List, Dict, Callable, Type +from allocation.adapters import email +from allocation.domain import events + + +def handle(event: events.Event): + for handler in HANDLERS[type(event)]: + handler(event) + + +def send_out_of_stock_notification(event: events.OutOfStock): + email.send_mail( + "stock@made.com", + f"Out of stock for {event.sku}", + ) + + +HANDLERS = { + events.OutOfStock: [send_out_of_stock_notification], +} # type: Dict[Type[events.Event], List[Callable]] diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 622c8679..3e66dd30 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -1,11 +1,13 @@ from __future__ import annotations -from typing import Optional +from typing import Optional, TYPE_CHECKING from datetime import date -from allocation.adapters import email from allocation.domain import model from allocation.domain.model import OrderLine -from allocation.service_layer import unit_of_work +from . import messagebus + +if TYPE_CHECKING: + from . import unit_of_work class InvalidSku(Exception): @@ -38,6 +40,5 @@ def allocate( batchref = product.allocate(line) uow.commit() return batchref - except model.OutOfStock: - email.send_mail("stock@made.com", f"Out of stock for {line.sku}") - raise + finally: + messagebus.handle(product.events) From e1faf88d80dd3698ec0d4bebc6abb397638a3632 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 12:57:11 +0100 Subject: [PATCH 051/145] uow now does messagebus magically. breaks tests [uow_has_messagebus] --- src/allocation/adapters/repository.py | 8 +++++++- src/allocation/service_layer/services.py | 10 +++------- src/allocation/service_layer/unit_of_work.py | 16 ++++++++++++++-- tests/unit/test_services.py | 2 +- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 7e18dbed..99d91284 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,4 +1,5 @@ import abc +from typing import Set from allocation.domain import model @@ -15,9 +16,14 @@ def get(self, sku) -> model.Product: class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): self.session = session + self.seen = set() # type: Set[model.Product] def add(self, product): + self.seen.add(product) self.session.add(product) def get(self, sku): - return self.session.query(model.Product).filter_by(sku=sku).first() + product = self.session.query(model.Product).filter_by(sku=sku).first() + if product: + self.seen.add(product) + return product diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 3e66dd30..0a935998 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -4,7 +4,6 @@ from allocation.domain import model from allocation.domain.model import OrderLine -from . import messagebus if TYPE_CHECKING: from . import unit_of_work @@ -36,9 +35,6 @@ def allocate( product = uow.products.get(sku=line.sku) if product is None: raise InvalidSku(f"Invalid sku {line.sku}") - try: - batchref = product.allocate(line) - uow.commit() - return batchref - finally: - messagebus.handle(product.events) + batchref = product.allocate(line) + uow.commit() + return batchref diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index 0c9a17e4..9a0a784c 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -5,8 +5,10 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.session import Session + from allocation import config from allocation.adapters import repository +from . import messagebus class AbstractUnitOfWork(abc.ABC): @@ -18,8 +20,18 @@ def __enter__(self) -> AbstractUnitOfWork: def __exit__(self, *args): self.rollback() - @abc.abstractmethod def commit(self): + self._commit() + self.publish_events() + + def publish_events(self): + for product in self.products.seen: + while product.events: + event = product.events.pop(0) + messagebus.handle(event) + + @abc.abstractmethod + def _commit(self): raise NotImplementedError @abc.abstractmethod @@ -48,7 +60,7 @@ def __exit__(self, *args): super().__exit__(*args) self.session.close() - def commit(self): + def _commit(self): self.session.commit() def rollback(self): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 97db3230..fde65a78 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -20,7 +20,7 @@ def __init__(self): self.products = FakeRepository([]) self.committed = False - def commit(self): + def _commit(self): self.committed = True def rollback(self): From 46c62d5d386b7888abaea65af3f07f0fbe247852 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 09:46:47 +0100 Subject: [PATCH 052/145] implement .seen on repository [repository_tracks_seen] --- src/allocation/adapters/repository.py | 29 ++++++++++++++++++--------- tests/unit/test_services.py | 5 +++-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 99d91284..07946f93 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -4,26 +4,35 @@ class AbstractRepository(abc.ABC): - @abc.abstractmethod + def __init__(self): + self.seen = set() # type: Set[model.Product] + def add(self, product: model.Product): + self._add(product) + self.seen.add(product) + + def get(self, sku) -> model.Product: + product = self._get(sku) + if product: + self.seen.add(product) + return product + + @abc.abstractmethod + def _add(self, product: model.Product): raise NotImplementedError @abc.abstractmethod - def get(self, sku) -> model.Product: + def _get(self, sku) -> model.Product: raise NotImplementedError class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): + super().__init__() self.session = session - self.seen = set() # type: Set[model.Product] - def add(self, product): - self.seen.add(product) + def _add(self, product): self.session.add(product) - def get(self, sku): - product = self.session.query(model.Product).filter_by(sku=sku).first() - if product: - self.seen.add(product) - return product + def _get(self, sku): + return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index fde65a78..0366fb2b 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -6,12 +6,13 @@ class FakeRepository(repository.AbstractRepository): def __init__(self, products): + super().__init__() self._products = set(products) - def add(self, product): + def _add(self, product): self._products.add(product) - def get(self, sku): + def _get(self, sku): return next((p for p in self._products if p.sku == sku), None) From 2c901784e5d498827262706069c91ae15a4a6551 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 13:07:55 +0100 Subject: [PATCH 053/145] a little hack in the orm so that events work --- src/allocation/adapters/orm.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index ddd1f3ab..ea76eed5 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -1,4 +1,13 @@ -from sqlalchemy import Table, MetaData, Column, Integer, String, Date, ForeignKey +from sqlalchemy import ( + Table, + MetaData, + Column, + Integer, + String, + Date, + ForeignKey, + event, +) from sqlalchemy.orm import mapper, relationship from allocation.domain import model @@ -55,5 +64,12 @@ def start_mappers(): }, ) mapper( - model.Product, products, properties={"batches": relationship(batches_mapper)} + model.Product, + products, + properties={"batches": relationship(batches_mapper)}, ) + + +@event.listens_for(model.Product, "load") +def receive_load(product, _): + product.events = [] From 1ba5848e49aabead9626c1aeef085a116790fe15 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 10:21:30 +0100 Subject: [PATCH 054/145] remove now unused out-of-stock exception [chapter_08_events_and_message_bus_ends] --- src/allocation/domain/model.py | 4 ---- src/allocation/entrypoints/flask_app.py | 5 +---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 159e2c75..02e91301 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -5,10 +5,6 @@ from . import events -class OutOfStock(Exception): - pass - - class Product: def __init__(self, sku: str, batches: List[Batch], version_number: int = 0): self.sku = sku diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 602a09c1..f30bd710 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,9 +1,6 @@ from datetime import datetime from flask import Flask, request -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from allocation.domain import model from allocation.adapters import orm from allocation.service_layer import services, unit_of_work @@ -35,7 +32,7 @@ def allocate_endpoint(): request.json["qty"], unit_of_work.SqlAlchemyUnitOfWork(), ) - except (model.OutOfStock, services.InvalidSku) as e: + except services.InvalidSku as e: return {"message": str(e)}, 400 return {"batchref": batchref}, 201 From 28b604996600e00ad6c96b4186f1a2a6c398471e Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 26 May 2019 21:29:23 +0100 Subject: [PATCH 055/145] allocationrequired and batchcreated events [two_new_events] --- src/allocation/domain/events.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index 332de931..5ea87bba 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -1,10 +1,28 @@ +# pylint: disable=too-few-public-methods from dataclasses import dataclass +from datetime import date +from typing import Optional class Event: pass +@dataclass +class BatchCreated(Event): + ref: str + sku: str + qty: int + eta: Optional[date] = None + + +@dataclass +class AllocationRequired(Event): + orderid: str + sku: str + qty: int + + @dataclass class OutOfStock(Event): sku: str From 3c45aa397f97f5459857834c1be528a6509c9ab9 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 02:58:58 +0000 Subject: [PATCH 056/145] change exception import in flask app --- src/allocation/entrypoints/flask_app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index f30bd710..b6b2f3f9 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -3,6 +3,7 @@ from allocation.adapters import orm from allocation.service_layer import services, unit_of_work +from allocation.service_layer.services import InvalidSku app = Flask(__name__) orm.start_mappers() @@ -32,7 +33,7 @@ def allocate_endpoint(): request.json["qty"], unit_of_work.SqlAlchemyUnitOfWork(), ) - except services.InvalidSku as e: + except InvalidSku as e: return {"message": str(e)}, 400 return {"batchref": batchref}, 201 From 6718fb2cd4a49cccb95fdb1f1005bb9e4c9f8f16 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 26 May 2019 21:45:42 +0100 Subject: [PATCH 057/145] move services.py to handlers.py rename test services to test handlers --- src/allocation/service_layer/{services.py => handlers.py} | 0 tests/unit/{test_services.py => test_handlers.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/allocation/service_layer/{services.py => handlers.py} (100%) rename tests/unit/{test_services.py => test_handlers.py} (100%) diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/handlers.py similarity index 100% rename from src/allocation/service_layer/services.py rename to src/allocation/service_layer/handlers.py diff --git a/tests/unit/test_services.py b/tests/unit/test_handlers.py similarity index 100% rename from tests/unit/test_services.py rename to tests/unit/test_handlers.py From 94c0c0c984423cdaf70e58361c55b1d3f2a555c3 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 13:50:45 +0100 Subject: [PATCH 058/145] Start moving to handlers [services_to_handlers] --- src/allocation/adapters/email.py | 2 +- src/allocation/service_layer/handlers.py | 33 +++++++++++++++------- src/allocation/service_layer/messagebus.py | 13 +++------ 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/allocation/adapters/email.py b/src/allocation/adapters/email.py index 18b17d60..1c37d427 100644 --- a/src/allocation/adapters/email.py +++ b/src/allocation/adapters/email.py @@ -1,2 +1,2 @@ -def send_mail(*args): +def send(*args): print("SENDING EMAIL:", *args) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 0a935998..7cbe1e8b 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import Optional, TYPE_CHECKING -from datetime import date - -from allocation.domain import model +from typing import TYPE_CHECKING +from allocation.domain import events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: @@ -14,23 +12,25 @@ class InvalidSku(Exception): def add_batch( - ref: str, sku: str, qty: int, eta: Optional[date], + event: events.BatchCreated, uow: unit_of_work.AbstractUnitOfWork, ): with uow: - product = uow.products.get(sku=sku) + product = uow.products.get(sku=event.sku) if product is None: - product = model.Product(sku, batches=[]) + product = model.Product(event.sku, batches=[]) uow.products.add(product) - product.batches.append(model.Batch(ref, sku, qty, eta)) + product.batches.append( + model.Batch(event.ref, event.sku, event.qty, event.eta) + ) uow.commit() def allocate( - orderid: str, sku: str, qty: int, + event: events.AllocationRequired, uow: unit_of_work.AbstractUnitOfWork, ) -> str: - line = OrderLine(orderid, sku, qty) + line = OrderLine(event.orderid, event.sku, event.qty) with uow: product = uow.products.get(sku=line.sku) if product is None: @@ -38,3 +38,16 @@ def allocate( batchref = product.allocate(line) uow.commit() return batchref + + +# pylint: disable=unused-argument + + +def send_out_of_stock_notification( + event: events.OutOfStock, + uow: unit_of_work.AbstractUnitOfWork, +): + email.send( + "stock@made.com", + f"Out of stock for {event.sku}", + ) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 497d555f..d1d5997b 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,6 +1,6 @@ from typing import List, Dict, Callable, Type -from allocation.adapters import email from allocation.domain import events +from . import handlers def handle(event: events.Event): @@ -8,13 +8,8 @@ def handle(event: events.Event): handler(event) -def send_out_of_stock_notification(event: events.OutOfStock): - email.send_mail( - "stock@made.com", - f"Out of stock for {event.sku}", - ) - - HANDLERS = { - events.OutOfStock: [send_out_of_stock_notification], + events.BatchCreated: [handlers.add_batch], + events.AllocationRequired: [handlers.allocate], + events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From c0a89dd6b23dcb6ff483c218d16133526684f3dd Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Feb 2020 12:02:51 +0000 Subject: [PATCH 059/145] use classes in services/handlers test [tests_use_classes] --- tests/unit/test_handlers.py | 91 ++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 0366fb2b..aca2180b 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,3 +1,4 @@ +# pylint: disable=no-self-use from unittest import mock import pytest from allocation.adapters import repository @@ -28,49 +29,47 @@ def rollback(self): pass -def test_add_batch_for_new_product(): - uow = FakeUnitOfWork() - services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) - assert uow.products.get("CRUNCHY-ARMCHAIR") is not None - assert uow.committed - - -def test_add_batch_for_existing_product(): - uow = FakeUnitOfWork() - services.add_batch("b1", "GARISH-RUG", 100, None, uow) - services.add_batch("b2", "GARISH-RUG", 99, None, uow) - assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] - - -def test_allocate_returns_allocation(): - uow = FakeUnitOfWork() - services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) - assert result == "batch1" - - -def test_allocate_errors_for_invalid_sku(): - uow = FakeUnitOfWork() - services.add_batch("b1", "AREALSKU", 100, None, uow) - - with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate("o1", "NONEXISTENTSKU", 10, uow) - - -def test_allocate_commits(): - uow = FakeUnitOfWork() - services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) - services.allocate("o1", "OMINOUS-MIRROR", 10, uow) - assert uow.committed - - -def test_sends_email_on_out_of_stock_error(): - uow = FakeUnitOfWork() - services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) - - with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: - services.allocate("o1", "POPULAR-CURTAINS", 10, uow) - assert mock_send_mail.call_args == mock.call( - "stock@made.com", - f"Out of stock for POPULAR-CURTAINS", - ) +class TestAddBatch: + def test_for_new_product(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) + assert uow.products.get("CRUNCHY-ARMCHAIR") is not None + assert uow.committed + + def test_for_existing_product(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "GARISH-RUG", 100, None, uow) + services.add_batch("b2", "GARISH-RUG", 99, None, uow) + assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] + + +class TestAllocate: + def test_returns_allocation(self): + uow = FakeUnitOfWork() + services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) + result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) + assert result == "batch1" + + def test_errors_for_invalid_sku(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "AREALSKU", 100, None, uow) + + with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): + services.allocate("o1", "NONEXISTENTSKU", 10, uow) + + def test_commits(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) + services.allocate("o1", "OMINOUS-MIRROR", 10, uow) + assert uow.committed + + def test_sends_email_on_out_of_stock_error(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + + with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: + services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + assert mock_send_mail.call_args == mock.call( + "stock@made.com", + f"Out of stock for POPULAR-CURTAINS", + ) From 5647d0f2909c34c40b3fbd3a20c0d823f7962e8c Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Feb 2020 12:08:05 +0000 Subject: [PATCH 060/145] tests change to use bus [handler_tests] --- tests/unit/test_handlers.py | 43 +++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index aca2180b..bb041b94 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,8 +1,10 @@ # pylint: disable=no-self-use from unittest import mock import pytest + from allocation.adapters import repository -from allocation.service_layer import services, unit_of_work +from allocation.domain import events +from allocation.service_layer import handlers, messagebus, unit_of_work class FakeRepository(repository.AbstractRepository): @@ -32,44 +34,53 @@ def rollback(self): class TestAddBatch: def test_for_new_product(self): uow = FakeUnitOfWork() - services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) + messagebus.handle( + events.BatchCreated("b1", "CRUNCHY-ARMCHAIR", 100, None), uow + ) assert uow.products.get("CRUNCHY-ARMCHAIR") is not None assert uow.committed def test_for_existing_product(self): uow = FakeUnitOfWork() - services.add_batch("b1", "GARISH-RUG", 100, None, uow) - services.add_batch("b2", "GARISH-RUG", 99, None, uow) + messagebus.handle(events.BatchCreated("b1", "GARISH-RUG", 100, None), uow) + messagebus.handle(events.BatchCreated("b2", "GARISH-RUG", 99, None), uow) assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] class TestAllocate: def test_returns_allocation(self): uow = FakeUnitOfWork() - services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) + messagebus.handle( + events.BatchCreated("batch1", "COMPLICATED-LAMP", 100, None), uow + ) + result = messagebus.handle( + events.AllocationRequired("o1", "COMPLICATED-LAMP", 10), uow + ) assert result == "batch1" def test_errors_for_invalid_sku(self): uow = FakeUnitOfWork() - services.add_batch("b1", "AREALSKU", 100, None, uow) + messagebus.handle(events.BatchCreated("b1", "AREALSKU", 100, None), uow) - with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate("o1", "NONEXISTENTSKU", 10, uow) + with pytest.raises(handlers.InvalidSku, match="Invalid sku NONEXISTENTSKU"): + messagebus.handle( + events.AllocationRequired("o1", "NONEXISTENTSKU", 10), uow + ) def test_commits(self): uow = FakeUnitOfWork() - services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) - services.allocate("o1", "OMINOUS-MIRROR", 10, uow) + messagebus.handle(events.BatchCreated("b1", "OMINOUS-MIRROR", 100, None), uow) + messagebus.handle(events.AllocationRequired("o1", "OMINOUS-MIRROR", 10), uow) assert uow.committed def test_sends_email_on_out_of_stock_error(self): uow = FakeUnitOfWork() - services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + messagebus.handle(events.BatchCreated("b1", "POPULAR-CURTAINS", 9, None), uow) - with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: - services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + with mock.patch("allocation.adapters.email.send") as mock_send_mail: + messagebus.handle( + events.AllocationRequired("o1", "POPULAR-CURTAINS", 10), uow + ) assert mock_send_mail.call_args == mock.call( - "stock@made.com", - f"Out of stock for POPULAR-CURTAINS", + "stock@made.com", f"Out of stock for POPULAR-CURTAINS" ) From ea72d42a63908d8d4a2299083b720b512f3b9569 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 02:46:32 +0000 Subject: [PATCH 061/145] messagebus takes a uow [handle_takes_uow] --- src/allocation/service_layer/messagebus.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index d1d5997b..cd9ba7ef 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,11 +1,18 @@ -from typing import List, Dict, Callable, Type +from __future__ import annotations +from typing import List, Dict, Callable, Type, TYPE_CHECKING from allocation.domain import events from . import handlers +if TYPE_CHECKING: + from allocation.service_layer import unit_of_work -def handle(event: events.Event): + +def handle( + event: events.Event, + uow: unit_of_work.AbstractUnitOfWork, +): for handler in HANDLERS[type(event)]: - handler(event) + handler(event, uow=uow) HANDLERS = { From b7c6e7e42addde1014a4a9ec2afa5e0a1782456e Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 22:07:35 +0000 Subject: [PATCH 062/145] messagebus has uow, manages queue [handle_has_uow_and_queue] --- src/allocation/service_layer/messagebus.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index cd9ba7ef..23b6c49e 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -4,15 +4,19 @@ from . import handlers if TYPE_CHECKING: - from allocation.service_layer import unit_of_work + from . import unit_of_work def handle( event: events.Event, uow: unit_of_work.AbstractUnitOfWork, ): - for handler in HANDLERS[type(event)]: - handler(event, uow=uow) + queue = [event] + while queue: + event = queue.pop(0) + for handler in HANDLERS[type(event)]: + handler(event, uow=uow) + queue.extend(uow.collect_new_events()) HANDLERS = { From b40f8ed0325902b5531354ebbe0d84232cbd99de Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 22:08:03 +0000 Subject: [PATCH 063/145] Uow no longer puts events directly on the bus [uow_collect_new_events] --- src/allocation/service_layer/unit_of_work.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index 9a0a784c..586ae027 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -8,7 +8,6 @@ from allocation import config from allocation.adapters import repository -from . import messagebus class AbstractUnitOfWork(abc.ABC): @@ -22,13 +21,11 @@ def __exit__(self, *args): def commit(self): self._commit() - self.publish_events() - def publish_events(self): + def collect_new_events(self): for product in self.products.seen: while product.events: - event = product.events.pop(0) - messagebus.handle(event) + yield product.events.pop(0) @abc.abstractmethod def _commit(self): From 8f3e8b217467e4e6902966790b874c36ccd2a95e Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 12:57:30 +0100 Subject: [PATCH 064/145] Ugly hack for messagebus to return results [hack_messagebus_results] --- src/allocation/service_layer/messagebus.py | 4 +++- tests/unit/test_handlers.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 23b6c49e..9c0217fc 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -11,12 +11,14 @@ def handle( event: events.Event, uow: unit_of_work.AbstractUnitOfWork, ): + results = [] queue = [event] while queue: event = queue.pop(0) for handler in HANDLERS[type(event)]: - handler(event, uow=uow) + results.append(handler(event, uow=uow)) queue.extend(uow.collect_new_events()) + return results HANDLERS = { diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index bb041b94..fea84fd7 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -53,10 +53,10 @@ def test_returns_allocation(self): messagebus.handle( events.BatchCreated("batch1", "COMPLICATED-LAMP", 100, None), uow ) - result = messagebus.handle( + results = messagebus.handle( events.AllocationRequired("o1", "COMPLICATED-LAMP", 10), uow ) - assert result == "batch1" + assert results.pop(0) == "batch1" def test_errors_for_invalid_sku(self): uow = FakeUnitOfWork() From 1f4068237964d5fa46ea42fa7cbe2dbdc434a98e Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 12:33:23 +0100 Subject: [PATCH 065/145] modify flask to use messagebus [flask_uses_messagebus] --- src/allocation/entrypoints/flask_app.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index b6b2f3f9..51922760 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,9 +1,10 @@ from datetime import datetime from flask import Flask, request +from allocation.domain import events from allocation.adapters import orm -from allocation.service_layer import services, unit_of_work -from allocation.service_layer.services import InvalidSku +from allocation.service_layer import messagebus, unit_of_work +from allocation.service_layer.handlers import InvalidSku app = Flask(__name__) orm.start_mappers() @@ -14,25 +15,21 @@ def add_batch(): eta = request.json["eta"] if eta is not None: eta = datetime.fromisoformat(eta).date() - services.add_batch( - request.json["ref"], - request.json["sku"], - request.json["qty"], - eta, - unit_of_work.SqlAlchemyUnitOfWork(), + event = events.BatchCreated( + request.json["ref"], request.json["sku"], request.json["qty"], eta ) + messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) return "OK", 201 @app.route("/allocate", methods=["POST"]) def allocate_endpoint(): try: - batchref = services.allocate( - request.json["orderid"], - request.json["sku"], - request.json["qty"], - unit_of_work.SqlAlchemyUnitOfWork(), + event = events.AllocationRequired( + request.json["orderid"], request.json["sku"], request.json["qty"] ) + results = messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) + batchref = results.pop(0) except InvalidSku as e: return {"message": str(e)}, 400 From e4e48bd28d10a1ea32906fbe5fcf959d4d30c945 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 13:27:32 +0100 Subject: [PATCH 066/145] new event for new input [batch_quantity_changed_event] --- src/allocation/domain/events.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index 5ea87bba..135ae74b 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -16,6 +16,12 @@ class BatchCreated(Event): eta: Optional[date] = None +@dataclass +class BatchQuantityChanged(Event): + ref: str + qty: int + + @dataclass class AllocationRequired(Event): orderid: str From d13475bfa2cb9e18f82d6a27c26f04815f13aa2f Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 25 May 2019 10:11:42 +0100 Subject: [PATCH 067/145] new test and put them into classes [test_change_batch_quantity_handler] --- tests/unit/test_handlers.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index fea84fd7..0ac97047 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,4 +1,5 @@ # pylint: disable=no-self-use +from datetime import date from unittest import mock import pytest @@ -84,3 +85,38 @@ def test_sends_email_on_out_of_stock_error(self): assert mock_send_mail.call_args == mock.call( "stock@made.com", f"Out of stock for POPULAR-CURTAINS" ) + + +class TestChangeBatchQuantity: + def test_changes_available_quantity(self): + uow = FakeUnitOfWork() + messagebus.handle( + events.BatchCreated("batch1", "ADORABLE-SETTEE", 100, None), uow + ) + [batch] = uow.products.get(sku="ADORABLE-SETTEE").batches + assert batch.available_quantity == 100 + + messagebus.handle(events.BatchQuantityChanged("batch1", 50), uow) + + assert batch.available_quantity == 50 + + def test_reallocates_if_necessary(self): + uow = FakeUnitOfWork() + event_history = [ + events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), + events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), + events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), + events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), + ] + for e in event_history: + messagebus.handle(e, uow) + [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches + assert batch1.available_quantity == 10 + assert batch2.available_quantity == 50 + + messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) + + # order1 or order2 will be deallocated, so we'll have 25 - 20 + assert batch1.available_quantity == 5 + # and 20 will be reallocated to the next batch + assert batch2.available_quantity == 30 From 1c365c798472560d97a3b3d0dc5ded96cabf0a7e Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 25 May 2019 12:14:35 +0100 Subject: [PATCH 068/145] start on handler for change quantity [change_quantity_handler] --- src/allocation/service_layer/handlers.py | 11 +++++++++++ src/allocation/service_layer/messagebus.py | 1 + 2 files changed, 12 insertions(+) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 7cbe1e8b..957aacfe 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING +from allocation.adapters import email from allocation.domain import events, model from allocation.domain.model import OrderLine @@ -40,6 +41,16 @@ def allocate( return batchref +def change_batch_quantity( + event: events.BatchQuantityChanged, + uow: unit_of_work.AbstractUnitOfWork, +): + with uow: + product = uow.products.get_by_batchref(batchref=event.ref) + product.change_batch_quantity(ref=event.ref, qty=event.qty) + uow.commit() + + # pylint: disable=unused-argument diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 9c0217fc..b5b72844 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -23,6 +23,7 @@ def handle( HANDLERS = { events.BatchCreated: [handlers.add_batch], + events.BatchQuantityChanged: [handlers.change_batch_quantity], events.AllocationRequired: [handlers.allocate], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From 71a6fa34af07669df4814dead1e3731ef65297f7 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 28 May 2019 10:40:46 +0100 Subject: [PATCH 069/145] get_by_batchref in abstract+real repo, and a new integration test for it [get_by_batchref] --- src/allocation/adapters/repository.py | 19 +++++++++++++++++++ tests/integration/test_repository.py | 15 +++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/integration/test_repository.py diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 07946f93..f8821758 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,5 +1,6 @@ import abc from typing import Set +from allocation.adapters import orm from allocation.domain import model @@ -17,6 +18,12 @@ def get(self, sku) -> model.Product: self.seen.add(product) return product + def get_by_batchref(self, batchref) -> model.Product: + product = self._get_by_batchref(batchref) + if product: + self.seen.add(product) + return product + @abc.abstractmethod def _add(self, product: model.Product): raise NotImplementedError @@ -25,6 +32,10 @@ def _add(self, product: model.Product): def _get(self, sku) -> model.Product: raise NotImplementedError + @abc.abstractmethod + def _get_by_batchref(self, batchref) -> model.Product: + raise NotImplementedError + class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): @@ -36,3 +47,11 @@ def _add(self, product): def _get(self, sku): return self.session.query(model.Product).filter_by(sku=sku).first() + + def _get_by_batchref(self, batchref): + return ( + self.session.query(model.Product) + .join(model.Batch) + .filter(orm.batches.c.reference == batchref) + .first() + ) diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py new file mode 100644 index 00000000..a73bcd51 --- /dev/null +++ b/tests/integration/test_repository.py @@ -0,0 +1,15 @@ +from allocation.adapters import repository +from allocation.domain import model + + +def test_get_by_batchref(session): + repo = repository.SqlAlchemyRepository(session) + b1 = model.Batch(ref="b1", sku="sku1", qty=100, eta=None) + b2 = model.Batch(ref="b2", sku="sku1", qty=100, eta=None) + b3 = model.Batch(ref="b3", sku="sku2", qty=100, eta=None) + p1 = model.Product(sku="sku1", batches=[b1, b2]) + p2 = model.Product(sku="sku2", batches=[b3]) + repo.add(p1) + repo.add(p2) + assert repo.get_by_batchref("b2") == p1 + assert repo.get_by_batchref("b3") == p2 From 8f67133de3e3dd94ede7df2bcb5ce4aa024fdaba Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 28 May 2019 10:41:12 +0100 Subject: [PATCH 070/145] fake repo get_by_batchref [fakerepo_get_by_batchref] --- tests/unit/test_handlers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 0ac97047..42b5b895 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -19,6 +19,12 @@ def _add(self, product): def _get(self, sku): return next((p for p in self._products if p.sku == sku), None) + def _get_by_batchref(self, batchref): + return next( + (p for p in self._products for b in p.batches if b.reference == batchref), + None, + ) + class FakeUnitOfWork(unit_of_work.AbstractUnitOfWork): def __init__(self): From be5b945aa5541b23346d1321d6db94e72adc2068 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 25 May 2019 18:01:26 +0100 Subject: [PATCH 071/145] change_batch_quantity on product, needed change to batch, also emit allocated event. [change_batch_model_layer] --- src/allocation/domain/model.py | 14 +++++++++++--- tests/unit/test_batches.py | 13 ------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 02e91301..7aa99417 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -22,6 +22,15 @@ def allocate(self, line: OrderLine) -> str: self.events.append(events.OutOfStock(line.sku)) return None + def change_batch_quantity(self, ref: str, qty: int): + batch = next(b for b in self.batches if b.reference == ref) + batch._purchased_quantity = qty + while batch.available_quantity < 0: + line = batch.deallocate_one() + self.events.append( + events.AllocationRequired(line.orderid, line.sku, line.qty) + ) + @dataclass(unsafe_hash=True) class OrderLine: @@ -60,9 +69,8 @@ def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) - def deallocate(self, line: OrderLine): - if line in self._allocations: - self._allocations.remove(line) + def deallocate_one(self) -> OrderLine: + return self._allocations.pop() @property def allocated_quantity(self) -> int: diff --git a/tests/unit/test_batches.py b/tests/unit/test_batches.py index 8735f41e..d6464b11 100644 --- a/tests/unit/test_batches.py +++ b/tests/unit/test_batches.py @@ -44,16 +44,3 @@ def test_allocation_is_idempotent(): batch.allocate(line) batch.allocate(line) assert batch.available_quantity == 18 - - -def test_deallocate(): - batch, line = make_batch_and_line("EXPENSIVE-FOOTSTOOL", 20, 2) - batch.allocate(line) - batch.deallocate(line) - assert batch.available_quantity == 20 - - -def test_can_only_deallocate_allocated_lines(): - batch, unallocated_line = make_batch_and_line("DECORATIVE-TRINKET", 20, 2) - batch.deallocate(unallocated_line) - assert batch.available_quantity == 20 From 1cad6e8a2f2c936700dec65ea7c51b731c8e2514 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 29 Oct 2019 12:37:46 +0000 Subject: [PATCH 072/145] sort-of fake messagebus on fake-uow. [fake_messagebus] --- tests/unit/test_handlers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 42b5b895..c887c84c 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,6 +1,7 @@ # pylint: disable=no-self-use from datetime import date from unittest import mock +from typing import List import pytest from allocation.adapters import repository @@ -126,3 +127,14 @@ def test_reallocates_if_necessary(self): assert batch1.available_quantity == 5 # and 20 will be reallocated to the next batch assert batch2.available_quantity == 30 + + +class FakeUnitOfWorkWithFakeMessageBus(FakeUnitOfWork): + def __init__(self): + super().__init__() + self.events_published = [] # type: List[events.Event] + + def publish_events(self): + for product in self.products.seen: + while product.events: + self.events_published.append(product.events.pop(0)) From 68ce4019440c7b86ebc0329737782a000e645f52 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 29 Oct 2019 12:39:06 +0000 Subject: [PATCH 073/145] isolated test for a handler [test_handler_in_isolation] --- tests/unit/test_handlers.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index c887c84c..4b6c1540 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -138,3 +138,28 @@ def publish_events(self): for product in self.products.seen: while product.events: self.events_published.append(product.events.pop(0)) + + +def test_reallocates_if_necessary_isolated(): + uow = FakeUnitOfWorkWithFakeMessageBus() + + # test setup as before + event_history = [ + events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), + events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), + events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), + events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), + ] + for e in event_history: + messagebus.handle(e, uow) + [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches + assert batch1.available_quantity == 10 + assert batch2.available_quantity == 50 + + messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) + + # assert on new events emitted rather than downstream side-effects + [reallocation_event] = uow.events_published + assert isinstance(reallocation_event, events.AllocationRequired) + assert reallocation_event.orderid in {"order1", "order2"} + assert reallocation_event.sku == "INDIFFERENT-TABLE" From 6e6937ca5d7a77a0e9e402298a9b7850a848aa0f Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 29 Oct 2019 12:40:06 +0000 Subject: [PATCH 074/145] Revert fake messagebus stuff [chapter_09_all_messagebus_ends] This reverts commit 8f2c36f43fbbc9b4cfa42f9ee75acb4f486d4342. This reverts commit d3988118e478db2748d7b41d3ad70c6a22ac9f9f. --- tests/unit/test_handlers.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 4b6c1540..42b5b895 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,7 +1,6 @@ # pylint: disable=no-self-use from datetime import date from unittest import mock -from typing import List import pytest from allocation.adapters import repository @@ -127,39 +126,3 @@ def test_reallocates_if_necessary(self): assert batch1.available_quantity == 5 # and 20 will be reallocated to the next batch assert batch2.available_quantity == 30 - - -class FakeUnitOfWorkWithFakeMessageBus(FakeUnitOfWork): - def __init__(self): - super().__init__() - self.events_published = [] # type: List[events.Event] - - def publish_events(self): - for product in self.products.seen: - while product.events: - self.events_published.append(product.events.pop(0)) - - -def test_reallocates_if_necessary_isolated(): - uow = FakeUnitOfWorkWithFakeMessageBus() - - # test setup as before - event_history = [ - events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), - events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), - events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), - events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), - ] - for e in event_history: - messagebus.handle(e, uow) - [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches - assert batch1.available_quantity == 10 - assert batch2.available_quantity == 50 - - messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) - - # assert on new events emitted rather than downstream side-effects - [reallocation_event] = uow.events_published - assert isinstance(reallocation_event, events.AllocationRequired) - assert reallocation_event.orderid in {"order1", "order2"} - assert reallocation_event.sku == "INDIFFERENT-TABLE" From bfbee2e9a3ae7e39a8d1ab3f3640e267a5850928 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:12:54 +0100 Subject: [PATCH 075/145] add commands [commands_dot_py] --- src/allocation/domain/commands.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/allocation/domain/commands.py diff --git a/src/allocation/domain/commands.py b/src/allocation/domain/commands.py new file mode 100644 index 00000000..5473eff4 --- /dev/null +++ b/src/allocation/domain/commands.py @@ -0,0 +1,29 @@ +# pylint: disable=too-few-public-methods +from datetime import date +from typing import Optional +from dataclasses import dataclass + + +class Command: + pass + + +@dataclass +class Allocate(Command): + orderid: str + sku: str + qty: int + + +@dataclass +class CreateBatch(Command): + ref: str + sku: str + qty: int + eta: Optional[date] = None + + +@dataclass +class ChangeBatchQuantity(Command): + ref: str + qty: int From 6157d486314c64cc37c78e47d61eff15383f4470 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 12:44:24 +0000 Subject: [PATCH 076/145] top-level messagebus.handle dispatches events vs comannds [messagebus_dispatches_differently] --- src/allocation/service_layer/messagebus.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index b5b72844..0765ee06 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,23 +1,33 @@ +# pylint: disable=bare-except from __future__ import annotations -from typing import List, Dict, Callable, Type, TYPE_CHECKING -from allocation.domain import events +import logging +from typing import List, Dict, Callable, Type, Union, TYPE_CHECKING +from allocation.domain import commands, events from . import handlers if TYPE_CHECKING: from . import unit_of_work +logger = logging.getLogger(__name__) + +Message = Union[commands.Command, events.Event] + def handle( - event: events.Event, + message: Message, uow: unit_of_work.AbstractUnitOfWork, ): results = [] - queue = [event] + queue = [message] while queue: - event = queue.pop(0) - for handler in HANDLERS[type(event)]: - results.append(handler(event, uow=uow)) - queue.extend(uow.collect_new_events()) + message = queue.pop(0) + if isinstance(message, events.Event): + handle_event(message, queue, uow) + elif isinstance(message, commands.Command): + cmd_result = handle_command(message, queue, uow) + results.append(cmd_result) + else: + raise Exception(f"{message} was not an Event or Command") return results From c31d7651a8edb9bb2b427f2478436c1ccc512817 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 14:33:15 +0000 Subject: [PATCH 077/145] subhandler for events [handle_event] --- src/allocation/service_layer/messagebus.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 0765ee06..f6f44c17 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -31,6 +31,21 @@ def handle( return results +def handle_event( + event: events.Event, + queue: List[Message], + uow: unit_of_work.AbstractUnitOfWork, +): + for handler in EVENT_HANDLERS[type(event)]: + try: + logger.debug("handling event %s with handler %s", event, handler) + handler(event, uow=uow) + queue.extend(uow.collect_new_events()) + except Exception: + logger.exception("Exception handling event %s", event) + continue + + HANDLERS = { events.BatchCreated: [handlers.add_batch], events.BatchQuantityChanged: [handlers.change_batch_quantity], From de1a7b4454ce5ba60ab5d73dbf78759f314b88ad Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 14:33:45 +0000 Subject: [PATCH 078/145] subhandler for commands [handle_command] --- src/allocation/service_layer/messagebus.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index f6f44c17..e1bb4589 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,4 +1,4 @@ -# pylint: disable=bare-except +# pylint: disable=broad-except from __future__ import annotations import logging from typing import List, Dict, Callable, Type, Union, TYPE_CHECKING @@ -46,6 +46,22 @@ def handle_event( continue +def handle_command( + command: commands.Command, + queue: List[Message], + uow: unit_of_work.AbstractUnitOfWork, +): + logger.debug("handling command %s", command) + try: + handler = COMMAND_HANDLERS[type(command)] + result = handler(command, uow=uow) + queue.extend(uow.collect_new_events()) + return result + except Exception: + logger.exception("Exception handling command %s", command) + raise + + HANDLERS = { events.BatchCreated: [handlers.add_batch], events.BatchQuantityChanged: [handlers.change_batch_quantity], From 3a18a9d86b3d725bffcededc68ec501115ddebc8 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 12:45:59 +0000 Subject: [PATCH 079/145] handlers dicts split in two [two_hander_dicts] --- src/allocation/service_layer/messagebus.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index e1bb4589..8bac77a2 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -62,9 +62,12 @@ def handle_command( raise -HANDLERS = { - events.BatchCreated: [handlers.add_batch], - events.BatchQuantityChanged: [handlers.change_batch_quantity], - events.AllocationRequired: [handlers.allocate], +EVENT_HANDLERS = { events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] + +COMMAND_HANDLERS = { + commands.Allocate: handlers.allocate, + commands.CreateBatch: handlers.add_batch, + commands.ChangeBatchQuantity: handlers.change_batch_quantity, +} # type: Dict[Type[commands.Command], Callable] From 8fdf1197f5904d674b73a701c0e88ebff3dd5357 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:32:38 +0100 Subject: [PATCH 080/145] remove old events --- src/allocation/domain/events.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index 135ae74b..ba90ea05 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -1,34 +1,11 @@ # pylint: disable=too-few-public-methods from dataclasses import dataclass -from datetime import date -from typing import Optional class Event: pass -@dataclass -class BatchCreated(Event): - ref: str - sku: str - qty: int - eta: Optional[date] = None - - -@dataclass -class BatchQuantityChanged(Event): - ref: str - qty: int - - -@dataclass -class AllocationRequired(Event): - orderid: str - sku: str - qty: int - - @dataclass class OutOfStock(Event): sku: str From 1d2b68c75b1256b6a06e583aabe9087fd4288ae9 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:37:40 +0100 Subject: [PATCH 081/145] handlers take commands now, modify tests too --- src/allocation/service_layer/handlers.py | 25 +++++------ tests/unit/test_handlers.py | 57 ++++++++++++------------ 2 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 957aacfe..8c7b23d5 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from allocation.adapters import email -from allocation.domain import events, model +from allocation.domain import commands, events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: @@ -13,25 +13,23 @@ class InvalidSku(Exception): def add_batch( - event: events.BatchCreated, + cmd: commands.CreateBatch, uow: unit_of_work.AbstractUnitOfWork, ): with uow: - product = uow.products.get(sku=event.sku) + product = uow.products.get(sku=cmd.sku) if product is None: - product = model.Product(event.sku, batches=[]) + product = model.Product(cmd.sku, batches=[]) uow.products.add(product) - product.batches.append( - model.Batch(event.ref, event.sku, event.qty, event.eta) - ) + product.batches.append(model.Batch(cmd.ref, cmd.sku, cmd.qty, cmd.eta)) uow.commit() def allocate( - event: events.AllocationRequired, + cmd: commands.Allocate, uow: unit_of_work.AbstractUnitOfWork, ) -> str: - line = OrderLine(event.orderid, event.sku, event.qty) + line = OrderLine(cmd.orderid, cmd.sku, cmd.qty) with uow: product = uow.products.get(sku=line.sku) if product is None: @@ -42,18 +40,15 @@ def allocate( def change_batch_quantity( - event: events.BatchQuantityChanged, + cmd: commands.ChangeBatchQuantity, uow: unit_of_work.AbstractUnitOfWork, ): with uow: - product = uow.products.get_by_batchref(batchref=event.ref) - product.change_batch_quantity(ref=event.ref, qty=event.qty) + product = uow.products.get_by_batchref(batchref=cmd.ref) + product.change_batch_quantity(ref=cmd.ref, qty=cmd.qty) uow.commit() -# pylint: disable=unused-argument - - def send_out_of_stock_notification( event: events.OutOfStock, uow: unit_of_work.AbstractUnitOfWork, diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 42b5b895..d73726fe 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -2,9 +2,8 @@ from datetime import date from unittest import mock import pytest - from allocation.adapters import repository -from allocation.domain import events +from allocation.domain import commands, events from allocation.service_layer import handlers, messagebus, unit_of_work @@ -42,52 +41,54 @@ class TestAddBatch: def test_for_new_product(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("b1", "CRUNCHY-ARMCHAIR", 100, None), uow + commands.CreateBatch("b1", "CRUNCHY-ARMCHAIR", 100, None), uow ) assert uow.products.get("CRUNCHY-ARMCHAIR") is not None assert uow.committed def test_for_existing_product(self): uow = FakeUnitOfWork() - messagebus.handle(events.BatchCreated("b1", "GARISH-RUG", 100, None), uow) - messagebus.handle(events.BatchCreated("b2", "GARISH-RUG", 99, None), uow) + messagebus.handle(commands.CreateBatch("b1", "GARISH-RUG", 100, None), uow) + messagebus.handle(commands.CreateBatch("b2", "GARISH-RUG", 99, None), uow) assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] class TestAllocate: - def test_returns_allocation(self): + def test_allocates(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("batch1", "COMPLICATED-LAMP", 100, None), uow + commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None), uow ) results = messagebus.handle( - events.AllocationRequired("o1", "COMPLICATED-LAMP", 10), uow + commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow ) assert results.pop(0) == "batch1" + [batch] = uow.products.get("COMPLICATED-LAMP").batches + assert batch.available_quantity == 90 def test_errors_for_invalid_sku(self): uow = FakeUnitOfWork() - messagebus.handle(events.BatchCreated("b1", "AREALSKU", 100, None), uow) + messagebus.handle(commands.CreateBatch("b1", "AREALSKU", 100, None), uow) with pytest.raises(handlers.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - messagebus.handle( - events.AllocationRequired("o1", "NONEXISTENTSKU", 10), uow - ) + messagebus.handle(commands.Allocate("o1", "NONEXISTENTSKU", 10), uow) def test_commits(self): uow = FakeUnitOfWork() - messagebus.handle(events.BatchCreated("b1", "OMINOUS-MIRROR", 100, None), uow) - messagebus.handle(events.AllocationRequired("o1", "OMINOUS-MIRROR", 10), uow) + messagebus.handle( + commands.CreateBatch("b1", "OMINOUS-MIRROR", 100, None), uow + ) + messagebus.handle(commands.Allocate("o1", "OMINOUS-MIRROR", 10), uow) assert uow.committed def test_sends_email_on_out_of_stock_error(self): uow = FakeUnitOfWork() - messagebus.handle(events.BatchCreated("b1", "POPULAR-CURTAINS", 9, None), uow) + messagebus.handle( + commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None), uow + ) with mock.patch("allocation.adapters.email.send") as mock_send_mail: - messagebus.handle( - events.AllocationRequired("o1", "POPULAR-CURTAINS", 10), uow - ) + messagebus.handle(commands.Allocate("o1", "POPULAR-CURTAINS", 10), uow) assert mock_send_mail.call_args == mock.call( "stock@made.com", f"Out of stock for POPULAR-CURTAINS" ) @@ -97,30 +98,30 @@ class TestChangeBatchQuantity: def test_changes_available_quantity(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("batch1", "ADORABLE-SETTEE", 100, None), uow + commands.CreateBatch("batch1", "ADORABLE-SETTEE", 100, None), uow ) [batch] = uow.products.get(sku="ADORABLE-SETTEE").batches assert batch.available_quantity == 100 - messagebus.handle(events.BatchQuantityChanged("batch1", 50), uow) + messagebus.handle(commands.ChangeBatchQuantity("batch1", 50), uow) assert batch.available_quantity == 50 def test_reallocates_if_necessary(self): uow = FakeUnitOfWork() - event_history = [ - events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), - events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), - events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), - events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), + history = [ + commands.CreateBatch("batch1", "INDIFFERENT-TABLE", 50, None), + commands.CreateBatch("batch2", "INDIFFERENT-TABLE", 50, date.today()), + commands.Allocate("order1", "INDIFFERENT-TABLE", 20), + commands.Allocate("order2", "INDIFFERENT-TABLE", 20), ] - for e in event_history: - messagebus.handle(e, uow) + for msg in history: + messagebus.handle(msg, uow) [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches assert batch1.available_quantity == 10 assert batch2.available_quantity == 50 - messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) + messagebus.handle(commands.ChangeBatchQuantity("batch1", 25), uow) # order1 or order2 will be deallocated, so we'll have 25 - 20 assert batch1.available_quantity == 5 From 7aed316fe580b32a7369b585d8f09048f062210d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:39:20 +0100 Subject: [PATCH 082/145] model method for change batch qty now raise commands hmmm --- src/allocation/domain/model.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 7aa99417..9d74742c 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set -from . import events +from . import commands, events class Product: @@ -27,9 +27,7 @@ def change_batch_quantity(self, ref: str, qty: int): batch._purchased_quantity = qty while batch.available_quantity < 0: line = batch.deallocate_one() - self.events.append( - events.AllocationRequired(line.orderid, line.sku, line.qty) - ) + self.events.append(commands.Allocate(line.orderid, line.sku, line.qty)) @dataclass(unsafe_hash=True) From d958887750de7c9a5b7ddfb388e067c03ce8f285 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:39:28 +0100 Subject: [PATCH 083/145] flask now uses commands [chapter_10_commands_ends] --- src/allocation/entrypoints/flask_app.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 51922760..4d8e3204 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,7 +1,7 @@ from datetime import datetime from flask import Flask, request -from allocation.domain import events +from allocation.domain import commands from allocation.adapters import orm from allocation.service_layer import messagebus, unit_of_work from allocation.service_layer.handlers import InvalidSku @@ -15,20 +15,22 @@ def add_batch(): eta = request.json["eta"] if eta is not None: eta = datetime.fromisoformat(eta).date() - event = events.BatchCreated( + cmd = commands.CreateBatch( request.json["ref"], request.json["sku"], request.json["qty"], eta ) - messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) + uow = unit_of_work.SqlAlchemyUnitOfWork() + messagebus.handle(cmd, uow) return "OK", 201 @app.route("/allocate", methods=["POST"]) def allocate_endpoint(): try: - event = events.AllocationRequired( + cmd = commands.Allocate( request.json["orderid"], request.json["sku"], request.json["qty"] ) - results = messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) + uow = unit_of_work.SqlAlchemyUnitOfWork() + results = messagebus.handle(cmd, uow) batchref = results.pop(0) except InvalidSku as e: return {"message": str(e)}, 400 From 120f5987c1bd720bf5bf1c76bb157d16b7154804 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 14:58:39 +0100 Subject: [PATCH 084/145] Add redis to docker-compose and config, conftest --- docker-compose.yml | 6 ++++++ mypy.ini | 2 +- requirements.txt | 2 ++ src/allocation/config.py | 6 ++++++ tests/conftest.py | 31 +++++++++++++++++++++++++++++-- 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 039400e9..74ac28b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: dockerfile: Dockerfile depends_on: - postgres + - redis environment: - DB_HOST=postgres - DB_PASSWORD=abc123 @@ -27,3 +28,8 @@ services: ports: - "54321:5432" + redis: + image: redis:alpine + ports: + - "63791:6379" + diff --git a/mypy.ini b/mypy.ini index 62194f35..601283d7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,5 +3,5 @@ ignore_missing_imports = False mypy_path = ./src check_untyped_defs = True -[mypy-pytest.*,sqlalchemy.*] +[mypy-pytest.*,sqlalchemy.*,redis.*] ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt index 8c779254..5ae975d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,11 @@ sqlalchemy flask psycopg2-binary +redis # tests pytest pytest-icdiff mypy requests +tenacity diff --git a/src/allocation/config.py b/src/allocation/config.py index f3b55cc9..30a8eb07 100644 --- a/src/allocation/config.py +++ b/src/allocation/config.py @@ -13,3 +13,9 @@ def get_api_url(): host = os.environ.get("API_HOST", "localhost") port = 5005 if host == "localhost" else 80 return f"http://{host}:{port}" + + +def get_redis_host_and_port(): + host = os.environ.get("REDIS_HOST", "localhost") + port = 63791 if host == "localhost" else 6379 + return dict(host=host, port=port) diff --git a/tests/conftest.py b/tests/conftest.py index d51c20f6..918a121f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,14 @@ # pylint: disable=redefined-outer-name +import shutil +import subprocess import time from pathlib import Path import pytest +import redis import requests -from requests.exceptions import ConnectionError +from requests.exceptions import RequestException +from redis.exceptions import RedisError from sqlalchemy.exc import OperationalError from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers @@ -48,11 +52,22 @@ def wait_for_webapp_to_come_up(): while time.time() < deadline: try: return requests.get(url) - except ConnectionError: + except RequestException: time.sleep(0.5) pytest.fail("API never came up") +def wait_for_redis_to_come_up(): + deadline = time.time() + 5 + r = redis.Redis(**config.get_redis_host_and_port()) + while time.time() < deadline: + try: + return r.ping() + except RedisError: + time.sleep(0.5) + pytest.fail("Redis never came up") + + @pytest.fixture(scope="session") def postgres_db(): engine = create_engine(config.get_postgres_uri()) @@ -78,3 +93,15 @@ def restart_api(): (Path(__file__).parent / "../src/allocation/entrypoints/flask_app.py").touch() time.sleep(0.5) wait_for_webapp_to_come_up() + + +@pytest.fixture +def restart_redis_pubsub(): + wait_for_redis_to_come_up() + if not shutil.which("docker-compose"): + print("skipping restart, assumes running in container") + return + subprocess.run( + ["docker-compose", "restart", "-t", "0", "redis_pubsub"], + check=True, + ) From f99ab968d5379f8c560dbafe1fa35fba8d1e8c16 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:02:06 +0100 Subject: [PATCH 085/145] refactor e2e tests, move random_refs and api_client out --- tests/e2e/api_client.py | 25 ++++++++++++++++++++++++ tests/e2e/test_api.py | 43 +++++++++++++++++------------------------ 2 files changed, 43 insertions(+), 25 deletions(-) create mode 100644 tests/e2e/api_client.py diff --git a/tests/e2e/api_client.py b/tests/e2e/api_client.py new file mode 100644 index 00000000..646ac4f7 --- /dev/null +++ b/tests/e2e/api_client.py @@ -0,0 +1,25 @@ +import requests +from allocation import config + + +def post_to_add_batch(ref, sku, qty, eta): + url = config.get_api_url() + r = requests.post( + f"{url}/add_batch", json={"ref": ref, "sku": sku, "qty": qty, "eta": eta} + ) + assert r.status_code == 201 + + +def post_to_allocate(orderid, sku, qty, expect_success=True): + url = config.get_api_url() + r = requests.post( + f"{url}/allocate", + json={ + "orderid": orderid, + "sku": sku, + "qty": qty, + }, + ) + if expect_success: + assert r.status_code == 201 + return r diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 79345fce..04883893 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -1,16 +1,6 @@ import pytest -import requests - -from allocation import config -from ..random_refs import random_sku, random_batchref, random_orderid - - -def post_to_add_batch(ref, sku, qty, eta): - url = config.get_api_url() - r = requests.post( - f"{url}/add_batch", json={"ref": ref, "sku": sku, "qty": qty, "eta": eta} - ) - assert r.status_code == 201 +from ..random_refs import random_batchref, random_orderid, random_sku +from . import api_client @pytest.mark.usefixtures("postgres_db") @@ -20,24 +10,27 @@ def test_happy_path_returns_201_and_allocated_batch(): earlybatch = random_batchref(1) laterbatch = random_batchref(2) otherbatch = random_batchref(3) - post_to_add_batch(laterbatch, sku, 100, "2011-01-02") - post_to_add_batch(earlybatch, sku, 100, "2011-01-01") - post_to_add_batch(otherbatch, othersku, 100, None) - data = {"orderid": random_orderid(), "sku": sku, "qty": 3} + api_client.post_to_add_batch(laterbatch, sku, 100, "2011-01-02") + api_client.post_to_add_batch(earlybatch, sku, 100, "2011-01-01") + api_client.post_to_add_batch(otherbatch, othersku, 100, None) - url = config.get_api_url() - r = requests.post(f"{url}/allocate", json=data) + response = api_client.post_to_allocate(random_orderid(), sku, qty=3) - assert r.status_code == 201 - assert r.json()["batchref"] == earlybatch + assert response.status_code == 201 + assert response.json()["batchref"] == earlybatch @pytest.mark.usefixtures("postgres_db") @pytest.mark.usefixtures("restart_api") def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() - data = {"orderid": orderid, "sku": unknown_sku, "qty": 20} - url = config.get_api_url() - r = requests.post(f"{url}/allocate", json=data) - assert r.status_code == 400 - assert r.json()["message"] == f"Invalid sku {unknown_sku}" + + response = api_client.post_to_allocate( + orderid, + unknown_sku, + qty=20, + expect_success=False, + ) + + assert response.status_code == 400 + assert response.json()["message"] == f"Invalid sku {unknown_sku}" From c69661dbf6c6872a50f981940661dca64a916ae5 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:02:41 +0100 Subject: [PATCH 086/145] redis client for tests [redis_client_for_tests] --- tests/e2e/redis_client.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/e2e/redis_client.py diff --git a/tests/e2e/redis_client.py b/tests/e2e/redis_client.py new file mode 100644 index 00000000..3392d026 --- /dev/null +++ b/tests/e2e/redis_client.py @@ -0,0 +1,18 @@ +import json +import redis + +from allocation import config + +r = redis.Redis(**config.get_redis_host_and_port()) + + +def subscribe_to(channel): + pubsub = r.pubsub() + pubsub.subscribe(channel) + confirmation = pubsub.get_message(timeout=3) + assert confirmation["type"] == "subscribe" + return pubsub + + +def publish_message(channel, message): + r.publish(channel, json.dumps(message)) From 8ee0a932d18fc4be73c52716b11ca2edc5a23322 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:07:14 +0100 Subject: [PATCH 087/145] Test for our external events [redis_e2e_test] --- requirements.txt | 3 ++- tests/e2e/test_external_events.py | 38 +++++++++++++++++++++++++++++++ tests/pytest.ini | 2 ++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/test_external_events.py diff --git a/requirements.txt b/requirements.txt index 5ae975d2..882cb352 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,10 @@ flask psycopg2-binary redis -# tests +# dev/tests pytest pytest-icdiff mypy +pylint requests tenacity diff --git a/tests/e2e/test_external_events.py b/tests/e2e/test_external_events.py new file mode 100644 index 00000000..49dbd4b9 --- /dev/null +++ b/tests/e2e/test_external_events.py @@ -0,0 +1,38 @@ +import json +import pytest +from tenacity import Retrying, RetryError, stop_after_delay +from . import api_client, redis_client +from ..random_refs import random_batchref, random_orderid, random_sku + + +@pytest.mark.usefixtures("postgres_db") +@pytest.mark.usefixtures("restart_api") +@pytest.mark.usefixtures("restart_redis_pubsub") +def test_change_batch_quantity_leading_to_reallocation(): + # start with two batches and an order allocated to one of them + orderid, sku = random_orderid(), random_sku() + earlier_batch, later_batch = random_batchref("old"), random_batchref("newer") + api_client.post_to_add_batch(earlier_batch, sku, qty=10, eta="2011-01-01") + api_client.post_to_add_batch(later_batch, sku, qty=10, eta="2011-01-02") + response = api_client.post_to_allocate(orderid, sku, 10) + assert response.json()["batchref"] == earlier_batch + + subscription = redis_client.subscribe_to("line_allocated") + + # change quantity on allocated batch so it's less than our order + redis_client.publish_message( + "change_batch_quantity", + {"batchref": earlier_batch, "qty": 5}, + ) + + # wait until we see a message saying the order has been reallocated + messages = [] + for attempt in Retrying(stop=stop_after_delay(3), reraise=True): + with attempt: + message = subscription.get_message(timeout=1) + if message: + messages.append(message) + print(messages) + data = json.loads(messages[-1]["data"]) + assert data["orderid"] == orderid + assert data["batchref"] == later_batch diff --git a/tests/pytest.ini b/tests/pytest.ini index bbd083ac..3fd8685e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,2 +1,4 @@ [pytest] addopts = --tb=short +filterwarnings = + ignore::DeprecationWarning From 28b8093c44f5ba721272738ddab8c604782e7ea1 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 15 Nov 2019 14:46:04 +0000 Subject: [PATCH 088/145] use tenacity in conftest --- tests/conftest.py | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 918a121f..dc695f4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,11 +7,9 @@ import pytest import redis import requests -from requests.exceptions import RequestException -from redis.exceptions import RedisError -from sqlalchemy.exc import OperationalError from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers +from tenacity import retry, stop_after_delay from allocation.adapters.orm import metadata, start_mappers from allocation import config @@ -36,36 +34,20 @@ def session(session_factory): return session_factory() +@retry(stop=stop_after_delay(10)) def wait_for_postgres_to_come_up(engine): - deadline = time.time() + 10 - while time.time() < deadline: - try: - return engine.connect() - except OperationalError: - time.sleep(0.5) - pytest.fail("Postgres never came up") + return engine.connect() +@retry(stop=stop_after_delay(10)) def wait_for_webapp_to_come_up(): - deadline = time.time() + 10 - url = config.get_api_url() - while time.time() < deadline: - try: - return requests.get(url) - except RequestException: - time.sleep(0.5) - pytest.fail("API never came up") + return requests.get(config.get_api_url()) +@retry(stop=stop_after_delay(10)) def wait_for_redis_to_come_up(): - deadline = time.time() + 5 r = redis.Redis(**config.get_redis_host_and_port()) - while time.time() < deadline: - try: - return r.ping() - except RedisError: - time.sleep(0.5) - pytest.fail("Redis never came up") + return r.ping() @pytest.fixture(scope="session") From 42d1dd8541e81310cbf6805353052cc2a4223778 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:07:35 +0100 Subject: [PATCH 089/145] Docker infrastructure for new redis event listener container --- Dockerfile | 2 -- Makefile | 12 ++++++------ docker-compose.yml | 32 +++++++++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 73024d18..1a3c9765 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,3 @@ RUN pip install -e /src COPY tests/ /tests/ WORKDIR /src -ENV FLASK_APP=allocation/entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 -CMD flask run --host=0.0.0.0 --port=80 diff --git a/Makefile b/Makefile index 6409e955..fb56493b 100644 --- a/Makefile +++ b/Makefile @@ -8,25 +8,25 @@ build: docker-compose build up: - docker-compose up -d app + docker-compose up -d down: docker-compose down --remove-orphans test: up - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit /tests/integration /tests/e2e + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/unit /tests/integration /tests/e2e unit-tests: - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/unit integration-tests: up - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/integration + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/integration e2e-tests: up - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/e2e + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/e2e logs: - docker-compose logs app | tail -100 + docker-compose logs --tail=25 api redis_pubsub black: black -l 86 $$(find * -name '*.py') diff --git a/docker-compose.yml b/docker-compose.yml index 74ac28b6..dc2cc369 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,51 @@ version: "3" + services: - app: + redis_pubsub: build: context: . dockerfile: Dockerfile + image: allocation-image depends_on: - postgres - redis environment: - DB_HOST=postgres - DB_PASSWORD=abc123 - - API_HOST=app + - REDIS_HOST=redis - PYTHONDONTWRITEBYTECODE=1 volumes: - ./src:/src - ./tests:/tests + entrypoint: + - python + - /src/allocation/entrypoints/redis_eventconsumer.py + + api: + image: allocation-image + depends_on: + - redis_pubsub + environment: + - DB_HOST=postgres + - DB_PASSWORD=abc123 + - API_HOST=api + - REDIS_HOST=redis + - PYTHONDONTWRITEBYTECODE=1 + - FLASK_APP=allocation/entrypoints/flask_app.py + - FLASK_DEBUG=1 + - PYTHONUNBUFFERED=1 + volumes: + - ./src:/src + - ./tests:/tests + entrypoint: + - flask + - run + - --host=0.0.0.0 + - --port=80 ports: - "5005:80" - postgres: image: postgres:9.6 environment: From 75b50c5c39e75f9bad716030bd4ff6ee33ee6d42 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 14 Aug 2019 15:02:48 +0100 Subject: [PATCH 090/145] add Allocated event [allocated_event] --- src/allocation/domain/events.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index ba90ea05..e2428f50 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -6,6 +6,14 @@ class Event: pass +@dataclass +class Allocated(Event): + orderid: str + sku: str + qty: int + batchref: str + + @dataclass class OutOfStock(Event): sku: str From b54e10e68087136e2fbacd45ced20242d105e11d Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 19:11:39 +0000 Subject: [PATCH 091/145] redis eventconsumer first cut [redis_eventconsumer_first_cut] --- .../entrypoints/redis_eventconsumer.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/allocation/entrypoints/redis_eventconsumer.py diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py new file mode 100644 index 00000000..e04a8142 --- /dev/null +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -0,0 +1,32 @@ +import json +import logging +import redis + +from allocation import config +from allocation.domain import commands +from allocation.adapters import orm +from allocation.service_layer import messagebus, unit_of_work + +logger = logging.getLogger(__name__) + +r = redis.Redis(**config.get_redis_host_and_port()) + + +def main(): + orm.start_mappers() + pubsub = r.pubsub(ignore_subscribe_messages=True) + pubsub.subscribe("change_batch_quantity") + + for m in pubsub.listen(): + handle_change_batch_quantity(m) + + +def handle_change_batch_quantity(m): + logging.debug("handling %s", m) + data = json.loads(m["data"]) + cmd = commands.ChangeBatchQuantity(ref=data["batchref"], qty=data["qty"]) + messagebus.handle(cmd, uow=unit_of_work.SqlAlchemyUnitOfWork()) + + +if __name__ == "__main__": + main() From c063c2744019f8e1ba0be3748ed51609235ed01e Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 19:12:02 +0000 Subject: [PATCH 092/145] redis eventpublisher first cut [redis_eventpublisher_first_cut] --- src/allocation/adapters/redis_eventpublisher.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/allocation/adapters/redis_eventpublisher.py diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py new file mode 100644 index 00000000..6100956f --- /dev/null +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -0,0 +1,16 @@ +import json +import logging +from dataclasses import asdict +import redis + +from allocation import config +from allocation.domain import events + +logger = logging.getLogger(__name__) + +r = redis.Redis(**config.get_redis_host_and_port()) + + +def publish(channel, event: events.Event): + logging.debug("publishing: channel=%s, event=%s", channel, event) + r.publish(channel, json.dumps(asdict(event))) From 9bd4a608fc14e05fd89f2e148f3f42c7de173e8a Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 18 Jul 2019 22:29:20 +0100 Subject: [PATCH 093/145] sneak in a redis patch so unit test dont need redis --- tests/unit/test_handlers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index d73726fe..c62a55dc 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -53,6 +53,12 @@ def test_for_existing_product(self): assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] +@pytest.fixture(autouse=True) +def fake_redis_publish(): + with mock.patch("allocation.adapters.redis_eventpublisher.publish"): + yield + + class TestAllocate: def test_allocates(self): uow = FakeUnitOfWork() From 7a75df790e56e506765d9eecc2964592d19bccee Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 14 Aug 2019 15:20:18 +0100 Subject: [PATCH 094/145] test and Product change to emit event [model_emits_allocated_event] --- src/allocation/domain/model.py | 8 ++++++++ tests/unit/test_product.py | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 9d74742c..dcdd639a 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -17,6 +17,14 @@ def allocate(self, line: OrderLine) -> str: batch = next(b for b in sorted(self.batches) if b.can_allocate(line)) batch.allocate(line) self.version_number += 1 + self.events.append( + events.Allocated( + orderid=line.orderid, + sku=line.sku, + qty=line.qty, + batchref=batch.reference, + ) + ) return batch.reference except StopIteration: self.events.append(events.OutOfStock(line.sku)) diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index 1a1482b6..fc2b7015 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -43,6 +43,17 @@ def test_returns_allocated_batch_ref(): assert allocation == in_stock_batch.reference +def test_outputs_allocated_event(): + batch = Batch("batchref", "RETRO-LAMPSHADE", 100, eta=None) + line = OrderLine("oref", "RETRO-LAMPSHADE", 10) + product = Product(sku="RETRO-LAMPSHADE", batches=[batch]) + product.allocate(line) + expected = events.Allocated( + orderid="oref", sku="RETRO-LAMPSHADE", qty=10, batchref=batch.reference + ) + assert product.events[-1] == expected + + def test_records_out_of_stock_event_if_cannot_allocate(): batch = Batch("batch1", "SMALL-FORK", 10, eta=today) product = Product(sku="SMALL-FORK", batches=[batch]) From 070eec61435bad3308720d65f45ebb70a5225e23 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 14 Aug 2019 15:22:23 +0100 Subject: [PATCH 095/145] add handler for Allocated [chapter_11_external_events_ends] --- src/allocation/service_layer/handlers.py | 12 +++++++++++- src/allocation/service_layer/messagebus.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 8c7b23d5..2d6657f0 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from allocation.adapters import email +from allocation.adapters import email, redis_eventpublisher from allocation.domain import commands, events, model from allocation.domain.model import OrderLine @@ -49,6 +49,9 @@ def change_batch_quantity( uow.commit() +# pylint: disable=unused-argument + + def send_out_of_stock_notification( event: events.OutOfStock, uow: unit_of_work.AbstractUnitOfWork, @@ -57,3 +60,10 @@ def send_out_of_stock_notification( "stock@made.com", f"Out of stock for {event.sku}", ) + + +def publish_allocated_event( + event: events.Allocated, + uow: unit_of_work.AbstractUnitOfWork, +): + redis_eventpublisher.publish("line_allocated", event) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 8bac77a2..fa6d0a71 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -63,6 +63,7 @@ def handle_command( EVENT_HANDLERS = { + events.Allocated: [handlers.publish_allocated_event], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From bd080f2d6e39866a47456d78772f244f417032ff Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:22:46 +0100 Subject: [PATCH 096/145] modify api tests to try and do a get after a post [get_after_post] --- tests/conftest.py | 2 ++ tests/e2e/api_client.py | 7 ++++++- tests/e2e/test_api.py | 27 +++++++++++++++------------ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dc695f4d..b9d18175 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,8 @@ from allocation.adapters.orm import metadata, start_mappers from allocation import config +pytest.register_assert_rewrite("tests.e2e.api_client") + @pytest.fixture def in_memory_db(): diff --git a/tests/e2e/api_client.py b/tests/e2e/api_client.py index 646ac4f7..9ce00e28 100644 --- a/tests/e2e/api_client.py +++ b/tests/e2e/api_client.py @@ -21,5 +21,10 @@ def post_to_allocate(orderid, sku, qty, expect_success=True): }, ) if expect_success: - assert r.status_code == 201 + assert r.status_code == 202 return r + + +def get_allocation(orderid): + url = config.get_api_url() + return requests.get(f"{url}/allocations/{orderid}") diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 04883893..13d86f6f 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -5,7 +5,8 @@ @pytest.mark.usefixtures("postgres_db") @pytest.mark.usefixtures("restart_api") -def test_happy_path_returns_201_and_allocated_batch(): +def test_happy_path_returns_202_and_batch_is_allocated(): + orderid = random_orderid() sku, othersku = random_sku(), random_sku("other") earlybatch = random_batchref(1) laterbatch = random_batchref(2) @@ -14,23 +15,25 @@ def test_happy_path_returns_201_and_allocated_batch(): api_client.post_to_add_batch(earlybatch, sku, 100, "2011-01-01") api_client.post_to_add_batch(otherbatch, othersku, 100, None) - response = api_client.post_to_allocate(random_orderid(), sku, qty=3) + r = api_client.post_to_allocate(orderid, sku, qty=3) + assert r.status_code == 202 - assert response.status_code == 201 - assert response.json()["batchref"] == earlybatch + r = api_client.get_allocation(orderid) + assert r.ok + assert r.json() == [ + {"sku": sku, "batchref": earlybatch}, + ] @pytest.mark.usefixtures("postgres_db") @pytest.mark.usefixtures("restart_api") def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() - - response = api_client.post_to_allocate( - orderid, - unknown_sku, - qty=20, - expect_success=False, + r = api_client.post_to_allocate( + orderid, unknown_sku, qty=20, expect_success=False ) + assert r.status_code == 400 + assert r.json()["message"] == f"Invalid sku {unknown_sku}" - assert response.status_code == 400 - assert response.json()["message"] == f"Invalid sku {unknown_sku}" + r = api_client.get_allocation(orderid) + assert r.status_code == 404 From 9b4bd65dbdf48dff4a3a452da6b8381b5c8d80ca Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:23:22 +0100 Subject: [PATCH 097/145] modify allocate handler to no longer return anything --- src/allocation/service_layer/handlers.py | 5 ++--- tests/unit/test_handlers.py | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 2d6657f0..15e7f08f 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -28,15 +28,14 @@ def add_batch( def allocate( cmd: commands.Allocate, uow: unit_of_work.AbstractUnitOfWork, -) -> str: +): line = OrderLine(cmd.orderid, cmd.sku, cmd.qty) with uow: product = uow.products.get(sku=line.sku) if product is None: raise InvalidSku(f"Invalid sku {line.sku}") - batchref = product.allocate(line) + product.allocate(line) uow.commit() - return batchref def change_batch_quantity( diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index c62a55dc..04fb9630 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -65,10 +65,7 @@ def test_allocates(self): messagebus.handle( commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None), uow ) - results = messagebus.handle( - commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow - ) - assert results.pop(0) == "batch1" + messagebus.handle(commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow) [batch] = uow.products.get("COMPLICATED-LAMP").batches assert batch.available_quantity == 90 From 94111f5bc9533e3ebb18df02ffaa5b27c26ebda7 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:50:01 +0100 Subject: [PATCH 098/145] modify flask to add new view endpoint and return 202s --- src/allocation/entrypoints/flask_app.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 4d8e3204..1070e173 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,10 +1,11 @@ from datetime import datetime -from flask import Flask, request +from flask import Flask, jsonify, request from allocation.domain import commands from allocation.adapters import orm from allocation.service_layer import messagebus, unit_of_work from allocation.service_layer.handlers import InvalidSku +from allocation import views app = Flask(__name__) orm.start_mappers() @@ -35,4 +36,13 @@ def allocate_endpoint(): except InvalidSku as e: return {"message": str(e)}, 400 - return {"batchref": batchref}, 201 + return "OK", 202 + + +@app.route("/allocations/", methods=["GET"]) +def allocations_view_endpoint(orderid): + uow = unit_of_work.SqlAlchemyUnitOfWork() + result = views.allocations(orderid, uow) + if not result: + return "not found", 404 + return jsonify(result), 200 From 3a55cde57ba2ddee9e0c4e658daac9da3063fb0e Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 12 Jun 2019 21:57:35 +0100 Subject: [PATCH 099/145] session_factory -> sqlite_session_factory (needs backport) --- tests/conftest.py | 6 +++--- tests/integration/test_repository.py | 4 ++-- tests/integration/test_uow.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b9d18175..3b2a8066 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,15 +25,15 @@ def in_memory_db(): @pytest.fixture -def session_factory(in_memory_db): +def sqlite_session_factory(in_memory_db): start_mappers() yield sessionmaker(bind=in_memory_db) clear_mappers() @pytest.fixture -def session(session_factory): - return session_factory() +def sqlite_session(sqlite_session_factory): + return sqlite_session_factory() @retry(stop=stop_after_delay(10)) diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index a73bcd51..9b637f1a 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -2,8 +2,8 @@ from allocation.domain import model -def test_get_by_batchref(session): - repo = repository.SqlAlchemyRepository(session) +def test_get_by_batchref(sqlite_session): + repo = repository.SqlAlchemyRepository(sqlite_session) b1 = model.Batch(ref="b1", sku="sku1", qty=100, eta=None) b2 = model.Batch(ref="b2", sku="sku1", qty=100, eta=None) b3 = model.Batch(ref="b3", sku="sku2", qty=100, eta=None) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index a95907cf..55c4f24d 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -34,12 +34,12 @@ def get_allocated_batch_ref(session, orderid, sku): return batchref -def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): - session = session_factory() +def test_uow_can_retrieve_a_batch_and_allocate_to_it(sqlite_session_factory): + session = sqlite_session_factory() insert_batch(session, "batch1", "HIPSTER-WORKBENCH", 100, None) session.commit() - uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) with uow: product = uow.products.get(sku="HIPSTER-WORKBENCH") line = model.OrderLine("o1", "HIPSTER-WORKBENCH", 10) @@ -50,27 +50,27 @@ def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): assert batchref == "batch1" -def test_rolls_back_uncommitted_work_by_default(session_factory): - uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) +def test_rolls_back_uncommitted_work_by_default(sqlite_session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) with uow: insert_batch(uow.session, "batch1", "MEDIUM-PLINTH", 100, None) - new_session = session_factory() + new_session = sqlite_session_factory() rows = list(new_session.execute('SELECT * FROM "batches"')) assert rows == [] -def test_rolls_back_on_error(session_factory): +def test_rolls_back_on_error(sqlite_session_factory): class MyException(Exception): pass - uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) with pytest.raises(MyException): with uow: insert_batch(uow.session, "batch1", "LARGE-FORK", 100, None) raise MyException() - new_session = session_factory() + new_session = sqlite_session_factory() rows = list(new_session.execute('SELECT * FROM "batches"')) assert rows == [] From 0866c72d552248b8a5024ae41bb0f6d604fd9d45 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:50:10 +0100 Subject: [PATCH 100/145] integration test for our view --- tests/integration/test_views.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/integration/test_views.py diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py new file mode 100644 index 00000000..99db03b0 --- /dev/null +++ b/tests/integration/test_views.py @@ -0,0 +1,23 @@ +from datetime import date +from allocation import views +from allocation.domain import commands +from allocation.service_layer import messagebus, unit_of_work + +today = date.today() + + +def test_allocations_view(sqlite_session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) + messagebus.handle(commands.CreateBatch("sku1batch", "sku1", 50, None), uow) + messagebus.handle(commands.CreateBatch("sku2batch", "sku2", 50, today), uow) + messagebus.handle(commands.Allocate("order1", "sku1", 20), uow) + messagebus.handle(commands.Allocate("order1", "sku2", 20), uow) + # add a spurious batch and order to make sure we're getting the right ones + messagebus.handle(commands.CreateBatch("sku1batch-later", "sku1", 50, today), uow) + messagebus.handle(commands.Allocate("otherorder", "sku1", 30), uow) + messagebus.handle(commands.Allocate("otherorder", "sku2", 10), uow) + + assert views.allocations("order1", uow) == [ + {"sku": "sku1", "batchref": "sku1batch"}, + {"sku": "sku2", "batchref": "sku2batch"}, + ] From f63f7a5cc1648a106332d689f75d8d229e4cbfd6 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:50:25 +0100 Subject: [PATCH 101/145] first cut of a view with raw sql [views_dot_py] --- src/allocation/views.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/allocation/views.py diff --git a/src/allocation/views.py b/src/allocation/views.py new file mode 100644 index 00000000..a564b8d0 --- /dev/null +++ b/src/allocation/views.py @@ -0,0 +1,16 @@ +from allocation.service_layer import unit_of_work + + +def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): + with uow: + results = uow.session.execute( + """ + SELECT ol.sku, b.reference + FROM allocations AS a + JOIN batches AS b ON a.batch_id = b.id + JOIN order_lines AS ol ON a.orderline_id = ol.id + WHERE ol.orderid = :orderid + """, + dict(orderid=orderid), + ) + return [{"sku": sku, "batchref": batchref} for sku, batchref in results] From 5e545e1a48591e4e915fc26f6dbd6acef356694a Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:26:44 +0000 Subject: [PATCH 102/145] use repository and go via Product [view_using_repo] --- src/allocation/views.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/allocation/views.py b/src/allocation/views.py index a564b8d0..50dc92e3 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -3,14 +3,10 @@ def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - results = uow.session.execute( - """ - SELECT ol.sku, b.reference - FROM allocations AS a - JOIN batches AS b ON a.batch_id = b.id - JOIN order_lines AS ol ON a.orderline_id = ol.id - WHERE ol.orderid = :orderid - """, - dict(orderid=orderid), - ) - return [{"sku": sku, "batchref": batchref} for sku, batchref in results] + products = uow.products.for_order(orderid=orderid) + batches = [b for p in products for b in p.batches] + return [ + {"sku": b.sku, "batchref": b.reference} + for b in batches + if orderid in b.orderids + ] From 074c8205b72de7c3605dc9955cb2279c73fd12c8 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:27:20 +0000 Subject: [PATCH 103/145] arguably-unnecessary helper property on model. [orderids_property] --- src/allocation/domain/model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index dcdd639a..e5aa4c37 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -71,6 +71,10 @@ def __gt__(self, other): return True return self.eta > other.eta + @property + def orderids(self): + return {l.orderid for l in self._allocations} + def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) From 38f89db439442a86e7a36eb9749f3c13abbf0c84 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:27:47 +0000 Subject: [PATCH 104/145] finder method on repo [for_order_method] --- src/allocation/adapters/repository.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index f8821758..b02a00a4 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -52,6 +52,17 @@ def _get_by_batchref(self, batchref): return ( self.session.query(model.Product) .join(model.Batch) - .filter(orm.batches.c.reference == batchref) + .filter( + orm.batches.c.reference == batchref, + ) .first() ) + + def for_order(self, orderid): + order_lines = self.session.query(model.OrderLine).filter_by(orderid=orderid) + skus = {l.sku for l in order_lines} + return ( + self.session.query(model.Product) + .join(model.Batch) + .filter(model.Batch.sku.in_(skus)) + ) From 71091ba64f923b60d9dd531fd8902626c5a55d4b Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:29:39 +0000 Subject: [PATCH 105/145] Use the ORM instead [view_using_orm] --- src/allocation/adapters/repository.py | 9 --------- src/allocation/domain/model.py | 4 ---- src/allocation/views.py | 14 +++++++------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index b02a00a4..15a6b0ae 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -57,12 +57,3 @@ def _get_by_batchref(self, batchref): ) .first() ) - - def for_order(self, orderid): - order_lines = self.session.query(model.OrderLine).filter_by(orderid=orderid) - skus = {l.sku for l in order_lines} - return ( - self.session.query(model.Product) - .join(model.Batch) - .filter(model.Batch.sku.in_(skus)) - ) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index e5aa4c37..dcdd639a 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -71,10 +71,6 @@ def __gt__(self, other): return True return self.eta > other.eta - @property - def orderids(self): - return {l.orderid for l in self._allocations} - def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) diff --git a/src/allocation/views.py b/src/allocation/views.py index 50dc92e3..80fc9aa6 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,12 +1,12 @@ +from allocation.domain import model from allocation.service_layer import unit_of_work def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - products = uow.products.for_order(orderid=orderid) - batches = [b for p in products for b in p.batches] - return [ - {"sku": b.sku, "batchref": b.reference} - for b in batches - if orderid in b.orderids - ] + batches = ( + uow.session.query(model.Batch) + .join(model.OrderLine, model.Batch._allocations) + .filter(model.OrderLine.orderid == orderid) + ) + return [{"sku": b.sku, "batchref": b.reference} for b in batches] From 50905cf8da30ab56c8d31be20f90445b756fa79a Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:33:33 +0100 Subject: [PATCH 106/145] Simpler view based on a new read model table --- src/allocation/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allocation/views.py b/src/allocation/views.py index 80fc9aa6..ca56ceed 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,12 +1,12 @@ -from allocation.domain import model from allocation.service_layer import unit_of_work def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - batches = ( - uow.session.query(model.Batch) - .join(model.OrderLine, model.Batch._allocations) - .filter(model.OrderLine.orderid == orderid) + results = list( + uow.session.execute( + "SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid", + dict(orderid=orderid), + ) ) - return [{"sku": b.sku, "batchref": b.reference} for b in batches] + return [dict(r) for r in results] From 7e58f4b171806b1d8c003c86340e290c1964680e Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:33:43 +0100 Subject: [PATCH 107/145] new table in orm --- src/allocation/adapters/orm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index ea76eed5..a87068a5 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -49,6 +49,14 @@ Column("batch_id", ForeignKey("batches.id")), ) +allocations_view = Table( + "allocations_view", + metadata, + Column("orderid", String(255)), + Column("sku", String(255)), + Column("batchref", String(255)), +) + def start_mappers(): lines_mapper = mapper(model.OrderLine, order_lines) From 011d640b18ab8289ce6a27ed0001f4fd6b170d1f Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:33:58 +0100 Subject: [PATCH 108/145] handler for view model update --- src/allocation/service_layer/handlers.py | 15 +++++++++++++++ src/allocation/views.py | 10 +++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 15e7f08f..34df09be 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -66,3 +66,18 @@ def publish_allocated_event( uow: unit_of_work.AbstractUnitOfWork, ): redis_eventpublisher.publish("line_allocated", event) + + +def add_allocation_to_read_model( + event: events.Allocated, + uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + """ + INSERT INTO allocations_view (orderid, sku, batchref) + VALUES (:orderid, :sku, :batchref) + """, + dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref), + ) + uow.commit() diff --git a/src/allocation/views.py b/src/allocation/views.py index ca56ceed..a952887f 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -3,10 +3,10 @@ def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - results = list( - uow.session.execute( - "SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid", - dict(orderid=orderid), - ) + results = uow.session.execute( + """ + SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid + """, + dict(orderid=orderid), ) return [dict(r) for r in results] From ac940910754c9cbec96cc1e18b05a9fad1088781 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:34:34 +0100 Subject: [PATCH 109/145] add handler for allocated --- src/allocation/service_layer/messagebus.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index fa6d0a71..fef5a91d 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -63,7 +63,10 @@ def handle_command( EVENT_HANDLERS = { - events.Allocated: [handlers.publish_allocated_event], + events.Allocated: [ + handlers.publish_allocated_event, + handlers.add_allocation_to_read_model, + ], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From c015f2aa5be4dd7adb4dac9aa5c802a34c2c96a3 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 06:21:27 +0100 Subject: [PATCH 110/145] handle_command no longer returns --- src/allocation/entrypoints/flask_app.py | 3 +-- src/allocation/service_layer/messagebus.py | 13 +++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 1070e173..ed482390 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -31,8 +31,7 @@ def allocate_endpoint(): request.json["orderid"], request.json["sku"], request.json["qty"] ) uow = unit_of_work.SqlAlchemyUnitOfWork() - results = messagebus.handle(cmd, uow) - batchref = results.pop(0) + messagebus.handle(cmd, uow) except InvalidSku as e: return {"message": str(e)}, 400 diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index fef5a91d..9527625a 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -13,22 +13,16 @@ Message = Union[commands.Command, events.Event] -def handle( - message: Message, - uow: unit_of_work.AbstractUnitOfWork, -): - results = [] +def handle(message: Message, uow: unit_of_work.AbstractUnitOfWork): queue = [message] while queue: message = queue.pop(0) if isinstance(message, events.Event): handle_event(message, queue, uow) elif isinstance(message, commands.Command): - cmd_result = handle_command(message, queue, uow) - results.append(cmd_result) + handle_command(message, queue, uow) else: raise Exception(f"{message} was not an Event or Command") - return results def handle_event( @@ -54,9 +48,8 @@ def handle_command( logger.debug("handling command %s", command) try: handler = COMMAND_HANDLERS[type(command)] - result = handler(command, uow=uow) + handler(command, uow=uow) queue.extend(uow.collect_new_events()) - return result except Exception: logger.exception("Exception handling command %s", command) raise From f98c68d8a7c2af5a8a823b9c46b5a58df7bc739e Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 06:21:35 +0100 Subject: [PATCH 111/145] pylint thing --- src/allocation/service_layer/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 34df09be..66a64b1c 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,3 +1,4 @@ +# pylint: disable=unused-argument from __future__ import annotations from typing import TYPE_CHECKING from allocation.adapters import email, redis_eventpublisher From 7d6be2cdc3a1af9d0fe83500e252b6f81c877186 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 06:20:52 +0100 Subject: [PATCH 112/145] fix redis e2e test --- tests/e2e/test_external_events.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_external_events.py b/tests/e2e/test_external_events.py index 49dbd4b9..c4fde79f 100644 --- a/tests/e2e/test_external_events.py +++ b/tests/e2e/test_external_events.py @@ -14,8 +14,10 @@ def test_change_batch_quantity_leading_to_reallocation(): earlier_batch, later_batch = random_batchref("old"), random_batchref("newer") api_client.post_to_add_batch(earlier_batch, sku, qty=10, eta="2011-01-01") api_client.post_to_add_batch(later_batch, sku, qty=10, eta="2011-01-02") - response = api_client.post_to_allocate(orderid, sku, 10) - assert response.json()["batchref"] == earlier_batch + r = api_client.post_to_allocate(orderid, sku, 10) + assert r.ok + response = api_client.get_allocation(orderid) + assert response.json()[0]["batchref"] == earlier_batch subscription = redis_client.subscribe_to("line_allocated") From 02e6b5d57aca230b9840a0e6d578ecebb502a0ef Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:43:35 +0100 Subject: [PATCH 113/145] make sure deallocation fixes view model too [deallocation_to_readmodel] --- src/allocation/domain/events.py | 7 ++++++ src/allocation/domain/model.py | 2 +- src/allocation/service_layer/handlers.py | 26 ++++++++++++++++++++++ src/allocation/service_layer/messagebus.py | 4 ++++ tests/integration/test_views.py | 12 ++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index e2428f50..47634f64 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -14,6 +14,13 @@ class Allocated(Event): batchref: str +@dataclass +class Deallocated(Event): + orderid: str + sku: str + qty: int + + @dataclass class OutOfStock(Event): sku: str diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index dcdd639a..dd4ac782 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -35,7 +35,7 @@ def change_batch_quantity(self, ref: str, qty: int): batch._purchased_quantity = qty while batch.available_quantity < 0: line = batch.deallocate_one() - self.events.append(commands.Allocate(line.orderid, line.sku, line.qty)) + self.events.append(events.Deallocated(line.orderid, line.sku, line.qty)) @dataclass(unsafe_hash=True) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 66a64b1c..fa2698a4 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,5 +1,6 @@ # pylint: disable=unused-argument from __future__ import annotations +from dataclasses import asdict from typing import TYPE_CHECKING from allocation.adapters import email, redis_eventpublisher from allocation.domain import commands, events, model @@ -39,6 +40,16 @@ def allocate( uow.commit() +def reallocate( + event: events.Deallocated, + uow: unit_of_work.AbstractUnitOfWork, +): + with uow: + product = uow.products.get(sku=event.sku) + product.events.append(commands.Allocate(**asdict(event))) + uow.commit() + + def change_batch_quantity( cmd: commands.ChangeBatchQuantity, uow: unit_of_work.AbstractUnitOfWork, @@ -82,3 +93,18 @@ def add_allocation_to_read_model( dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref), ) uow.commit() + + +def remove_allocation_from_read_model( + event: events.Deallocated, + uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + """ + DELETE FROM allocations_view + WHERE orderid = :orderid AND sku = :sku + """, + dict(orderid=event.orderid, sku=event.sku), + ) + uow.commit() diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 9527625a..104111fa 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -60,6 +60,10 @@ def handle_command( handlers.publish_allocated_event, handlers.add_allocation_to_read_model, ], + events.Deallocated: [ + handlers.remove_allocation_from_read_model, + handlers.reallocate, + ], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 99db03b0..4dda3e54 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -21,3 +21,15 @@ def test_allocations_view(sqlite_session_factory): {"sku": "sku1", "batchref": "sku1batch"}, {"sku": "sku2", "batchref": "sku2batch"}, ] + + +def test_deallocation(sqlite_session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) + messagebus.handle(commands.CreateBatch("b1", "sku1", 50, None), uow) + messagebus.handle(commands.CreateBatch("b2", "sku1", 50, today), uow) + messagebus.handle(commands.Allocate("o1", "sku1", 40), uow) + messagebus.handle(commands.ChangeBatchQuantity("b1", 10), uow) + + assert views.allocations("o1", uow) == [ + {"sku": "sku1", "batchref": "b2"}, + ] From d090fb8e7acc1869208e215d9e9b53370b5b9777 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 12:22:25 +0000 Subject: [PATCH 114/145] handlers talk to redis [redis_readmodel_handlers] --- src/allocation/service_layer/handlers.py | 30 ++++-------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index fa2698a4..0ad6d849 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -80,31 +80,9 @@ def publish_allocated_event( redis_eventpublisher.publish("line_allocated", event) -def add_allocation_to_read_model( - event: events.Allocated, - uow: unit_of_work.SqlAlchemyUnitOfWork, -): - with uow: - uow.session.execute( - """ - INSERT INTO allocations_view (orderid, sku, batchref) - VALUES (:orderid, :sku, :batchref) - """, - dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref), - ) - uow.commit() +def add_allocation_to_read_model(event: events.Allocated, _): + redis_eventpublisher.update_readmodel(event.orderid, event.sku, event.batchref) -def remove_allocation_from_read_model( - event: events.Deallocated, - uow: unit_of_work.SqlAlchemyUnitOfWork, -): - with uow: - uow.session.execute( - """ - DELETE FROM allocations_view - WHERE orderid = :orderid AND sku = :sku - """, - dict(orderid=event.orderid, sku=event.sku), - ) - uow.commit() +def remove_allocation_from_read_model(event: events.Deallocated, _): + redis_eventpublisher.update_readmodel(event.orderid, event.sku, None) From 856ec5fcda774cb53ca1edf830de66846a53e3aa Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 16:16:46 +0000 Subject: [PATCH 115/145] new helpers to update read model [redis_readmodel_client] --- src/allocation/adapters/redis_eventpublisher.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py index 6100956f..a0d1246b 100644 --- a/src/allocation/adapters/redis_eventpublisher.py +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -14,3 +14,11 @@ def publish(channel, event: events.Event): logging.debug("publishing: channel=%s, event=%s", channel, event) r.publish(channel, json.dumps(asdict(event))) + + +def update_readmodel(orderid, sku, batchref): + r.hset(orderid, sku, batchref) + + +def get_readmodel(orderid): + return r.hgetall(orderid) From ec9fb19af5723a58aee2aa9dcff4d4d06bfe60c2 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 16:17:08 +0000 Subject: [PATCH 116/145] view now users redis, tweak tests+app. [redis_readmodel_view] --- src/allocation/entrypoints/flask_app.py | 3 +-- src/allocation/views.py | 16 +++++++--------- tests/integration/test_views.py | 20 +++++++++++++++++--- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index ed482390..c1cb915b 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -40,8 +40,7 @@ def allocate_endpoint(): @app.route("/allocations/", methods=["GET"]) def allocations_view_endpoint(orderid): - uow = unit_of_work.SqlAlchemyUnitOfWork() - result = views.allocations(orderid, uow) + result = views.allocations(orderid) if not result: return "not found", 404 return jsonify(result), 200 diff --git a/src/allocation/views.py b/src/allocation/views.py index a952887f..b9dc175f 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,12 +1,10 @@ +from allocation.adapters import redis_eventpublisher from allocation.service_layer import unit_of_work -def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): - with uow: - results = uow.session.execute( - """ - SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid - """, - dict(orderid=orderid), - ) - return [dict(r) for r in results] +def allocations(orderid: str): + batches = redis_eventpublisher.get_readmodel(orderid) + return [ + {"batchref": b.decode(), "sku": s.decode()} + for s, b in batches.items() + ] diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 4dda3e54..f38f75a4 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,11 +1,25 @@ from datetime import date -from allocation import views +import pytest +import redis +from allocation import config, views from allocation.domain import commands from allocation.service_layer import messagebus, unit_of_work today = date.today() +@pytest.fixture +def cleanup_redis(): + r = redis.Redis(**config.get_redis_host_and_port()) + yield + for k in r.keys(): + print("cleaning up redis key", k) + r.delete(k) + + +pytestmark = pytest.mark.usefixtures("cleanup_redis") + + def test_allocations_view(sqlite_session_factory): uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) messagebus.handle(commands.CreateBatch("sku1batch", "sku1", 50, None), uow) @@ -17,7 +31,7 @@ def test_allocations_view(sqlite_session_factory): messagebus.handle(commands.Allocate("otherorder", "sku1", 30), uow) messagebus.handle(commands.Allocate("otherorder", "sku2", 10), uow) - assert views.allocations("order1", uow) == [ + assert views.allocations("order1") == [ {"sku": "sku1", "batchref": "sku1batch"}, {"sku": "sku2", "batchref": "sku2batch"}, ] @@ -30,6 +44,6 @@ def test_deallocation(sqlite_session_factory): messagebus.handle(commands.Allocate("o1", "sku1", 40), uow) messagebus.handle(commands.ChangeBatchQuantity("b1", 10), uow) - assert views.allocations("o1", uow) == [ + assert views.allocations("o1") == [ {"sku": "sku1", "batchref": "b2"}, ] From d4f651dfe6cdf6b5a822898b3100d5fdfce6c24b Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 16:17:16 +0000 Subject: [PATCH 117/145] Revert "view now users redis, tweak tests+app. [...]" [chapter_12_cqrs_ends] This reverts commit 6dd38db04ea7c637f2d2510e7f587ac2009dd2ac. Revert "new helpers to update read model ..." This reverts commit dbaebdf249d952f2c3915526fd4d1bc6fe3cd18f. Revert "handlers talk to redis ..." This reverts commit 00658e2181de3e118579fa0a0da9b7ebb9e081f9. --- .../adapters/redis_eventpublisher.py | 8 ----- src/allocation/entrypoints/flask_app.py | 3 +- src/allocation/service_layer/handlers.py | 30 ++++++++++++++++--- src/allocation/views.py | 16 +++++----- tests/integration/test_views.py | 20 ++----------- 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py index a0d1246b..6100956f 100644 --- a/src/allocation/adapters/redis_eventpublisher.py +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -14,11 +14,3 @@ def publish(channel, event: events.Event): logging.debug("publishing: channel=%s, event=%s", channel, event) r.publish(channel, json.dumps(asdict(event))) - - -def update_readmodel(orderid, sku, batchref): - r.hset(orderid, sku, batchref) - - -def get_readmodel(orderid): - return r.hgetall(orderid) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index c1cb915b..ed482390 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -40,7 +40,8 @@ def allocate_endpoint(): @app.route("/allocations/", methods=["GET"]) def allocations_view_endpoint(orderid): - result = views.allocations(orderid) + uow = unit_of_work.SqlAlchemyUnitOfWork() + result = views.allocations(orderid, uow) if not result: return "not found", 404 return jsonify(result), 200 diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 0ad6d849..fa2698a4 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -80,9 +80,31 @@ def publish_allocated_event( redis_eventpublisher.publish("line_allocated", event) -def add_allocation_to_read_model(event: events.Allocated, _): - redis_eventpublisher.update_readmodel(event.orderid, event.sku, event.batchref) +def add_allocation_to_read_model( + event: events.Allocated, + uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + """ + INSERT INTO allocations_view (orderid, sku, batchref) + VALUES (:orderid, :sku, :batchref) + """, + dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref), + ) + uow.commit() -def remove_allocation_from_read_model(event: events.Deallocated, _): - redis_eventpublisher.update_readmodel(event.orderid, event.sku, None) +def remove_allocation_from_read_model( + event: events.Deallocated, + uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + """ + DELETE FROM allocations_view + WHERE orderid = :orderid AND sku = :sku + """, + dict(orderid=event.orderid, sku=event.sku), + ) + uow.commit() diff --git a/src/allocation/views.py b/src/allocation/views.py index b9dc175f..a952887f 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,10 +1,12 @@ -from allocation.adapters import redis_eventpublisher from allocation.service_layer import unit_of_work -def allocations(orderid: str): - batches = redis_eventpublisher.get_readmodel(orderid) - return [ - {"batchref": b.decode(), "sku": s.decode()} - for s, b in batches.items() - ] +def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): + with uow: + results = uow.session.execute( + """ + SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid + """, + dict(orderid=orderid), + ) + return [dict(r) for r in results] diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index f38f75a4..4dda3e54 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,25 +1,11 @@ from datetime import date -import pytest -import redis -from allocation import config, views +from allocation import views from allocation.domain import commands from allocation.service_layer import messagebus, unit_of_work today = date.today() -@pytest.fixture -def cleanup_redis(): - r = redis.Redis(**config.get_redis_host_and_port()) - yield - for k in r.keys(): - print("cleaning up redis key", k) - r.delete(k) - - -pytestmark = pytest.mark.usefixtures("cleanup_redis") - - def test_allocations_view(sqlite_session_factory): uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) messagebus.handle(commands.CreateBatch("sku1batch", "sku1", 50, None), uow) @@ -31,7 +17,7 @@ def test_allocations_view(sqlite_session_factory): messagebus.handle(commands.Allocate("otherorder", "sku1", 30), uow) messagebus.handle(commands.Allocate("otherorder", "sku2", 10), uow) - assert views.allocations("order1") == [ + assert views.allocations("order1", uow) == [ {"sku": "sku1", "batchref": "sku1batch"}, {"sku": "sku2", "batchref": "sku2batch"}, ] @@ -44,6 +30,6 @@ def test_deallocation(sqlite_session_factory): messagebus.handle(commands.Allocate("o1", "sku1", 40), uow) messagebus.handle(commands.ChangeBatchQuantity("b1", 10), uow) - assert views.allocations("o1") == [ + assert views.allocations("o1", uow) == [ {"sku": "sku1", "batchref": "b2"}, ] From 286f106616cfa23779f661981043ca46175c695d Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 16:57:14 +0100 Subject: [PATCH 118/145] handlers now have all and only explicit dependencies [handler_with_explicit_dependency] --- src/allocation/service_layer/handlers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index fa2698a4..bcaf1f44 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,8 +1,7 @@ # pylint: disable=unused-argument from __future__ import annotations from dataclasses import asdict -from typing import TYPE_CHECKING -from allocation.adapters import email, redis_eventpublisher +from typing import Callable, TYPE_CHECKING from allocation.domain import commands, events, model from allocation.domain.model import OrderLine @@ -65,9 +64,9 @@ def change_batch_quantity( def send_out_of_stock_notification( event: events.OutOfStock, - uow: unit_of_work.AbstractUnitOfWork, + send_mail: Callable, ): - email.send( + send_mail( "stock@made.com", f"Out of stock for {event.sku}", ) @@ -75,9 +74,9 @@ def send_out_of_stock_notification( def publish_allocated_event( event: events.Allocated, - uow: unit_of_work.AbstractUnitOfWork, + publish: Callable, ): - redis_eventpublisher.publish("line_allocated", event) + publish("line_allocated", event) def add_allocation_to_read_model( From 257f29fc1c1bfd22439bd453b339a2d510cc5df1 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 07:47:34 +0000 Subject: [PATCH 119/145] change reallocate handler to avoid cmd/event clash --- src/allocation/service_layer/handlers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index bcaf1f44..f56e6f77 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -43,10 +43,7 @@ def reallocate( event: events.Deallocated, uow: unit_of_work.AbstractUnitOfWork, ): - with uow: - product = uow.products.get(sku=event.sku) - product.events.append(commands.Allocate(**asdict(event))) - uow.commit() + allocate(commands.Allocate(**asdict(event)), uow=uow) def change_batch_quantity( From a0c2c47c9710258e4957cc41775e735287f2ccdd Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 07:47:56 +0000 Subject: [PATCH 120/145] bring static handlers dicts across from messagebus --- src/allocation/service_layer/handlers.py | 15 ++++++++++++++- src/allocation/service_layer/messagebus.py | 21 +-------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index f56e6f77..6ab2bbe9 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,7 +1,7 @@ # pylint: disable=unused-argument from __future__ import annotations from dataclasses import asdict -from typing import Callable, TYPE_CHECKING +from typing import List, Dict, Callable, Type, TYPE_CHECKING from allocation.domain import commands, events, model from allocation.domain.model import OrderLine @@ -104,3 +104,16 @@ def remove_allocation_from_read_model( dict(orderid=event.orderid, sku=event.sku), ) uow.commit() + + +EVENT_HANDLERS = { + events.Allocated: [publish_allocated_event, add_allocation_to_read_model], + events.Deallocated: [remove_allocation_from_read_model, reallocate], + events.OutOfStock: [send_out_of_stock_notification], +} # type: Dict[Type[events.Event], List[Callable]] + +COMMAND_HANDLERS = { + commands.Allocate: allocate, + commands.CreateBatch: add_batch, + commands.ChangeBatchQuantity: change_batch_quantity, +} # type: Dict[Type[commands.Command], Callable] diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 104111fa..12b2965b 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,7 +1,7 @@ # pylint: disable=broad-except from __future__ import annotations import logging -from typing import List, Dict, Callable, Type, Union, TYPE_CHECKING +from typing import Union, TYPE_CHECKING from allocation.domain import commands, events from . import handlers @@ -53,22 +53,3 @@ def handle_command( except Exception: logger.exception("Exception handling command %s", command) raise - - -EVENT_HANDLERS = { - events.Allocated: [ - handlers.publish_allocated_event, - handlers.add_allocation_to_read_model, - ], - events.Deallocated: [ - handlers.remove_allocation_from_read_model, - handlers.reallocate, - ], - events.OutOfStock: [handlers.send_out_of_stock_notification], -} # type: Dict[Type[events.Event], List[Callable]] - -COMMAND_HANDLERS = { - commands.Allocate: handlers.allocate, - commands.CreateBatch: handlers.add_batch, - commands.ChangeBatchQuantity: handlers.change_batch_quantity, -} # type: Dict[Type[commands.Command], Callable] From 1da2541b7916cdacd178f7c06340ca26fafdd1bb Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 5 Jan 2020 17:18:11 +0000 Subject: [PATCH 121/145] messagebus becomes a class, requires handlers [messagebus_as_class] --- src/allocation/service_layer/messagebus.py | 34 ++++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 12b2965b..d210d66f 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,9 +1,8 @@ # pylint: disable=broad-except from __future__ import annotations import logging -from typing import Union, TYPE_CHECKING +from typing import Callable, Dict, List, Union, Type, TYPE_CHECKING from allocation.domain import commands, events -from . import handlers if TYPE_CHECKING: from . import unit_of_work @@ -13,16 +12,27 @@ Message = Union[commands.Command, events.Event] -def handle(message: Message, uow: unit_of_work.AbstractUnitOfWork): - queue = [message] - while queue: - message = queue.pop(0) - if isinstance(message, events.Event): - handle_event(message, queue, uow) - elif isinstance(message, commands.Command): - handle_command(message, queue, uow) - else: - raise Exception(f"{message} was not an Event or Command") +class MessageBus: + def __init__( + self, + uow: unit_of_work.AbstractUnitOfWork, + event_handlers: Dict[Type[events.Event], List[Callable]], + command_handlers: Dict[Type[commands.Command], Callable], + ): + self.uow = uow + self.event_handlers = event_handlers + self.command_handlers = command_handlers + + def handle(self, message: Message): + self.queue = [message] + while self.queue: + message = self.queue.pop(0) + if isinstance(message, events.Event): + self.handle_event(message) + elif isinstance(message, commands.Command): + self.handle_command(message) + else: + raise Exception(f"{message} was not an Event or Command") def handle_event( From e9ae6dbd85deaf9893a847d8222d5cdeb295954c Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 14:48:53 +0000 Subject: [PATCH 122/145] use self.handlers for handle_event and handle_command [messagebus_handlers_change] --- src/allocation/service_layer/messagebus.py | 46 +++++++++------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index d210d66f..45679341 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,4 +1,4 @@ -# pylint: disable=broad-except +# pylint: disable=broad-except, attribute-defined-outside-init from __future__ import annotations import logging from typing import Callable, Dict, List, Union, Type, TYPE_CHECKING @@ -34,32 +34,22 @@ def handle(self, message: Message): else: raise Exception(f"{message} was not an Event or Command") - -def handle_event( - event: events.Event, - queue: List[Message], - uow: unit_of_work.AbstractUnitOfWork, -): - for handler in EVENT_HANDLERS[type(event)]: + def handle_event(self, event: events.Event): + for handler in self.event_handlers[type(event)]: + try: + logger.debug("handling event %s with handler %s", event, handler) + handler(event) + self.queue.extend(self.uow.collect_new_events()) + except Exception: + logger.exception("Exception handling event %s", event) + continue + + def handle_command(self, command: commands.Command): + logger.debug("handling command %s", command) try: - logger.debug("handling event %s with handler %s", event, handler) - handler(event, uow=uow) - queue.extend(uow.collect_new_events()) + handler = self.command_handlers[type(command)] + handler(command) + self.queue.extend(self.uow.collect_new_events()) except Exception: - logger.exception("Exception handling event %s", event) - continue - - -def handle_command( - command: commands.Command, - queue: List[Message], - uow: unit_of_work.AbstractUnitOfWork, -): - logger.debug("handling command %s", command) - try: - handler = COMMAND_HANDLERS[type(command)] - handler(command, uow=uow) - queue.extend(uow.collect_new_events()) - except Exception: - logger.exception("Exception handling command %s", command) - raise + logger.exception("Exception handling command %s", command) + raise From 82d79086ed30d1d5146ea1074dab09a668b7d61a Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:08:54 +0100 Subject: [PATCH 123/145] conftest change to backport, session_factory -> sqlite_session_factory --- tests/conftest.py | 11 +++-------- tests/integration/test_repository.py | 5 +++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3b2a8066..8c1efcf7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,24 +18,19 @@ @pytest.fixture -def in_memory_db(): +def in_memory_sqlite_db(): engine = create_engine("sqlite:///:memory:") metadata.create_all(engine) return engine @pytest.fixture -def sqlite_session_factory(in_memory_db): +def sqlite_session_factory(in_memory_sqlite_db): start_mappers() - yield sessionmaker(bind=in_memory_db) + yield sessionmaker(bind=in_memory_sqlite_db) clear_mappers() -@pytest.fixture -def sqlite_session(sqlite_session_factory): - return sqlite_session_factory() - - @retry(stop=stop_after_delay(10)) def wait_for_postgres_to_come_up(engine): return engine.connect() diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index 9b637f1a..245b26f2 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -2,8 +2,9 @@ from allocation.domain import model -def test_get_by_batchref(sqlite_session): - repo = repository.SqlAlchemyRepository(sqlite_session) +def test_get_by_batchref(sqlite_session_factory): + session = sqlite_session_factory() + repo = repository.SqlAlchemyRepository(session) b1 = model.Batch(ref="b1", sku="sku1", qty=100, eta=None) b2 = model.Batch(ref="b2", sku="sku1", qty=100, eta=None) b3 = model.Batch(ref="b3", sku="sku2", qty=100, eta=None) From 36b2c82aa36dcb5214db90a5ec0b68ebc78c50d1 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:09:18 +0100 Subject: [PATCH 124/145] conftest change to backport to ch 5, isolation serializable in all pg tests --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8c1efcf7..b0ceb0f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,7 +49,7 @@ def wait_for_redis_to_come_up(): @pytest.fixture(scope="session") def postgres_db(): - engine = create_engine(config.get_postgres_uri()) + engine = create_engine(config.get_postgres_uri(), isolation_level="SERIALIZABLE") wait_for_postgres_to_come_up(engine) metadata.create_all(engine) return engine From 5b055701740ec9fcf190dd5d5b8faee2b61a498d Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:09:59 +0100 Subject: [PATCH 125/145] uow tests maybe backport, pass explicit uow to threads --- tests/integration/test_uow.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 55c4f24d..db705849 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,4 +1,4 @@ -# pylint: disable=broad-except +# pylint: disable=broad-except, too-many-arguments import threading import time import traceback @@ -75,15 +75,15 @@ class MyException(Exception): assert rows == [] -def try_to_allocate(orderid, sku, exceptions): +def try_to_allocate(orderid, sku, exceptions, session_factory): line = model.OrderLine(orderid, sku, 10) try: - with unit_of_work.SqlAlchemyUnitOfWork() as uow: + with unit_of_work.SqlAlchemyUnitOfWork(session_factory) as uow: product = uow.products.get(sku=sku) product.allocate(line) time.sleep(0.2) uow.commit() - except Exception as e: + except Exception as e: # pylint: disable=broad-except print(traceback.format_exc()) exceptions.append(e) @@ -96,8 +96,12 @@ def test_concurrent_updates_to_version_are_not_allowed(postgres_session_factory) order1, order2 = random_orderid(1), random_orderid(2) exceptions = [] # type: List[Exception] - try_to_allocate_order1 = lambda: try_to_allocate(order1, sku, exceptions) - try_to_allocate_order2 = lambda: try_to_allocate(order2, sku, exceptions) + try_to_allocate_order1 = lambda: try_to_allocate( + order1, sku, exceptions, postgres_session_factory + ) + try_to_allocate_order2 = lambda: try_to_allocate( + order2, sku, exceptions, postgres_session_factory + ) thread1 = threading.Thread(target=try_to_allocate_order1) thread2 = threading.Thread(target=try_to_allocate_order2) thread1.start() @@ -121,5 +125,5 @@ def test_concurrent_updates_to_version_are_not_allowed(postgres_session_factory) dict(sku=sku), ) assert orders.rowcount == 1 - with unit_of_work.SqlAlchemyUnitOfWork() as uow: + with unit_of_work.SqlAlchemyUnitOfWork(postgres_session_factory) as uow: uow.session.execute("select 1") From c3282c9fb6a74e8eb6e1ac7424606eea642c11eb Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 7 Oct 2019 14:29:59 +0100 Subject: [PATCH 126/145] bootstrap script preps DI'd handlers and start orm [bootstrap_script] --- src/allocation/bootstrap.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/allocation/bootstrap.py diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py new file mode 100644 index 00000000..805b13f5 --- /dev/null +++ b/src/allocation/bootstrap.py @@ -0,0 +1,44 @@ +import inspect +from typing import Callable +from allocation.adapters import email, orm, redis_eventpublisher +from allocation.service_layer import handlers, messagebus, unit_of_work + + +def bootstrap( + start_orm: bool = True, + uow: unit_of_work.AbstractUnitOfWork = unit_of_work.SqlAlchemyUnitOfWork(), + send_mail: Callable = email.send, + publish: Callable = redis_eventpublisher.publish, +) -> messagebus.MessageBus: + + if start_orm: + orm.start_mappers() + + dependencies = {"uow": uow, "send_mail": send_mail, "publish": publish} + injected_event_handlers = { + event_type: [ + inject_dependencies(handler, dependencies) + for handler in event_handlers + ] + for event_type, event_handlers in handlers.EVENT_HANDLERS.items() + } + injected_command_handlers = { + command_type: inject_dependencies(handler, dependencies) + for command_type, handler in handlers.COMMAND_HANDLERS.items() + } + + return messagebus.MessageBus( + uow=uow, + event_handlers=injected_event_handlers, + command_handlers=injected_command_handlers, + ) + + +def inject_dependencies(handler, dependencies): + params = inspect.signature(handler).parameters + deps = { + name: dependency + for name, dependency in dependencies.items() + if name in params + } + return lambda message: handler(message, **deps) From f92376002916f73e61aa0f76c93222038d93c9fd Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:03:43 +0100 Subject: [PATCH 127/145] use bootstrap in service layer tests [bootstrap_tests] --- tests/unit/test_handlers.py | 110 +++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 04fb9630..cb21461e 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,10 +1,12 @@ # pylint: disable=no-self-use +from __future__ import annotations from datetime import date from unittest import mock import pytest +from allocation import bootstrap from allocation.adapters import repository -from allocation.domain import commands, events -from allocation.service_layer import handlers, messagebus, unit_of_work +from allocation.domain import commands +from allocation.service_layer import handlers, unit_of_work class FakeRepository(repository.AbstractRepository): @@ -37,81 +39,83 @@ def rollback(self): pass +def bootstrap_test_app(): + return bootstrap.bootstrap( + start_orm=False, + uow=FakeUnitOfWork(), + send_mail=lambda *args: None, + publish=lambda *args: None, + ) + + class TestAddBatch: def test_for_new_product(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("b1", "CRUNCHY-ARMCHAIR", 100, None), uow - ) - assert uow.products.get("CRUNCHY-ARMCHAIR") is not None - assert uow.committed + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "CRUNCHY-ARMCHAIR", 100, None)) + assert bus.uow.products.get("CRUNCHY-ARMCHAIR") is not None + assert bus.uow.committed def test_for_existing_product(self): - uow = FakeUnitOfWork() - messagebus.handle(commands.CreateBatch("b1", "GARISH-RUG", 100, None), uow) - messagebus.handle(commands.CreateBatch("b2", "GARISH-RUG", 99, None), uow) - assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] - - -@pytest.fixture(autouse=True) -def fake_redis_publish(): - with mock.patch("allocation.adapters.redis_eventpublisher.publish"): - yield + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "GARISH-RUG", 100, None)) + bus.handle(commands.CreateBatch("b2", "GARISH-RUG", 99, None)) + assert "b2" in [ + b.reference for b in bus.uow.products.get("GARISH-RUG").batches + ] class TestAllocate: def test_allocates(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None), uow - ) - messagebus.handle(commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow) - [batch] = uow.products.get("COMPLICATED-LAMP").batches + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None)) + bus.handle(commands.Allocate("o1", "COMPLICATED-LAMP", 10)) + [batch] = bus.uow.products.get("COMPLICATED-LAMP").batches assert batch.available_quantity == 90 def test_errors_for_invalid_sku(self): - uow = FakeUnitOfWork() - messagebus.handle(commands.CreateBatch("b1", "AREALSKU", 100, None), uow) + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "AREALSKU", 100, None)) with pytest.raises(handlers.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - messagebus.handle(commands.Allocate("o1", "NONEXISTENTSKU", 10), uow) + bus.handle(commands.Allocate("o1", "NONEXISTENTSKU", 10)) def test_commits(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("b1", "OMINOUS-MIRROR", 100, None), uow - ) - messagebus.handle(commands.Allocate("o1", "OMINOUS-MIRROR", 10), uow) - assert uow.committed + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "OMINOUS-MIRROR", 100, None)) + bus.handle(commands.Allocate("o1", "OMINOUS-MIRROR", 10)) + assert bus.uow.committed def test_sends_email_on_out_of_stock_error(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None), uow - ) + emails = [] - with mock.patch("allocation.adapters.email.send") as mock_send_mail: - messagebus.handle(commands.Allocate("o1", "POPULAR-CURTAINS", 10), uow) - assert mock_send_mail.call_args == mock.call( - "stock@made.com", f"Out of stock for POPULAR-CURTAINS" - ) + def fake_send_mail(*args): + emails.append(args) + + bus = bootstrap.bootstrap( + start_orm=False, + uow=FakeUnitOfWork(), + send_mail=fake_send_mail, + publish=lambda *args: None, + ) + bus.handle(commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None)) + bus.handle(commands.Allocate("o1", "POPULAR-CURTAINS", 10)) + assert emails == [ + ("stock@made.com", f"Out of stock for POPULAR-CURTAINS"), + ] class TestChangeBatchQuantity: def test_changes_available_quantity(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("batch1", "ADORABLE-SETTEE", 100, None), uow - ) - [batch] = uow.products.get(sku="ADORABLE-SETTEE").batches + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("batch1", "ADORABLE-SETTEE", 100, None)) + [batch] = bus.uow.products.get(sku="ADORABLE-SETTEE").batches assert batch.available_quantity == 100 - messagebus.handle(commands.ChangeBatchQuantity("batch1", 50), uow) - + bus.handle(commands.ChangeBatchQuantity("batch1", 50)) assert batch.available_quantity == 50 def test_reallocates_if_necessary(self): - uow = FakeUnitOfWork() + bus = bootstrap_test_app() history = [ commands.CreateBatch("batch1", "INDIFFERENT-TABLE", 50, None), commands.CreateBatch("batch2", "INDIFFERENT-TABLE", 50, date.today()), @@ -119,12 +123,12 @@ def test_reallocates_if_necessary(self): commands.Allocate("order2", "INDIFFERENT-TABLE", 20), ] for msg in history: - messagebus.handle(msg, uow) - [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches + bus.handle(msg) + [batch1, batch2] = bus.uow.products.get(sku="INDIFFERENT-TABLE").batches assert batch1.available_quantity == 10 assert batch2.available_quantity == 50 - messagebus.handle(commands.ChangeBatchQuantity("batch1", 25), uow) + bus.handle(commands.ChangeBatchQuantity("batch1", 25)) # order1 or order2 will be deallocated, so we'll have 25 - 20 assert batch1.available_quantity == 5 From e883f2049f24805b21f9a606d5db6f99e9935a42 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 07:32:55 +0000 Subject: [PATCH 128/145] fixture to start mappers explicitly, use in repo and uow tests --- tests/conftest.py | 9 ++++++--- tests/integration/test_repository.py | 3 +++ tests/integration/test_uow.py | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b0ceb0f2..f91f93a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,8 +26,13 @@ def in_memory_sqlite_db(): @pytest.fixture def sqlite_session_factory(in_memory_sqlite_db): - start_mappers() yield sessionmaker(bind=in_memory_sqlite_db) + + +@pytest.fixture +def mappers(): + start_mappers() + yield clear_mappers() @@ -57,9 +62,7 @@ def postgres_db(): @pytest.fixture def postgres_session_factory(postgres_db): - start_mappers() yield sessionmaker(bind=postgres_db) - clear_mappers() @pytest.fixture diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index 245b26f2..7961be2a 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,6 +1,9 @@ +import pytest from allocation.adapters import repository from allocation.domain import model +pytestmark = pytest.mark.usefixtures("mappers") + def test_get_by_batchref(sqlite_session_factory): session = sqlite_session_factory() diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index db705849..61f3aae5 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -3,11 +3,14 @@ import time import traceback from typing import List +from unittest.mock import Mock import pytest from allocation.domain import model from allocation.service_layer import unit_of_work from ..random_refs import random_sku, random_batchref, random_orderid +pytestmark = pytest.mark.usefixtures("mappers") + def insert_batch(session, ref, sku, qty, eta, product_version=1): session.execute( From c6f9429aa8116133c0bf86e2a2fb045a35472f62 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 05:07:03 +0000 Subject: [PATCH 129/145] fix view tests to use bootstrap. [bootstrap_view_tests] --- tests/integration/test_views.py | 51 +++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 4dda3e54..1eea394e 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,35 +1,48 @@ +# pylint: disable=redefined-outer-name from datetime import date -from allocation import views +from sqlalchemy.orm import clear_mappers +import pytest +from allocation import bootstrap, views from allocation.domain import commands -from allocation.service_layer import messagebus, unit_of_work +from allocation.service_layer import unit_of_work today = date.today() -def test_allocations_view(sqlite_session_factory): - uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) - messagebus.handle(commands.CreateBatch("sku1batch", "sku1", 50, None), uow) - messagebus.handle(commands.CreateBatch("sku2batch", "sku2", 50, today), uow) - messagebus.handle(commands.Allocate("order1", "sku1", 20), uow) - messagebus.handle(commands.Allocate("order1", "sku2", 20), uow) +@pytest.fixture +def sqlite_bus(sqlite_session_factory): + bus = bootstrap.bootstrap( + start_orm=True, + uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory), + send_mail=lambda *args: None, + publish=lambda *args: None, + ) + yield bus + clear_mappers() + + +def test_allocations_view(sqlite_bus): + sqlite_bus.handle(commands.CreateBatch("sku1batch", "sku1", 50, None)) + sqlite_bus.handle(commands.CreateBatch("sku2batch", "sku2", 50, today)) + sqlite_bus.handle(commands.Allocate("order1", "sku1", 20)) + sqlite_bus.handle(commands.Allocate("order1", "sku2", 20)) # add a spurious batch and order to make sure we're getting the right ones - messagebus.handle(commands.CreateBatch("sku1batch-later", "sku1", 50, today), uow) - messagebus.handle(commands.Allocate("otherorder", "sku1", 30), uow) - messagebus.handle(commands.Allocate("otherorder", "sku2", 10), uow) + sqlite_bus.handle(commands.CreateBatch("sku1batch-later", "sku1", 50, today)) + sqlite_bus.handle(commands.Allocate("otherorder", "sku1", 30)) + sqlite_bus.handle(commands.Allocate("otherorder", "sku2", 10)) - assert views.allocations("order1", uow) == [ + assert views.allocations("order1", sqlite_bus.uow) == [ {"sku": "sku1", "batchref": "sku1batch"}, {"sku": "sku2", "batchref": "sku2batch"}, ] -def test_deallocation(sqlite_session_factory): - uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) - messagebus.handle(commands.CreateBatch("b1", "sku1", 50, None), uow) - messagebus.handle(commands.CreateBatch("b2", "sku1", 50, today), uow) - messagebus.handle(commands.Allocate("o1", "sku1", 40), uow) - messagebus.handle(commands.ChangeBatchQuantity("b1", 10), uow) +def test_deallocation(sqlite_bus): + sqlite_bus.handle(commands.CreateBatch("b1", "sku1", 50, None)) + sqlite_bus.handle(commands.CreateBatch("b2", "sku1", 50, today)) + sqlite_bus.handle(commands.Allocate("o1", "sku1", 40)) + sqlite_bus.handle(commands.ChangeBatchQuantity("b1", 10)) - assert views.allocations("o1", uow) == [ + assert views.allocations("o1", sqlite_bus.uow) == [ {"sku": "sku1", "batchref": "b2"}, ] From d4a867494a4338aa3b67f1f80ed2f1fced98672f Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 7 Oct 2019 14:44:15 +0100 Subject: [PATCH 130/145] use bootstrap in flask [flask_calls_bootstrap] --- src/allocation/entrypoints/flask_app.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index ed482390..f50f3edd 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,14 +1,11 @@ from datetime import datetime from flask import Flask, jsonify, request - from allocation.domain import commands -from allocation.adapters import orm -from allocation.service_layer import messagebus, unit_of_work from allocation.service_layer.handlers import InvalidSku -from allocation import views +from allocation import bootstrap, views app = Flask(__name__) -orm.start_mappers() +bus = bootstrap.bootstrap() @app.route("/add_batch", methods=["POST"]) @@ -19,8 +16,7 @@ def add_batch(): cmd = commands.CreateBatch( request.json["ref"], request.json["sku"], request.json["qty"], eta ) - uow = unit_of_work.SqlAlchemyUnitOfWork() - messagebus.handle(cmd, uow) + bus.handle(cmd) return "OK", 201 @@ -30,8 +26,7 @@ def allocate_endpoint(): cmd = commands.Allocate( request.json["orderid"], request.json["sku"], request.json["qty"] ) - uow = unit_of_work.SqlAlchemyUnitOfWork() - messagebus.handle(cmd, uow) + bus.handle(cmd) except InvalidSku as e: return {"message": str(e)}, 400 @@ -40,8 +35,7 @@ def allocate_endpoint(): @app.route("/allocations/", methods=["GET"]) def allocations_view_endpoint(orderid): - uow = unit_of_work.SqlAlchemyUnitOfWork() - result = views.allocations(orderid, uow) + result = views.allocations(orderid, bus.uow) if not result: return "not found", 404 return jsonify(result), 200 From 4ca05b0d65f558bb3cd09a9efb6b735f725d16a9 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 06:59:59 +0000 Subject: [PATCH 131/145] use bootstrap for redis --- src/allocation/entrypoints/redis_eventconsumer.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py index e04a8142..9b5b93d0 100644 --- a/src/allocation/entrypoints/redis_eventconsumer.py +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -2,10 +2,8 @@ import logging import redis -from allocation import config +from allocation import bootstrap, config from allocation.domain import commands -from allocation.adapters import orm -from allocation.service_layer import messagebus, unit_of_work logger = logging.getLogger(__name__) @@ -13,19 +11,19 @@ def main(): - orm.start_mappers() + bus = bootstrap.bootstrap() pubsub = r.pubsub(ignore_subscribe_messages=True) pubsub.subscribe("change_batch_quantity") for m in pubsub.listen(): - handle_change_batch_quantity(m) + handle_change_batch_quantity(m, bus) -def handle_change_batch_quantity(m): +def handle_change_batch_quantity(m, bus): logging.debug("handling %s", m) data = json.loads(m["data"]) cmd = commands.ChangeBatchQuantity(ref=data["batchref"], qty=data["qty"]) - messagebus.handle(cmd, uow=unit_of_work.SqlAlchemyUnitOfWork()) + bus.handle(cmd) if __name__ == "__main__": From 5f9b143583c3d784543c582370cb1fbd2a5a77b3 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 22:32:52 +0000 Subject: [PATCH 132/145] experiment with nonmagic DI zzzz [nomagic_di] --- src/allocation/bootstrap.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py index 805b13f5..94f26dd1 100644 --- a/src/allocation/bootstrap.py +++ b/src/allocation/bootstrap.py @@ -1,5 +1,6 @@ import inspect from typing import Callable +from allocation.domain import commands, events from allocation.adapters import email, orm, redis_eventpublisher from allocation.service_layer import handlers, messagebus, unit_of_work @@ -14,17 +15,24 @@ def bootstrap( if start_orm: orm.start_mappers() - dependencies = {"uow": uow, "send_mail": send_mail, "publish": publish} injected_event_handlers = { - event_type: [ - inject_dependencies(handler, dependencies) - for handler in event_handlers - ] - for event_type, event_handlers in handlers.EVENT_HANDLERS.items() + events.Allocated: [ + lambda e: handlers.publish_allocated_event(e, publish), + lambda e: handlers.add_allocation_to_read_model(e, uow), + ], + events.Deallocated: [ + lambda e: handlers.remove_allocation_from_read_model(e, uow), + lambda e: handlers.reallocate(e, uow), + ], + events.OutOfStock: [ + lambda e: handlers.send_out_of_stock_notification(e, send_mail) + ], } injected_command_handlers = { - command_type: inject_dependencies(handler, dependencies) - for command_type, handler in handlers.COMMAND_HANDLERS.items() + commands.Allocate: lambda c: handlers.allocate(c, uow), + commands.CreateBatch: lambda c: handlers.add_batch(c, uow), + commands.ChangeBatchQuantity: \ + lambda c: handlers.change_batch_quantity(c, uow), } return messagebus.MessageBus( From 2e5e5de872a88e5fcc7522ec37106dbbdabf862b Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 22:36:23 +0000 Subject: [PATCH 133/145] Revert "experiment with nonmagic DI zzzz" This reverts commit f42b193857eec8560443767de9fd7ada9f2acb96. --- src/allocation/bootstrap.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py index 94f26dd1..805b13f5 100644 --- a/src/allocation/bootstrap.py +++ b/src/allocation/bootstrap.py @@ -1,6 +1,5 @@ import inspect from typing import Callable -from allocation.domain import commands, events from allocation.adapters import email, orm, redis_eventpublisher from allocation.service_layer import handlers, messagebus, unit_of_work @@ -15,24 +14,17 @@ def bootstrap( if start_orm: orm.start_mappers() + dependencies = {"uow": uow, "send_mail": send_mail, "publish": publish} injected_event_handlers = { - events.Allocated: [ - lambda e: handlers.publish_allocated_event(e, publish), - lambda e: handlers.add_allocation_to_read_model(e, uow), - ], - events.Deallocated: [ - lambda e: handlers.remove_allocation_from_read_model(e, uow), - lambda e: handlers.reallocate(e, uow), - ], - events.OutOfStock: [ - lambda e: handlers.send_out_of_stock_notification(e, send_mail) - ], + event_type: [ + inject_dependencies(handler, dependencies) + for handler in event_handlers + ] + for event_type, event_handlers in handlers.EVENT_HANDLERS.items() } injected_command_handlers = { - commands.Allocate: lambda c: handlers.allocate(c, uow), - commands.CreateBatch: lambda c: handlers.add_batch(c, uow), - commands.ChangeBatchQuantity: \ - lambda c: handlers.change_batch_quantity(c, uow), + command_type: inject_dependencies(handler, dependencies) + for command_type, handler in handlers.COMMAND_HANDLERS.items() } return messagebus.MessageBus( From 09c0214e55b6d1781f577518c111e80463d7319f Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 02:55:46 +0100 Subject: [PATCH 134/145] switch to a notifications class [notifications_class] --- src/allocation/adapters/email.py | 2 -- src/allocation/adapters/notifications.py | 28 ++++++++++++++++++++++++ src/allocation/bootstrap.py | 13 ++++++++--- src/allocation/config.py | 7 ++++++ src/allocation/service_layer/handlers.py | 5 +++-- 5 files changed, 48 insertions(+), 7 deletions(-) delete mode 100644 src/allocation/adapters/email.py create mode 100644 src/allocation/adapters/notifications.py diff --git a/src/allocation/adapters/email.py b/src/allocation/adapters/email.py deleted file mode 100644 index 1c37d427..00000000 --- a/src/allocation/adapters/email.py +++ /dev/null @@ -1,2 +0,0 @@ -def send(*args): - print("SENDING EMAIL:", *args) diff --git a/src/allocation/adapters/notifications.py b/src/allocation/adapters/notifications.py new file mode 100644 index 00000000..db29f7c8 --- /dev/null +++ b/src/allocation/adapters/notifications.py @@ -0,0 +1,28 @@ +# pylint: disable=too-few-public-methods +import abc +import smtplib +from allocation import config + + +class AbstractNotifications(abc.ABC): + @abc.abstractmethod + def send(self, destination, message): + raise NotImplementedError + + +DEFAULT_HOST = config.get_email_host_and_port()["host"] +DEFAULT_PORT = config.get_email_host_and_port()["port"] + + +class EmailNotifications(AbstractNotifications): + def __init__(self, smtp_host=DEFAULT_HOST, port=DEFAULT_PORT): + self.server = smtplib.SMTP(smtp_host, port=port) + self.server.noop() + + def send(self, destination, message): + msg = f"Subject: allocation service notification\n{message}" + self.server.sendmail( + from_addr="allocations@example.com", + to_addrs=[destination], + msg=msg, + ) diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py index 805b13f5..22112a06 100644 --- a/src/allocation/bootstrap.py +++ b/src/allocation/bootstrap.py @@ -1,20 +1,27 @@ import inspect from typing import Callable -from allocation.adapters import email, orm, redis_eventpublisher +from allocation.adapters import orm, redis_eventpublisher +from allocation.adapters.notifications import ( + AbstractNotifications, + EmailNotifications, +) from allocation.service_layer import handlers, messagebus, unit_of_work def bootstrap( start_orm: bool = True, uow: unit_of_work.AbstractUnitOfWork = unit_of_work.SqlAlchemyUnitOfWork(), - send_mail: Callable = email.send, + notifications: AbstractNotifications = None, publish: Callable = redis_eventpublisher.publish, ) -> messagebus.MessageBus: + if notifications is None: + notifications = EmailNotifications() + if start_orm: orm.start_mappers() - dependencies = {"uow": uow, "send_mail": send_mail, "publish": publish} + dependencies = {"uow": uow, "notifications": notifications, "publish": publish} injected_event_handlers = { event_type: [ inject_dependencies(handler, dependencies) diff --git a/src/allocation/config.py b/src/allocation/config.py index 30a8eb07..bda1bbf2 100644 --- a/src/allocation/config.py +++ b/src/allocation/config.py @@ -19,3 +19,10 @@ def get_redis_host_and_port(): host = os.environ.get("REDIS_HOST", "localhost") port = 63791 if host == "localhost" else 6379 return dict(host=host, port=port) + + +def get_email_host_and_port(): + host = os.environ.get("EMAIL_HOST", "localhost") + port = 11025 if host == "localhost" else 1025 + http_port = 18025 if host == "localhost" else 8025 + return dict(host=host, port=port, http_port=http_port) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 6ab2bbe9..2d7aa8d4 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -6,6 +6,7 @@ from allocation.domain.model import OrderLine if TYPE_CHECKING: + from allocation.adapters import notifications from . import unit_of_work @@ -61,9 +62,9 @@ def change_batch_quantity( def send_out_of_stock_notification( event: events.OutOfStock, - send_mail: Callable, + notifications: notifications.AbstractNotifications, ): - send_mail( + notifications.send( "stock@made.com", f"Out of stock for {event.sku}", ) From 0943337cb8197a4d56b8f1f29935f76fab533438 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 02:56:00 +0100 Subject: [PATCH 135/145] tests for notifcations [notifications_unit_tests] --- tests/integration/test_views.py | 3 ++- tests/unit/test_handlers.py | 30 ++++++++++++++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 1eea394e..ccd5d542 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,6 +1,7 @@ # pylint: disable=redefined-outer-name from datetime import date from sqlalchemy.orm import clear_mappers +from unittest import mock import pytest from allocation import bootstrap, views from allocation.domain import commands @@ -14,7 +15,7 @@ def sqlite_bus(sqlite_session_factory): bus = bootstrap.bootstrap( start_orm=True, uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory), - send_mail=lambda *args: None, + notifications=mock.Mock(), publish=lambda *args: None, ) yield bus diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index cb21461e..f1218540 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,12 +1,14 @@ # pylint: disable=no-self-use from __future__ import annotations +from collections import defaultdict from datetime import date -from unittest import mock +from typing import Dict, List import pytest from allocation import bootstrap -from allocation.adapters import repository from allocation.domain import commands -from allocation.service_layer import handlers, unit_of_work +from allocation.service_layer import handlers +from allocation.adapters import notifications, repository +from allocation.service_layer import unit_of_work class FakeRepository(repository.AbstractRepository): @@ -39,11 +41,19 @@ def rollback(self): pass +class FakeNotifications(notifications.AbstractNotifications): + def __init__(self): + self.sent = defaultdict(list) # type: Dict[str, List[str]] + + def send(self, destination, message): + self.sent[destination].append(message) + + def bootstrap_test_app(): return bootstrap.bootstrap( start_orm=False, uow=FakeUnitOfWork(), - send_mail=lambda *args: None, + notifications=FakeNotifications(), publish=lambda *args: None, ) @@ -86,21 +96,17 @@ def test_commits(self): assert bus.uow.committed def test_sends_email_on_out_of_stock_error(self): - emails = [] - - def fake_send_mail(*args): - emails.append(args) - + fake_notifs = FakeNotifications() bus = bootstrap.bootstrap( start_orm=False, uow=FakeUnitOfWork(), - send_mail=fake_send_mail, + notifications=fake_notifs, publish=lambda *args: None, ) bus.handle(commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None)) bus.handle(commands.Allocate("o1", "POPULAR-CURTAINS", 10)) - assert emails == [ - ("stock@made.com", f"Out of stock for POPULAR-CURTAINS"), + assert fake_notifs.sent["stock@made.com"] == [ + f"Out of stock for POPULAR-CURTAINS", ] From 3f1d63d3a2819c917d78832671b9fb84585c755b Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 03:23:07 +0100 Subject: [PATCH 136/145] add a mailhog fake email server to docker-compose --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index dc2cc369..f964ab74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,10 +10,12 @@ services: depends_on: - postgres - redis + - mailhog environment: - DB_HOST=postgres - DB_PASSWORD=abc123 - REDIS_HOST=redis + - EMAIL_HOST=mailhog - PYTHONDONTWRITEBYTECODE=1 volumes: - ./src:/src @@ -26,11 +28,13 @@ services: image: allocation-image depends_on: - redis_pubsub + - mailhog environment: - DB_HOST=postgres - DB_PASSWORD=abc123 - API_HOST=api - REDIS_HOST=redis + - EMAIL_HOST=mailhog - PYTHONDONTWRITEBYTECODE=1 - FLASK_APP=allocation/entrypoints/flask_app.py - FLASK_DEBUG=1 @@ -59,3 +63,8 @@ services: ports: - "63791:6379" + mailhog: + image: mailhog/mailhog + ports: + - "11025:1025" + - "18025:8025" From c112af908849cc645e4a6f29e4b00495ca9864e1 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 8 Oct 2019 12:56:58 +0100 Subject: [PATCH 137/145] logging.info needs backport --- src/allocation/adapters/orm.py | 3 +++ src/allocation/adapters/redis_eventpublisher.py | 2 +- src/allocation/entrypoints/redis_eventconsumer.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index a87068a5..81d704ca 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -1,3 +1,4 @@ +import logging from sqlalchemy import ( Table, MetaData, @@ -12,6 +13,7 @@ from allocation.domain import model +logger = logging.getLogger(__name__) metadata = MetaData() @@ -59,6 +61,7 @@ def start_mappers(): + logger.info("Starting mappers") lines_mapper = mapper(model.OrderLine, order_lines) batches_mapper = mapper( model.Batch, diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py index 6100956f..d607d6ac 100644 --- a/src/allocation/adapters/redis_eventpublisher.py +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -12,5 +12,5 @@ def publish(channel, event: events.Event): - logging.debug("publishing: channel=%s, event=%s", channel, event) + logging.info("publishing: channel=%s, event=%s", channel, event) r.publish(channel, json.dumps(asdict(event))) diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py index 9b5b93d0..6d0d49a7 100644 --- a/src/allocation/entrypoints/redis_eventconsumer.py +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -11,6 +11,7 @@ def main(): + logger.info("Redis pubsub starting") bus = bootstrap.bootstrap() pubsub = r.pubsub(ignore_subscribe_messages=True) pubsub.subscribe("change_batch_quantity") @@ -20,7 +21,7 @@ def main(): def handle_change_batch_quantity(m, bus): - logging.debug("handling %s", m) + logger.info("handling %s", m) data = json.loads(m["data"]) cmd = commands.ChangeBatchQuantity(ref=data["batchref"], qty=data["qty"]) bus.handle(cmd) From 5d8136c646d28dad43696509082fddb9fe416f25 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 03:23:44 +0100 Subject: [PATCH 138/145] integration test for emails [chapter_13_dependency_injection_ends] --- tests/integration/test_email.py | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/integration/test_email.py diff --git a/tests/integration/test_email.py b/tests/integration/test_email.py new file mode 100644 index 00000000..4aade37b --- /dev/null +++ b/tests/integration/test_email.py @@ -0,0 +1,37 @@ +# pylint: disable=redefined-outer-name +import pytest +import requests +from sqlalchemy.orm import clear_mappers +from allocation import bootstrap, config +from allocation.domain import commands +from allocation.adapters import notifications +from allocation.service_layer import unit_of_work +from ..random_refs import random_sku + + +@pytest.fixture +def bus(sqlite_session_factory): + bus = bootstrap.bootstrap( + start_orm=True, + uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory), + notifications=notifications.EmailNotifications(), + publish=lambda *args: None, + ) + yield bus + clear_mappers() + + +def get_email_from_mailhog(sku): + host, port = map(config.get_email_host_and_port().get, ["host", "http_port"]) + all_emails = requests.get(f"http://{host}:{port}/api/v2/messages").json() + return next(m for m in all_emails["items"] if sku in str(m)) + + +def test_out_of_stock_email(bus): + sku = random_sku() + bus.handle(commands.CreateBatch("batch1", sku, 9, None)) + bus.handle(commands.Allocate("order1", sku, 10)) + email = get_email_from_mailhog(sku) + assert email["Raw"]["From"] == "allocations@example.com" + assert email["Raw"]["To"] == ["stock@made.com"] + assert f"Out of stock for {sku}" in email["Raw"]["Data"] From 2c62b9d1ed973462f3d6c7cbb4725d16bd6a9955 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 25 Feb 2021 01:02:38 +0000 Subject: [PATCH 139/145] update travis config --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcd3ceea..bf0b9de2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ dist: xenial language: python python: 3.8 - script: - make all - branches: except: - - /.*_exercise$/ + - "/.*_exercise$/" +env: + global: + secure: cNctlzVCjUj1oOrRW0aryxhJHB/u0b6vmn532jcgyCRS1aQjMJtT61O7tW6yMk4wOaH4Lr0kAI0J6+1lnRYf5g11H1M+IpEHGMWgJgImsysDpLRWUGmDJTez/ii8psk0SfOP/0ZwQp+QxOB92CHdPeOPOmu2HFa00V82/H7gousXR7ywQRNthLHwso36O8+UoHc4qw8nIbjcHzbfD6ysJNynmaUMlB3mRTU1hkjGKKpA2Xyl4tmkIhp3NCPJc0WR4SgB3y0u3dVOC+RtbRzl/XpEbjsZHHNloBirK+8ERn9ISBBh/mvfo6qTix743e+xvhtBlLJjk3o4H0VMH+wQ3zIpIh4TKbhPCMqWY3gvtKDVRHD+Sywk2TE6zSz0sDPWk248MC2QsL7sgeFwcnFHOWy2iKf4YyuZtoaJuX+2tw23cDCdMS6wbARlT8Kb5QwMlsxuKYN/04kQB+9nXTVsWKJGIwLKdYRzshnlzqB/UEe2vrjZcbBixCp4pbZ2jSzw2881he4KSbVGIJdZYSFetMuaN0P9obtdaJU4V+IhwzFyyapjZhEGCTl+l/m8uGdJ5DOhFlZ7OczHja7DKUuQvB3AbnMGvN518C+fJkJpWxAn5UeIp3d0ZZm32XVKt3k8PJaP7LBYdxnr3JCRit7+kNnlP7Ho0NjvX6GTHQ+r+JM= From 4157d6db43206ed4f6e3350767e6c58b7977bd09 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 25 Feb 2021 01:32:55 +0000 Subject: [PATCH 140/145] readme fixes --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index df5c823c..636ed7c3 100644 --- a/README.md +++ b/README.md @@ -3,23 +3,23 @@ ## Chapters Each chapter has its own branch which contains all the commits for that chapter, -so it has the state that corresponds to the _end_ of that chapter. If you want -to try and code along with a chapter, you'll want to check out the branch for the -previous chapter. +so it has the state that corresponds to the _end_ of that chapter. +If you want to try and code along with a chapter, +you'll want to check out the branch for the previous chapter. -https://github.com/python-leap/code/branches/all +https://github.com/cosmicpython/code/branches/all ## Exercises -Branches for the exercises follow the convention `{chatper_name}_exercise`, eg -https://github.com/python-leap/code/tree/chapter_04_service_layer_exercise +Branches for the exercises follow the convention `{chapter_name}_exercise`, +eg https://github.com/cosmicpython/code/tree/chapter_04_service_layer_exercise ## Requirements * docker with docker-compose -* for chapters 1 and 2, and optionally for the rest: a local python3.7 virtualenv +* for chapters 1 and 2, and optionally for the rest: a local python3.8 virtualenv ## Building the containers From 79392e864d041e2edda02c57bd87c627290fba20 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 25 Feb 2021 01:45:17 +0000 Subject: [PATCH 141/145] attempt docker login --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index bf0b9de2..d69e8e14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ dist: xenial language: python python: 3.8 +before_install: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin script: - make all branches: From 3ed6ff0fab52e14edba6ced4b258af68c521115f Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 25 Feb 2021 01:48:56 +0000 Subject: [PATCH 142/145] upgrade travis to focal + 3.9 why the heck not --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d69e8e14..8b4550c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ -dist: xenial +dist: focal language: python -python: 3.8 +python: 3.9 before_install: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin script: From 2045cd7ffe66ad6129caebce653c47ef5159b0d2 Mon Sep 17 00:00:00 2001 From: Sergey Fursov Date: Fri, 2 Apr 2021 18:00:45 +0300 Subject: [PATCH 143/145] fix requirements installation command in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 636ed7c3..63e8510c 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,10 @@ pip install pytest pip install pytest sqlalchemy # for chapter 4+5 -pip install requirements.txt +pip install -r requirements.txt # for chapter 6+ -pip install requirements.txt +pip install -r requirements.txt pip install -e src/ ``` From 28b984ed155c3b3c4de5e40ead8f64d16727f09a Mon Sep 17 00:00:00 2001 From: adamculp Date: Sat, 22 May 2021 16:23:39 -0400 Subject: [PATCH 144/145] Corrected commands of the individual tests make commands --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 63e8510c..561cbb70 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ pip install -e src/ ```sh make test # or, to run individual test types -make unit -make integration -make e2e +make unit-tests +make integration-tests +make e2e-tests # or, if you have a local virtualenv make up pytest tests/unit From 7e9592811741d3d960979b684c4b51556eb850bc Mon Sep 17 00:00:00 2001 From: lotusirous <5105237+lotusirous@users.noreply.github.com> Date: Wed, 27 Oct 2021 22:58:18 +0900 Subject: [PATCH 145/145] Update mapper to sqlalchemy 1.4 --- src/allocation/adapters/orm.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 81d704ca..71ad4a39 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -1,21 +1,15 @@ import logging -from sqlalchemy import ( - Table, - MetaData, - Column, - Integer, - String, - Date, - ForeignKey, - event, -) -from sqlalchemy.orm import mapper, relationship + +from sqlalchemy import Column, Date, ForeignKey, Integer, String, Table, event +from sqlalchemy.orm import registry, relationship from allocation.domain import model logger = logging.getLogger(__name__) -metadata = MetaData() + +mapper_registry = registry() +metadata = mapper_registry.metadata order_lines = Table( "order_lines", @@ -62,8 +56,8 @@ def start_mappers(): logger.info("Starting mappers") - lines_mapper = mapper(model.OrderLine, order_lines) - batches_mapper = mapper( + lines_mapper = mapper_registry.map_imperatively(model.OrderLine, order_lines) + batches_mapper = mapper_registry.map_imperatively( model.Batch, batches, properties={ @@ -74,7 +68,7 @@ def start_mappers(): ) }, ) - mapper( + mapper_registry.map_imperatively( model.Product, products, properties={"batches": relationship(batches_mapper)},