erge remote-tracking branch 'origin/develop' into gtf-label-parsing

.github/workflows/cd.yml

@@ -1,6 +1,6 @@
 name: continuous deployment
 
-on: 
+on:
   pull_request:
     branches:
     - develop
@@ -19,11 +19,13 @@ jobs:
     steps:
       # setup, checkout pull_request.head.ref for repo-vis
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{github.event.pull_request.head.ref}}
       - name: setup-python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
       # # update repo visualization for docs
       # - name: repo-visualizer
       #   uses: githubocto/[email protected]
@@ -32,8 +34,13 @@ jobs:
       #     excluded_paths: ".github"
       #     commit_message: "repo-visualizer [skip actions]"
       # install project, which is required for autodoc of code
+      - name: install-poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.4.0
+          virtualenvs-create: false
       - name: install buildingmotif
-        run: pip install .
+        run: poetry install --all-extras
       # install jupyter-book, which for some reason isn't available with poetry install
       - name: install jupyter book
         run: pip install jupyter-book
@@ -47,7 +54,7 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./docs/_build/html      
+          publish_dir: ./docs/_build/html
 
   # deploy distribution if a new release and tag are created
   deploy-dist:
@@ -57,9 +64,9 @@ jobs:
     steps:
       # setup
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: setup-python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
       # install poetry and build dist
       - name: install-poetry
         uses: snok/install-poetry@v1

.github/workflows/ci.yml

@@ -9,9 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: setup-python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
       - name: install-poetry
         uses: snok/install-poetry@v1
         with:
@@ -30,12 +32,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: setup-python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: install-poetry

buildingmotif/api/views/library.py

@@ -71,9 +71,7 @@ def get_library(library_id: int) -> flask.Response:
     :rtype: flask.Response
     """
     try:
-        db_lib = current_app.building_motif.table_connection.get_db_library_by_id(
-            library_id
-        )
+        db_lib = current_app.building_motif.table_connection.get_db_library(library_id)
     except NoResultFound:
         return {
             "message": f"No library with id {library_id}"

buildingmotif/api/views/template.py

@@ -39,7 +39,7 @@ def get_template(templates_id: int) -> flask.Response:
     include_parameters = request.args.get("parameters", False)
 
     try:
-        template = current_app.building_motif.table_connection.get_db_template_by_id(
+        template = current_app.building_motif.table_connection.get_db_template(
             templates_id
         )
     except NoResultFound:

buildingmotif/database/table_connection.py

@@ -208,7 +208,7 @@ def get_all_db_libraries(self) -> List[DBLibrary]:
         db_libraries = self.bm.session.query(DBLibrary).all()
         return db_libraries
 
-    def get_db_library_by_id(self, id: int) -> DBLibrary:
+    def get_db_library(self, id: int) -> DBLibrary:
         """Get database library by id.
 
         :param id: id of DBLibrary
@@ -237,7 +237,7 @@ def update_db_library_name(self, id: int, name: str) -> None:
         :param name: new name
         :type name: str
         """
-        db_library = self.get_db_library_by_id(id)
+        db_library = self.get_db_library(id)
         self.logger.debug(
             f"Updating database library name: '{db_library.name}' -> '{name}'"
         )
@@ -250,7 +250,7 @@ def delete_db_library(self, id: int) -> None:
         :type id: int
         """
 
-        db_library = self.get_db_library_by_id(id)
+        db_library = self.get_db_library(id)
 
         self.logger.debug(f"Deleting database library: '{db_library.name}'")
         self.bm.session.delete(db_library)
@@ -268,7 +268,7 @@ def create_db_template(self, name: str, library_id: int) -> DBTemplate:
         :rtype: DBTemplate
         """
         self.logger.debug(f"Creating database template: '{name}'")
-        library = self.get_db_library_by_id(library_id)
+        library = self.get_db_library(library_id)
         template = DBTemplate(
             name=name,
             body_id=str(uuid.uuid4()),
@@ -290,7 +290,7 @@ def get_all_db_templates(self) -> List[DBTemplate]:
         db_templates = self.bm.session.query(DBTemplate).all()
         return db_templates
 
-    def get_db_template_by_id(self, id: int) -> DBTemplate:
+    def get_db_template(self, id: int) -> DBTemplate:
         """Get database template by id.
 
         :param id: id of DBTemplate
@@ -316,7 +316,7 @@ def get_db_template_by_name(self, name: str) -> DBTemplate:
                 self.bm.session.query(DBTemplate).filter(DBTemplate.name == name).one()
             )
         except NoResultFound:
-            raise NoResultFound(f"No tempalte found with name {name}")
+            raise NoResultFound(f"No template found with name {name}")
         return db_template
 
     def get_library_defining_db_template(self, id: int) -> DBLibrary:
@@ -327,7 +327,7 @@ def get_library_defining_db_template(self, id: int) -> DBLibrary:
         :return: DBLibrary
         :rtype: DBLibrary
         """
-        return self.get_db_template_by_id(id).library
+        return self.get_db_template(id).library
 
     def get_db_template_dependencies(self, id: int) -> Tuple[DepsAssociation, ...]:
         """Get a template's dependencies and its arguments.
@@ -356,7 +356,7 @@ def update_db_template_name(self, id: int, name: str) -> None:
         :param name: new name
         :type name: str
         """
-        db_template = self.get_db_template_by_id(id)
+        db_template = self.get_db_template(id)
         self.logger.debug(
             f"Updating database template name: '{db_template.name}' -> '{name}'"
         )
@@ -405,7 +405,7 @@ def add_template_dependency_preliminary(
         self.logger.debug(
             f"Creating depencency from templates with ids: '{template_id}' and: '{dependency_id}'"
         )
-        templ = self.get_db_template_by_id(template_id)
+        templ = self.get_db_template(template_id)
         if "name" not in args.keys():
             raise ValueError(
                 f"The name parameter is required for the dependency '{templ.name}'."
@@ -490,8 +490,8 @@ def check_template_dependency_relationship(self, dep: DepsAssociation):
                 f"'name' was bound to {args['name']} but available params are {params}"
             )
 
-    def remove_template_dependency(self, template_id: int, dependency_id: int):
-        """Remove dependency between two templates.
+    def delete_template_dependency(self, template_id: int, dependency_id: int):
+        """Delete dependency between two templates.
 
         :param template_id: dependant template id
         :type template_id: int
@@ -520,7 +520,7 @@ def update_db_template_library(self, id: int, library_id: int) -> None:
         :param library_id: id of the new library
         :type library_id: int
         """
-        db_template = self.get_db_template_by_id(id)
+        db_template = self.get_db_template(id)
         self.logger.debug(
             f"Updating database template library: '{db_template.library_id}' -> '{library_id}'"  # noqa
         )
@@ -532,7 +532,7 @@ def delete_db_template(self, id: int) -> None:
         :param id: id of deleted DBTemplate
         :type id: int
         """
-        db_template = self.get_db_template_by_id(id)
+        db_template = self.get_db_template(id)
         self.logger.debug(f"Deleting template: '{db_template.name}'")
 
         self.bm.session.delete(db_template)

buildingmotif/dataclasses/library.py

@@ -209,7 +209,7 @@ def _load_from_db(cls, id: int) -> "Library":
         :rtype: Library
         """
         bm = get_building_motif()
-        db_library = bm.table_connection.get_db_library_by_id(id)
+        db_library = bm.table_connection.get_db_library(id)
 
         return cls(_id=db_library.id, _name=db_library.name, _bm=bm)
 
@@ -260,24 +260,8 @@ def _load_from_ontology(
 
         lib = cls.create(ontology_name, overwrite=overwrite)
 
-        class_candidates = set(ontology.subjects(rdflib.RDF.type, rdflib.OWL.Class))
-        shape_candidates = set(ontology.subjects(rdflib.RDF.type, rdflib.SH.NodeShape))
-        candidates = class_candidates.intersection(shape_candidates)
-
-        # stores the lookup from template *names* to template *ids*
-        # this is necessary because while we know the *name* of the dependee templates
-        # for each dependent template, we don't know the *id* of the dependee templates,
-        # which is necessary to populate the dependencies
-        template_id_lookup: Dict[str, int] = {}
-        dependency_cache: Dict[int, List[Dict[Any, Any]]] = {}
-        for candidate in candidates:
-            assert isinstance(candidate, rdflib.URIRef)
-            partial_body, deps = get_template_parts_from_shape(candidate, ontology)
-            templ = lib.create_template(str(candidate), partial_body)
-            dependency_cache[templ.id] = deps
-            template_id_lookup[str(candidate)] = templ.id
-
-        lib._resolve_template_dependencies(template_id_lookup, dependency_cache)
+        # infer shapes from any class/nodeshape candidates in the graph
+        lib._infer_shapes_from_graph(ontology)
 
         # load the ontology graph as a shape_collection
         shape_col_id = lib.get_shape_collection().id
@@ -287,6 +271,26 @@ def _load_from_ontology(
 
         return lib
 
+    def _infer_shapes_from_graph(self, graph: rdflib.Graph):
+        """Infer shapes from a graph and add them to this library.
+
+        :param graph: graph to infer shapes from
+        :type graph: rdflib.Graph
+        """
+        class_candidates = set(graph.subjects(rdflib.RDF.type, rdflib.OWL.Class))
+        shape_candidates = set(graph.subjects(rdflib.RDF.type, rdflib.SH.NodeShape))
+        candidates = class_candidates.intersection(shape_candidates)
+        template_id_lookup: Dict[str, int] = {}
+        dependency_cache: Dict[int, List[Dict[Any, Any]]] = {}
+        for candidate in candidates:
+            assert isinstance(candidate, rdflib.URIRef)
+            partial_body, deps = get_template_parts_from_shape(candidate, graph)
+            templ = self.create_template(str(candidate), partial_body)
+            dependency_cache[templ.id] = deps
+            template_id_lookup[str(candidate)] = templ.id
+
+        self._resolve_template_dependencies(template_id_lookup, dependency_cache)
+
     def _load_shapes_from_directory(self, directory: pathlib.Path):
         """Helper method to read all graphs in the given directory into this
         library.
@@ -305,6 +309,8 @@ def _load_shapes_from_directory(self, directory: pathlib.Path):
                     f"Could not parse file {filename}: {e}"
                 )
                 raise e
+        # infer shapes from any class/nodeshape candidates in the graph
+        self._infer_shapes_from_graph(shape_col.graph)
 
     @classmethod
     def _load_from_directory(
@@ -497,7 +503,7 @@ def get_templates(self) -> List[Template]:
         :return: list of templates
         :rtype: List[Template]
         """
-        db_library = self._bm.table_connection.get_db_library_by_id(self._id)
+        db_library = self._bm.table_connection.get_db_library(self._id)
         templates: List[DBTemplate] = db_library.templates
         return [Template.load(t.id) for t in templates]
 
@@ -509,7 +515,7 @@ def get_shape_collection(self) -> ShapeCollection:
         """
         # TODO: we should save the libraries shape_collection to a class attr on load/create. That
         # way we wont need an additional db query each time we call this function.
-        db_library = self._bm.table_connection.get_db_library_by_id(self._id)
+        db_library = self._bm.table_connection.get_db_library(self._id)
 
         return ShapeCollection.load(db_library.shape_collection.id)
 

buildingmotif/dataclasses/template.py

@@ -50,7 +50,7 @@ def load(cls, id: int) -> "Template":
         :rtype: Template
         """
         bm = get_building_motif()
-        db_template = bm.table_connection.get_db_template_by_id(id)
+        db_template = bm.table_connection.get_db_template(id)
         body = bm.graph_connection.get_graph(db_template.body_id)
 
         return cls(
@@ -137,7 +137,7 @@ def remove_dependency(self, dependency: "Template") -> None:
         :param dependency: dependency to remove
         :type dependency: Template
         """
-        self._bm.table_connection.remove_template_dependency(self.id, dependency.id)
+        self._bm.table_connection.delete_template_dependency(self.id, dependency.id)
 
     @property
     def all_parameters(self) -> Set[str]:

buildingmotif/utils.py

@@ -207,6 +207,8 @@ def get_template_parts_from_shape(
     pshapes = shape_graph.objects(subject=shape_name, predicate=SH["property"])
     for pshape in pshapes:
         property_path = shape_graph.value(pshape, SH["path"])
+        if property_path is None:
+            raise Exception(f"no sh:path detected on {shape_name}")
         # TODO: expand otypes to include sh:in, sh:or, or no datatype at all!
         otypes = list(
             shape_graph.objects(
@@ -232,11 +234,16 @@ def get_template_parts_from_shape(
         (path, otype, mincount) = property_path, otypes[0], mincounts[0]
         assert isinstance(mincount, Literal)
 
-        for _ in range(int(mincount)):
-            param = _gensym()
+        param_name = shape_graph.value(pshape, SH["name"])
+
+        for num in range(int(mincount)):
+            if param_name is not None:
+                param = PARAM[f"{param_name}{num}"]
+            else:
+                param = _gensym()
             body.add((root_param, path, param))
-            deps.append({"template": otype, "args": {"name": param}})
-            # body.add((param, RDF.type, otype))
+            deps.append({"template": str(otype), "args": {"name": param}})
+            body.add((param, RDF.type, otype))
 
     if (shape_name, RDF.type, OWL.Class) in shape_graph:
         body.add((root_param, RDF.type, shape_name))
@@ -245,9 +252,15 @@ def get_template_parts_from_shape(
     for cls in classes:
         body.add((root_param, RDF.type, cls))
 
+    classes = shape_graph.objects(shape_name, SH["targetClass"])
+    for cls in classes:
+        body.add((root_param, RDF.type, cls))
+
     nodes = shape_graph.objects(shape_name, SH["node"])
     for node in nodes:
-        deps.append({"template": node, "args": {"name": "name"}})  # tie to root param
+        deps.append(
+            {"template": str(node), "args": {"name": "name"}}
+        )  # tie to root param
 
     return body, deps
 

docs/_toc.yml

@@ -23,6 +23,7 @@ parts:
 - caption: Explainations
   chapters:
   - file: explanations/ingresses.md
+  - file: explanations/shapes-and-templates.md
 - caption: Appendix
   chapters:
   - file: bibliography.md