Refactor the Linker, Ranker, Recogniser and Pipeline

.github/workflows/ci.yml

@@ -53,4 +53,4 @@ jobs:
 
       - name: Test with pytest
         run: |
-          poetry run pytest
+          poetry run pytest tests

.gitignore

@@ -55,6 +55,9 @@ coverage.xml
 *.mo
 *.pot
 
+# OS stuff:
+.DS_Store
+
 # Django stuff:
 *.log
 local_settings.py
@@ -150,4 +153,4 @@ preprocessing/toponymmatching/experiments/
 
 # Docs
 _build
-test.ipynb
+test.ipynb

app/app_template.py

@@ -1,35 +1,45 @@
+import importlib
 import os
 import sys
 import time
 from pathlib import Path
-from typing import Union, Optional, List
+from typing import List, Optional, Union
 
 import uvicorn
 from fastapi import FastAPI, Request
 from pydantic import BaseModel
 
-from config import CONFIG as pipeline_config
-
 from t_res.geoparser import pipeline
+from t_res.utils.dataclasses import SentenceMentions, Candidates
+
+os.environ["APP_CONFIG_NAME"] = "t-res_deezy_reldisamb-wpubl-wmtops"
+
+config_mod = importlib.import_module(
+    ".t-res_deezy_reldisamb-wpubl-wmtops", "app.configs"
+)
+pipeline_config = config_mod.CONFIG
 
 geoparser = pipeline.Pipeline(**pipeline_config)
 
 
-class APIQuery(BaseModel):
+class RecognitionAPIQuery(BaseModel):
     text: str
-    place: Optional[Union[str, None]] = None
-    place_wqid: Optional[Union[str, None]] = None
 
 
 class CandidatesAPIQuery(BaseModel):
-    toponyms: List[dict]
+    sentence_mentions: List[dict]
+    place_of_pub_wqid: Optional[str] = None
+    place_of_pub: Optional[str] = None
 
 
 class DisambiguationAPIQuery(BaseModel):
-    dataset: List[dict]
-    wk_cands: dict
-    place: Optional[Union[str, None]] = None
-    place_wqid: Optional[Union[str, None]] = None
+    candidates: dict
+
+
+class PipelineAPIQuery(BaseModel):
+    text: str
+    place_of_pub_wqid: Optional[str] = None
+    place_of_pub: Optional[str] = None
 
 
 app_config_name = os.environ["APP_CONFIG_NAME"]
@@ -38,74 +48,61 @@ class DisambiguationAPIQuery(BaseModel):
 
 @app.get("/")
 async def read_root(request: Request):
-    return {"Welcome to T-Res!": request.app.title}
-
-
-@app.get("/test")
-async def test_pipeline():
-    resolved = geoparser.run_sentence(
-        "Harvey, from London;Thomas and Elizabeth, Barnett.",
-        place="Manchester",
-        place_wqid="Q18125",
-    )
-
-    return resolved
-
-
-@app.get("/resolve_sentence")
-async def run_sentence(api_query: APIQuery, request_id: Union[str, None] = None):
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
-    resolved = geoparser.run_sentence(
-        api_query.text, place=place, place_wqid=place_wqid
-    )
-
-    return resolved
-
-
-@app.get("/resolve_full_text")
-async def run_text(api_query: APIQuery):
-
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
-    resolved = geoparser.run_text(api_query.text, place=place, place_wqid=place_wqid)
-
-    return resolved
-
+    return {
+        "Title": request.app.title,
+        "request.url": request.url,
+        "request.query_params": request.query_params,
+        "root_path": request.scope.get("root_path"),
+        "request.client": request.client,
+        "hostname": os.uname()[1],
+        "worker_id": os.getpid(),
+    }
 
 @app.get("/run_ner")
-async def run_ner(api_query: APIQuery):
-
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+async def run_ner(api_query: RecognitionAPIQuery):
     ner_output = geoparser.run_text_recognition(
-        api_query.text, place=place, place_wqid=place_wqid
+        api_query.text
     )
-
     return ner_output
 
-
 @app.get("/run_candidate_selection")
 async def run_candidate_selection(cand_api_query: CandidatesAPIQuery):
-
-    wk_cands = geoparser.run_candidate_selection(cand_api_query.toponyms)
-    return wk_cands
-
+    sentence_mentions = SentenceMentions.from_json(cand_api_query.sentence_mentions)
+    candidates = geoparser.run_candidate_selection(
+        sentence_mentions,
+        place_of_pub_wqid=cand_api_query.place_of_pub_wqid,
+        place_of_pub=cand_api_query.place_of_pub,
+        )
+    return candidates
 
 @app.get("/run_disambiguation")
 async def run_disambiguation(api_query: DisambiguationAPIQuery):
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
-    disamb_output = geoparser.run_disambiguation(
-        api_query.dataset, api_query.wk_cands, place, place_wqid
+    candidates = Candidates.from_dict(api_query.candidates)
+    predictions = geoparser.run_disambiguation(candidates)
+    return predictions
+
+@app.get("/run_pipeline")
+async def run_pipeline(api_query: PipelineAPIQuery):
+    predictions = geoparser.run(
+        text=api_query.text,
+        place_of_pub_wqid=api_query.place_of_pub_wqid,
+        place_of_pub=api_query.place_of_pub,
     )
-    return disamb_output
+    return predictions
 
+@app.get("/test")
+async def test_pipeline():
+    predictions = geoparser.run(
+        "Harvey, from London;Thomas and Elizabeth, Barnett.",
+        place_of_pub_wqid="Q18125",
+        place_of_pub="Manchester",
+    )
+    return predictions
 
 @app.get("/health")
 async def healthcheck():
     return {"status": "ok"}
 
-
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    # poetry run uvicorn app.run_local_app:app --port 8123
+    uvicorn.run(app, host="0.0.0.0", port=8123)

app/app_template_old.py

@@ -0,0 +1,111 @@
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Union, Optional, List
+
+import uvicorn
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+
+from config import CONFIG as pipeline_config
+
+from t_res.geoparser import pipeline
+
+geoparser = pipeline.Pipeline(**pipeline_config)
+
+
+class APIQuery(BaseModel):
+    text: str
+    place: Optional[Union[str, None]] = None
+    place_wqid: Optional[Union[str, None]] = None
+
+
+class CandidatesAPIQuery(BaseModel):
+    toponyms: List[dict]
+
+
+class DisambiguationAPIQuery(BaseModel):
+    dataset: List[dict]
+    wk_cands: dict
+    place: Optional[Union[str, None]] = None
+    place_wqid: Optional[Union[str, None]] = None
+
+
+app_config_name = os.environ["APP_CONFIG_NAME"]
+app = FastAPI(title=f"Toponym Resolution Pipeline API ({app_config_name})")
+
+
+@app.get("/")
+async def read_root(request: Request):
+    return {"Welcome to T-Res!": request.app.title}
+
+
+@app.get("/test")
+async def test_pipeline():
+    resolved = geoparser.run_sentence_deprecated(
+        "Harvey, from London;Thomas and Elizabeth, Barnett.",
+        place="Manchester",
+        place_wqid="Q18125",
+    )
+
+    return resolved
+
+
+@app.get("/resolve_sentence")
+async def run_sentence(api_query: APIQuery, request_id: Union[str, None] = None):
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    resolved = geoparser.run_sentence_deprecated(
+        api_query.text, place=place, place_wqid=place_wqid
+    )
+
+    return resolved
+
+
+@app.get("/resolve_full_text")
+async def run_text(api_query: APIQuery):
+
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    resolved = geoparser.run_text_deprecated(api_query.text, place=place, place_wqid=place_wqid)
+
+    return resolved
+
+
+@app.get("/run_ner")
+async def run_ner(api_query: APIQuery):
+
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    ner_output = geoparser.run_text_recognition_deprecated(
+        api_query.text, place=place, place_wqid=place_wqid
+    )
+
+    return ner_output
+
+
+@app.get("/run_candidate_selection")
+async def run_candidate_selection(cand_api_query: CandidatesAPIQuery):
+
+    wk_cands = geoparser.run_candidate_selection_deprecated(cand_api_query.toponyms)
+    return wk_cands
+
+
+@app.get("/run_disambiguation")
+async def run_disambiguation(api_query: DisambiguationAPIQuery):
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    disamb_output = geoparser.run_disambiguation_deprecated(
+        api_query.dataset, api_query.wk_cands, place, place_wqid
+    )
+    return disamb_output
+
+
+@app.get("/health")
+async def healthcheck():
+    return {"status": "ok"}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app/configs/t-res_deezy_reldisamb-wpubl-wmtops.py

@@ -5,8 +5,7 @@
 
 # --------------------------------------
 # Instantiate the ranker:
-myranker = ranking.Ranker(
-    method="deezymatch",
+ranker = ranking.DeezyMatchRanker(
     resources_path="./resources/",
     strvar_parameters={
         # Parameters to create the string pair dataset:
@@ -37,9 +36,9 @@
 
 with sqlite3.connect("./resources/rel_db/embeddings_database.db") as conn:
     cursor = conn.cursor()
-    mylinker = linking.Linker(
-        method="reldisamb",
+    linker = linking.RelDisambLinker(
         resources_path="./resources/",
+        ranker=ranker,
         experiments_path="./experiments/",
         linking_resources=dict(),
         rel_params={
@@ -56,5 +55,5 @@
         overwrite_training=False,
     )
 
-# geoparser = pipeline.Pipeline(myranker=myranker, mylinker=mylinker)
-CONFIG = {"myranker": myranker, "mylinker": mylinker}
+# geoparser = pipeline.Pipeline(ranker=ranker, linker=linker)
+CONFIG = {"ranker": ranker, "linker": linker}