diff --git a/doc/example_odMLs/THGTTG.odml b/doc/example_odMLs/THGTTG.odml index e5ab854d..26a6cb4b 100644 --- a/doc/example_odMLs/THGTTG.odml +++ b/doc/example_odMLs/THGTTG.odml @@ -2,7 +2,7 @@ - http://portal.g-node.org/odml/terminologies/v1.1/terminologies.xml + https://terminologies.g-node.org/v1.1/terminologies.xml 42
TheCrew diff --git a/doc/example_odMLs/sample_odml.odml b/doc/example_odMLs/sample_odml.odml index fa4d91f4..9e2b95b6 100644 --- a/doc/example_odMLs/sample_odml.odml +++ b/doc/example_odMLs/sample_odml.odml @@ -3,7 +3,7 @@ D. N. Adams - http://portal.g-node.org/odml/terminologies/v1.1/terminologies.xml + https://terminologies.g-node.org/v1.1/terminologies.xml
person diff --git a/doc/example_odMLs/sample_odml.rdf b/doc/example_odMLs/sample_odml.rdf index d0de65fd..c31e05b6 100644 --- a/doc/example_odMLs/sample_odml.rdf +++ b/doc/example_odMLs/sample_odml.rdf @@ -83,5 +83,5 @@ odml:d98afe9b-3982-44bf-9373-12aaa4798628 a rdf:Bag ; "Tricia Marie McMillan", "Zaphod Beeblebrox" . - a . + a . diff --git a/doc/example_odMLs/thgttg.py b/doc/example_odMLs/thgttg.py index c0e2a5c6..9d44da64 100644 --- a/doc/example_odMLs/thgttg.py +++ b/doc/example_odMLs/thgttg.py @@ -25,7 +25,7 @@ save_to = os.path.join(output_directory, "THGTTG.odml") -odmlrepo = 'http://portal.g-node.org/odml/terminologies/v1.1/terminologies.xml' +odmlrepo = 'https://terminologies.g-node.org/v1.1/terminologies.xml' # CREATE A DOCUMENT doc = odml.Document(author="D. N. Adams", diff --git a/doc/example_rdfs/example_data/drosophila_2.ttl b/doc/example_rdfs/example_data/drosophila_2.ttl index c07ca44c..5f0609dc 100644 --- a/doc/example_rdfs/example_data/drosophila_2.ttl +++ b/doc/example_rdfs/example_data/drosophila_2.ttl @@ -643,5 +643,5 @@ odml:fadffec7-6b23-454e-bfd1-9d5884802abb a odml:Property ; odml:ff5fb5e5-5104-41c8-ab45-9d8787e7fbdc a rdf:Bag ; rdf:li "simpleErg" . - a . + a . diff --git a/doc/example_rdfs/example_data/drosophila_4.ttl b/doc/example_rdfs/example_data/drosophila_4.ttl index 065df7f3..b97db426 100644 --- a/doc/example_rdfs/example_data/drosophila_4.ttl +++ b/doc/example_rdfs/example_data/drosophila_4.ttl @@ -1352,5 +1352,5 @@ odml:fedefe26-0f36-433f-993d-ad827dcb0a50 a odml:Property ; odml:hasUnit "ms" ; odml:hasValue . - a . + a . diff --git a/doc/example_rdfs/example_data/drosophila_8.ttl b/doc/example_rdfs/example_data/drosophila_8.ttl index 198c589c..96273b14 100644 --- a/doc/example_rdfs/example_data/drosophila_8.ttl +++ b/doc/example_rdfs/example_data/drosophila_8.ttl @@ -1351,5 +1351,5 @@ odml:fe6a8363-a5f4-46ed-8eea-b57257d4823b a odml:Property ; odml:ff532446-ca25-4092-8abb-302e12344b3b a rdf:Bag ; rdf:li "1.0" . -odml:b0d95b14-a12f-4057-aed3-fdd3a6ea5f82 a . +odml:b0d95b14-a12f-4057-aed3-fdd3a6ea5f82 a . diff --git a/doc/odml_ontology/root-ontology.ttl b/doc/odml_ontology/root-ontology.ttl index dfb3c5eb..b8e503e5 100644 --- a/doc/odml_ontology/root-ontology.ttl +++ b/doc/odml_ontology/root-ontology.ttl @@ -227,7 +227,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Cell" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#CellProperties @@ -242,7 +242,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "DataAcquisition" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Dataset @@ -250,7 +250,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Dataset" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Document @@ -298,7 +298,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Electrode" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Hardware @@ -306,7 +306,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Hardware" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#HardwareSettings @@ -314,7 +314,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "HardwareSettings" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Hub @@ -338,7 +338,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Preparation" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Property @@ -394,7 +394,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Recording" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Section @@ -457,7 +457,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Setup" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Stimulus @@ -465,7 +465,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description of the Stimulus."^^xsd:string ; rdfs:label "Stimulus" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Subject @@ -473,7 +473,7 @@ rdf:Seq rdf:type owl:Class ; rdfs:subClassOf :Section ; rdfs:comment "Description"^^xsd:string ; rdfs:label "Subject" ; - rdfs:seeAlso . + rdfs:seeAlso . ### https://g-node.org/projects/odml-rdf#Terminology diff --git a/doc/tutorial.rst b/doc/tutorial.rst index a5b89fd2..44231032 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -71,7 +71,7 @@ The code for the example odML files, which we use within this tutorial is part of the documentation package (see doc/example_odMLs/). A summary of available odML terminologies and templates can be found `here -`_. +`_. ------------------------------------------------------------------------------- @@ -221,7 +221,7 @@ them to build your own metadata odML file will be described in later chapters. Further advanced functions you can use to navigate through your odML files, or to create an odML template file, or to make use of common odML terminologies provided via `the G-Node repository -`_ can also +`_ can also be found later on in this tutorial. But now, let us first have a look at the example odML file (THGTTG.odml)! @@ -289,7 +289,7 @@ Let's check out all attributes with the following commands:: >>> print(odmlEX.parent) None >>> print(odmlEX.repository) - http://portal.g-node.org/odml/terminologies/v1.1/terminologies.xml + https://terminologies.g-node.org/v1.1/terminologies.xml >>> print(odmlEX.version) 42 diff --git a/docs/data_model.md b/docs/data_model.md index d86784ed..64f22cc5 100644 --- a/docs/data_model.md +++ b/docs/data_model.md @@ -1,19 +1,19 @@ -# odml data model +# The odML data model -Data exchange requires that also annoations, metadata, are -exchanged. In oder to allow interoperability we need both a common +Data exchange requires that annotations, metadata, are also +shared. In order to allow interoperability both a common (meta) data model, the format in which the metadata are exchanged, and -a common terminology. +a common terminology are required. Here, we briefly describe the *odML* data model. It is based on the idea of key-value pairs like ``temperature = 26°C``. The model is as simple as possible while being flexible, allowing -interoperability, and being customizable. The model defines four -entities (Property, Section, Value, RootSection) whose relations and +interoperability, and being customizable. The model defines three +entities (Property, Section, Document) whose relations and elements are shown in the figure below. -![odml_logo](images/erModel.png "odml data model") +![odml_logo](images/erModel.png "odML data model") Property and Section are the core entities. A Section contains Properties and can further have subsection thus building a tree-like diff --git a/docs/index.md b/docs/index.md index abea00b8..b5e5b622 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,6 +28,7 @@ It is a registered research resource with the - [odml-ui](https://github.com/g-node/odml-ui "odml-ui - editor for odml metadata files"): Graphical editor - [odmlTables](https://github.com/INM-6/python-odmltables) Spreadsheet interface (by INM-6 FZ JÛlich) for odml files. + ### Terminologies *odML* facilitates and encourages standardization by providing [terminologies](https://github.com/G-Node/odml-terminologies). An @@ -35,6 +36,10 @@ odml-file can be based on such a terminology. In that case one does not need to provide definitions since they are part of the linked terminology. +Existing terminologies can be browsed and imported from +[terminologies.g-node.org](https://terminologies.g-node.org). + + ### Templates Templates are odML documents that can be re-used when collecting the same kind of information over the course of multiple identical experiments. @@ -46,7 +51,10 @@ discussed and shared at the If you have created your own templates that others might find useful, you are encouraged to share your templates via this repository. - + +Currently available odML templates can be browsed and imported from +[templates.g-node.org](https://templates.g-node.org). + * * * @@ -67,15 +75,17 @@ pip install odml ### Python convenience scripts -The Python installation features two convenience commandline scripts. +The Python installation features multiple convenience commandline scripts. - `odmlconversion`: Converts odML files of previous file versions into the current one. - `odmltordf`: Converts odML files to the supported RDF version of odML. +- `odmlview`: Render and browse local XML odML files in the webbrowser. -Both scripts provide detailed usage descriptions by adding the help flag to the command. +All scripts provide detailed usage descriptions by adding the `help` flag to the command. odmlconversion -h odmltordf -h + odmlview -h ### odML - NIX conversion script @@ -90,7 +100,7 @@ You can install the command line script via pip: The script can then be run from the command line and provides a detailed usage -description by adding the help flag to the command. +description by adding the `help` flag to the command. nixodmlconverter -h diff --git a/odml/templates.py b/odml/templates.py new file mode 100644 index 00000000..d0084918 --- /dev/null +++ b/odml/templates.py @@ -0,0 +1,189 @@ +""" +Handles (deferred) loading of odML templates +""" + +import os +import sys +import tempfile +import threading +try: + import urllib.request as urllib2 + from urllib.error import URLError + from urllib.parse import urljoin + +except ImportError: + import urllib2 + from urllib2 import URLError + from urlparse import urljoin + +from datetime import datetime as dati +from datetime import timedelta +from hashlib import md5 + +from .tools.parser_utils import ParserException +from .tools.xmlparser import XMLReader + + +REPOSITORY_BASE = 'https://templates.g-node.org/' +REPOSITORY = urljoin(REPOSITORY_BASE, 'templates.xml') + +CACHE_AGE = timedelta(days=1) +CACHE_DIR = "odml.cache" + + +# TODO after prototyping move functions common with +# terminologies to a common file. + + +def cache_load(url): + """ + Load the url and store the file in a temporary cache directory. + Subsequent requests for this url will use the cached version until + the file is older than the CACHE_AGE. + + Exceptions are caught and not re-raised to enable loading of nested + odML files without breaking if one of the child files is unavailable. + + :param url: location of an odML template XML file. + :return: Local file location of the requested file. + """ + + filename = '.'.join([md5(url.encode()).hexdigest(), os.path.basename(url)]) + cache_dir = os.path.join(tempfile.gettempdir(), CACHE_DIR) + + # Create temporary folder if required + if not os.path.exists(cache_dir): + try: + os.makedirs(cache_dir) + except OSError: # might happen due to concurrency + if not os.path.exists(cache_dir): + raise + + cache_file = os.path.join(cache_dir, filename) + + if not os.path.exists(cache_file) or dati.fromtimestamp(os.path.getmtime(cache_file)) < (dati.now() - CACHE_AGE): + try: + data = urllib2.urlopen(url).read() + if sys.version_info.major > 2: + data = data.decode("utf-8") + except (ValueError, URLError) as exc: + msg = "Failed to load resource from '%s': %s" % (url, exc) + exc.args = (msg,) # needs to be a tuple + raise exc + + with open(cache_file, "w") as local_file: + local_file.write(str(data)) + + return cache_file + + +class TemplateHandler(dict): + """ + TemplateHandler facilitates synchronous and deferred + loading, caching, browsing and importing of full or partial + odML templates. + """ + # Used for deferred loading + loading = {} + + def browse(self, url): + """ + Load, cache and pretty print an odML template XML file from a URL. + + :param url: location of an odML template XML file. + :return: The odML document loaded from url. + """ + doc = self.load(url) + + if not doc: + raise ValueError("Failed to load resource from '%s'" % url) + + doc.pprint(max_depth=0) + + return doc + + def clone_section(self, url, section_name, children=True, keep_id=False): + """ + Load a section by name from an odML template found at the provided URL + and return a clone. By default it will return a clone with all child + sections and properties as well as changed IDs for every entity. + The named section has to be a root (direct) child of the referenced + odML document. + + :param url: location of an odML template XML file. + :param section_name: Unique name of the requested Section. + :param children: Boolean whether the child entities of a Section will be + returned as well. Default is True. + :param keep_id: Boolean whether all returned entities will keep the + original ID or have a new one assigned. Default is False. + :return: The cloned odML section loaded from url. + """ + doc = self.load(url) + if not doc: + raise ValueError("Failed to load resource from '%s'" % url) + + try: + sec = doc[section_name] + except KeyError: + raise KeyError("Section '%s' not found in document at '%s'" % (section_name, url)) + + return sec.clone(children=children, keep_id=keep_id) + + def load(self, url): + """ + Load and cache an odML template from a URL. + + :param url: location of an odML template XML file. + :return: The odML document loaded from url. + """ + # Some feedback for the user when loading large or + # nested (include) odML files. + print("\nLoading file %s" % url) + + if url in self: + doc = self[url] + elif url in self.loading: + self.loading[url].join() + self.loading.pop(url, None) + doc = self.load(url) + else: + doc = self._load(url) + + return doc + + def _load(self, url): + """ + Cache loads an odML template for a URL and returns + the result as a parsed odML document. + + :param url: location of an odML template XML file. + :return: The odML document loaded from url. + It will silently return None, if any exceptions + occur to enable loading of nested odML files. + """ + try: + local_file = cache_load(url) + except (ValueError, URLError): + return None + + try: + doc = XMLReader(filename=url, ignore_errors=True).from_file(local_file) + doc.finalize() + except ParserException as exc: + print("Failed to load '%s' due to parser errors:\n %s" % (url, exc)) + return None + + self[url] = doc + return doc + + def deferred_load(self, url): + """ + Start a background thread to load an odML template from a URL. + + :param url: location of an odML template XML file. + """ + if url in self or url in self.loading: + return + + self.loading[url] = threading.Thread(target=self._load, args=(url,)) + self.loading[url].start() diff --git a/odml/terminology.py b/odml/terminology.py index 6162e517..1b4170fd 100644 --- a/odml/terminology.py +++ b/odml/terminology.py @@ -19,7 +19,7 @@ from .tools.xmlparser import XMLReader -REPOSITORY_BASE = 'http://portal.g-node.org/odml/terminologies' +REPOSITORY_BASE = 'https://terminologies.g-node.org' REPOSITORY = '/'.join([REPOSITORY_BASE, 'v1.1', 'terminologies.xml']) CACHE_AGE = datetime.timedelta(days=1) diff --git a/test/resources/version_conversion_int.json b/test/resources/version_conversion_int.json index c2280897..7d7d1cd9 100644 --- a/test/resources/version_conversion_int.json +++ b/test/resources/version_conversion_int.json @@ -1,6 +1,6 @@ { "Document": { - "repository": "http://portal.g-node.org/odml/terminologies/v1.0/terminologies.xml", + "repository": "https://terminologies.g-node.org/v1.0/terminologies.xml", "version": "v1.13", "date": "2018-02-02", "sections": [ @@ -238,7 +238,7 @@ { "definition": "def s3", "name": "section three", - "include": "http://portal.g-node.org/odml/terminologies/v1.0/terminologies.xml", + "include": "https://terminologies.g-node.org/v1.0/terminologies.xml", "properties": [], "sections": [], "type": "mainsec", diff --git a/test/resources/version_conversion_int.xml b/test/resources/version_conversion_int.xml index e79abe72..fbbd4462 100644 --- a/test/resources/version_conversion_int.xml +++ b/test/resources/version_conversion_int.xml @@ -107,6 +107,6 @@
author 2018-07-07 - http://portal.g-node.org/odml/terminologies/v1.0/terminologies.xml + https://terminologies.g-node.org/v1.0/terminologies.xml v1.13 diff --git a/test/resources/version_conversion_int.yaml b/test/resources/version_conversion_int.yaml index f9dab1df..0629d406 100644 --- a/test/resources/version_conversion_int.yaml +++ b/test/resources/version_conversion_int.yaml @@ -1,7 +1,7 @@ Document: author: author date: '2018-02-02' - repository: http://portal.g-node.org/odml/terminologies/v1.0/terminologies.xml + repository: https://terminologies.g-node.org/v1.0/terminologies.xml sections: - definition: def s1 include: url s1 diff --git a/test/test_version_converter_integration.py b/test/test_version_converter_integration.py index 571dfc45..7f489cb2 100644 --- a/test/test_version_converter_integration.py +++ b/test/test_version_converter_integration.py @@ -58,7 +58,7 @@ def check_result(self): # Test document attribute export self.assertEqual(doc.author, "author") self.assertEqual(doc.version, "v1.13") - repo = "http://portal.g-node.org/odml/terminologies/v1.1/terminologies.xml" + repo = "https://terminologies.g-node.org/v1.1/terminologies.xml" self.assertEqual(doc.repository, repo) self.assertEqual(len(doc.sections), 3)