-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
f721c83
commit 300be40
Showing
6 changed files
with
273 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
163 changes: 163 additions & 0 deletions
163
ckanext/datasetsnippets/mappings/resource_format_mappings.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| { | ||
| "Textual Document": { | ||
| "code": "textual", | ||
| "types": { | ||
| "PDF": [ | ||
| "PDF", | ||
| ".pdf", | ||
| "zip:pdf" | ||
| ], | ||
| "DOCX": [ | ||
| "DOCX", | ||
| "DOC" | ||
| ], | ||
| "TXT": [ | ||
| "TXT" | ||
| ] | ||
| } | ||
| }, | ||
| "Tabular Data": { | ||
| "code": "tabular", | ||
| "types": { | ||
| "CSV": [ | ||
| "CSV", | ||
| ".csv", | ||
| "zip:csv" | ||
| ], | ||
| "XLSX": [ | ||
| "XLSX", | ||
| "xlsx", | ||
| "XLS", | ||
| ".xls" | ||
| ], | ||
| "ODS": [ | ||
| "ODS" | ||
| ] | ||
| } | ||
| }, | ||
| "Archive": { | ||
| "code": "archive", | ||
| "types": { | ||
| "ZIP": [ | ||
| "ZIP", | ||
| ".zip" | ||
| ] | ||
| } | ||
| }, | ||
| "Website": { | ||
| "code": "website", | ||
| "types": { | ||
| "HTML": [ | ||
| "HTML", | ||
| "Webseite", | ||
| "webseite", | ||
| "application/x-php" | ||
| ] | ||
| } | ||
| }, | ||
| "GIS Data": { | ||
| "code": "gis", | ||
| "types": { | ||
| "GEOJSON": [ | ||
| "GEOJSON-Datei", | ||
| "GeoJSON", | ||
| "gjson" | ||
| ], | ||
| "KML/KMZ": [ | ||
| "KML", | ||
| "kmz", | ||
| "zip:KMZ" | ||
| ], | ||
| "SHAPE": [ | ||
| "shape", | ||
| "zip:SHP", | ||
| "zip:shp", | ||
| "zip:dbf, zip:prj, zip:shp, zip:shx" | ||
| ], | ||
| "MapInfo TAB": [ | ||
| "zip:dat, zip:id, zip:map, zip:tab" | ||
| ], | ||
| "WFS": [ | ||
| "WFS" | ||
| ], | ||
| "WMS": [ | ||
| "WMS" | ||
| ], | ||
| "GPX": [ | ||
| "gpx" | ||
| ], | ||
| "GML": [ | ||
| "zip:gml, zip:xsd" | ||
| ] | ||
| } | ||
| }, | ||
| "Feed": { | ||
| "code": "feed", | ||
| "types": { | ||
| "Atom": [ | ||
| "Atom" | ||
| ], | ||
| "RSS": [ | ||
| "RSS" | ||
| ], | ||
| "JSON Feed": [ | ||
| "jrss" | ||
| ] | ||
| } | ||
| }, | ||
| "Tree-shaped Data": { | ||
| "code": "tree", | ||
| "types": { | ||
| "JSON": [ | ||
| "JSON" | ||
| ], | ||
| "XML": [ | ||
| "XML" | ||
| ] | ||
| } | ||
| }, | ||
| "Bitmap Image": { | ||
| "code": "bitmap", | ||
| "types": { | ||
| "TIF": [ | ||
| "JPG", | ||
| "JPEG", | ||
| "PNG", | ||
| "TIF" | ||
| ] | ||
| } | ||
| }, | ||
| "3D Data": { | ||
| "code": "3d", | ||
| "types": { | ||
| "DXF": [ | ||
| "zip:DXF", | ||
| "zip:dxf" | ||
| ] | ||
| } | ||
| }, | ||
| "Transport Data": { | ||
| "code": "transport", | ||
| "types": { | ||
| "GTFS": [ | ||
| "GTFS" | ||
| ] | ||
| } | ||
| }, | ||
| "API": { | ||
| "code": "api", | ||
| "types": { | ||
| "API": [ | ||
| "API" | ||
| ] | ||
| } | ||
| }, | ||
| "Generic Data": { | ||
| "code": "generic", | ||
| "types": { | ||
| "Protobuf": [ | ||
| "pbf" | ||
| ] | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| # encoding: utf-8 | ||
| """ | ||
| Group various format strings (`CSV`, `.csv`, `zip:csv`) into abstract formats (`CSV`). Group formats (`CSV`, `XLSX`) into more general resource classes (`tabular data`). | ||
| """ | ||
|
|
||
| import json | ||
| import logging | ||
| import ckan.plugins as plugins | ||
|
|
||
| LOG = logging.getLogger(__name__) | ||
|
|
||
| class ResourceMapping(plugins.SingletonPlugin): | ||
| """ | ||
| Class representing the resource mappings. | ||
| """ | ||
|
|
||
| def load_mappings(self, mappings_path: str): | ||
| """ | ||
| Load the mappings. | ||
| """ | ||
| self._category_mappings = {} | ||
| try: | ||
| with open(mappings_path) as json_data: | ||
| self._category_mappings = json.load(json_data) | ||
| self._format_string_mappings = self.reverse_category_mapping(self._category_mappings) | ||
| LOG.info(self._category_mappings) | ||
| LOG.info(self._format_string_mappings) | ||
| except Exception: | ||
| raise MappingsError(f"Could not load mappings from {mappings_path}.") | ||
|
|
||
| def unload_mappings(self): | ||
| """ | ||
| Unload the schema. | ||
| """ | ||
| del self._category_mappings | ||
|
|
||
| def reverse_category_mapping(self, data: dict) -> dict: | ||
| reverse_mapping = {} | ||
| for category, definition in data.items(): | ||
| format_strings = [format_string for group in definition['types'].values() for format_string in group] | ||
| for format_string in format_strings: | ||
| reverse_mapping[format_string] = category | ||
| return reverse_mapping | ||
|
|
||
| def category_mappings(self) -> dict: | ||
| """ | ||
| Return the loaded resource mappings (categories -> formats -> format strings). | ||
| """ | ||
| try: | ||
| return self._category_mappings | ||
| except AttributeError: | ||
| raise MappingsError("Resource mappings file not loaded yet") | ||
|
|
||
| def format_string_mappings(self) -> dict: | ||
| """ | ||
| Return the loaded resource mappings (format_strings -> categories). | ||
| """ | ||
| try: | ||
| return self._format_string_mappings | ||
| except AttributeError: | ||
| raise MappingsError("Resource mappings file not loaded yet") | ||
|
|
||
| class MappingsError(Exception): | ||
| """ | ||
| Errors when handling the mappings. | ||
| """ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters