Fix integer enumeration parsing (#14)

Schema parsing failed with integer enumeration, because we tried to get field size from the enumeration elements `len`, which does not work for `int`

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xml2db"
-version = "0.12.2"
+version = "0.12.3"
 authors = [
   { name="Commission de régulation de l'énergie", email="[email protected]" },
 ]

src/xml2db/model.py

@@ -384,7 +384,7 @@ def recurse_parse_simple_type(elem_type):
                         else None
                     )
                     ae = ae and bt_ae if ae is not None and bt_ae is not None else None
-                if elem_type.enumeration is not None:
+                if elem_type.enumeration is not None and dt in ["string", "NMTOKEN", "duration", "token"]:
                     mil = min([len(val) for val in elem_type.enumeration])
                     mal = max([len(val) for val in elem_type.enumeration])
                 return dt, mil, mal, ae

src/xml2db/table/column.py

@@ -32,7 +32,7 @@ def types_mapping_default(temp: bool, col: "DataModelColumn") -> Any:
     """
     if col.occurs[1] != 1:
         return String(8000)
-    if col.data_type in ["decimal", "float"]:
+    if col.data_type in ["decimal", "float", "double"]:
         return Double
     if col.data_type == "dateTime":
         return DateTime(timezone=True)
@@ -77,7 +77,7 @@ def types_mapping_mssql(temp: bool, col: "DataModelColumn") -> Any:
     """
     if col.occurs[1] != 1:
         return mssql.VARCHAR(8000)
-    if col.data_type in ["decimal", "float"]:
+    if col.data_type in ["decimal", "float", "double"]:
         return Double
     if col.data_type == "dateTime":
         # using the DATETIMEOFFSET directly in the temporary table caused issues when inserting data in the target

tests/sample_models/orders/base_types.xsd

@@ -8,6 +8,27 @@
     <xs:simpleType name="inttype">
         <xs:restriction base="xs:integer"/>
     </xs:simpleType>
+    <xs:simpleType name="quantitytype">
+        <xs:restriction base="xs:integer">
+            <xs:maxInclusive value="99" />
+            <xs:minInclusive value="1" />
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="currencytype">
+        <xs:restriction base="xs:string">
+            <xs:enumeration value="EUR" />
+            <xs:enumeration value="CHF" />
+            <xs:enumeration value="GBP" />
+            <xs:enumeration value="USD" />
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="versiontype">
+        <xs:restriction base="xs:int">
+            <xs:enumeration value="1" />
+            <xs:enumeration value="2" />
+            <xs:enumeration value="3" />
+        </xs:restriction>
+    </xs:simpleType>
     <xs:simpleType name="dectype">
         <xs:restriction base="xs:decimal"/>
     </xs:simpleType>

tests/sample_models/orders/invalid_xml/malformed_recover.xml

@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
 <orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="5" xsi:noNamespaceSchemaLocation="../orders.xsd">
+  <version>3</version>
   <shiporder orderid="1" processed_at="2023-09-10T09:37:00.000+02:00">
     <orderperson>
       <name>Bob</name>
@@ -20,6 +21,7 @@
       </product>
       <quantity>13</quantity>
       <price>340.23</price>
+      <currency>CHF</currency>
     </item>
     <item>
       <product>
@@ -29,6 +31,7 @@
       <note>string</note>
       <quantity>8</quantity>
       <price>56.2</price>
+      <currency>CHF</currency>
     </item>
     <item>
       <product>
@@ -37,6 +40,7 @@
       </product>
       <quantity>25</quantity>
       <price>21.7</price>
+      <currency>CHF</currency>
     </item>
   </shiporder>
 </orders>

tests/sample_models/orders/orders.xsd

@@ -48,8 +48,9 @@
         <xs:sequence>
             <xs:element name="product" type="producttype" minOccurs="1" maxOccurs="1"/>
             <xs:element name="note" type="bt:stringtype" minOccurs="0"/>
-            <xs:element name="quantity" type="bt:inttype"/>
+            <xs:element name="quantity" type="bt:quantitytype"/>
             <xs:element name="price" type="bt:dectype"/>
+            <xs:element name="currency" type="bt:currencytype"/>
         </xs:sequence>
     </xs:complexType>
 
@@ -65,6 +66,7 @@
 
     <xs:complexType name="orderstype">
         <xs:sequence>
+            <xs:element name="version" type="bt:versiontype" />
             <xs:element name="shiporder" type="shipordertype" minOccurs="0" maxOccurs="unbounded" />
         </xs:sequence>
         <xs:attribute name="batch_id" type="bt:stringtype" />

tests/sample_models/orders/orders_ddl_mssql_version0.sql

@@ -23,6 +23,7 @@ CREATE TABLE item (
 	note VARCHAR(1000) NULL, 
 	quantity INTEGER NULL, 
 	price DOUBLE PRECISION NULL, 
+	currency CHAR(3) NULL, 
 	record_hash BINARY(20) NULL, 
 	CONSTRAINT cx_pk_item PRIMARY KEY CLUSTERED (pk_item), 
 	CONSTRAINT item_xml2db_record_hash UNIQUE (record_hash)
@@ -54,6 +55,7 @@ CREATE TABLE shiporder_item (
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL IDENTITY, 
 	batch_id VARCHAR(1000) NULL, 
+	version INTEGER NULL, 
 	input_file_path VARCHAR(256) NULL, 
 	record_hash BINARY(20) NULL, 
 	CONSTRAINT cx_pk_orders PRIMARY KEY CLUSTERED (pk_orders), 

tests/sample_models/orders/orders_ddl_mssql_version1.sql

@@ -34,6 +34,7 @@ CREATE TABLE shiporder (
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL IDENTITY, 
 	batch_id VARCHAR(1000) NULL, 
+	version INTEGER NULL, 
 	xml2db_processed_at DATETIMEOFFSET NULL, 
 	input_file_path VARCHAR(256) NULL, 
 	record_hash BINARY(16) NULL, 
@@ -60,6 +61,7 @@ CREATE TABLE item (
 	note VARCHAR(1000) NULL, 
 	quantity INTEGER NULL, 
 	price DOUBLE PRECISION NULL, 
+	currency CHAR(3) NULL, 
 	CONSTRAINT cx_pk_item PRIMARY KEY CLUSTERED (pk_item), 
 	FOREIGN KEY(fk_parent_shiporder) REFERENCES shiporder (pk_shiporder)
 )

tests/sample_models/orders/orders_ddl_mssql_version2.sql

@@ -2,6 +2,7 @@
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL IDENTITY, 
 	batch_id VARCHAR(1000) NULL, 
+	version INTEGER NULL, 
 	input_file_path VARCHAR(256) NULL, 
 	xml2db_record_hash BINARY(20) NULL, 
 	CONSTRAINT cx_pk_orders PRIMARY KEY CLUSTERED (pk_orders), 
@@ -42,6 +43,7 @@ CREATE TABLE item (
 	note VARCHAR(1000) NULL, 
 	quantity INTEGER NULL, 
 	price DOUBLE PRECISION NULL, 
+	currency CHAR(3) NULL, 
 	xml2db_record_hash BINARY(20) NULL, 
 	CONSTRAINT cx_pk_item PRIMARY KEY CLUSTERED (pk_item), 
 	CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), 

tests/sample_models/orders/orders_ddl_mysql_version0.sql

@@ -23,6 +23,7 @@ CREATE TABLE item (
 	note VARCHAR(255), 
 	quantity INTEGER, 
 	price DOUBLE, 
+	currency VARCHAR(3), 
 	record_hash BINARY(20), 
 	CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), 
 	CONSTRAINT item_xml2db_record_hash UNIQUE (record_hash)
@@ -54,6 +55,7 @@ CREATE TABLE shiporder_item (
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL AUTO_INCREMENT, 
 	batch_id VARCHAR(255), 
+	version INTEGER, 
 	input_file_path VARCHAR(256), 
 	record_hash BINARY(20), 
 	CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), 

tests/sample_models/orders/orders_ddl_mysql_version1.sql

@@ -34,6 +34,7 @@ CREATE TABLE shiporder (
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL AUTO_INCREMENT, 
 	batch_id VARCHAR(255), 
+	version INTEGER, 
 	xml2db_processed_at DATETIME, 
 	input_file_path VARCHAR(256), 
 	record_hash BINARY(16), 
@@ -60,6 +61,7 @@ CREATE TABLE item (
 	note VARCHAR(255), 
 	quantity INTEGER, 
 	price DOUBLE, 
+	currency VARCHAR(3), 
 	CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), 
 	FOREIGN KEY(fk_parent_shiporder) REFERENCES shiporder (pk_shiporder)
 )

tests/sample_models/orders/orders_ddl_mysql_version2.sql

@@ -2,6 +2,7 @@
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL AUTO_INCREMENT, 
 	batch_id VARCHAR(255), 
+	version INTEGER, 
 	input_file_path VARCHAR(256), 
 	xml2db_record_hash BINARY(20), 
 	CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), 
@@ -42,6 +43,7 @@ CREATE TABLE item (
 	note VARCHAR(255), 
 	quantity INTEGER, 
 	price DOUBLE, 
+	currency VARCHAR(3), 
 	xml2db_record_hash BINARY(20), 
 	CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), 
 	CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), 

tests/sample_models/orders/orders_ddl_postgresql_version0.sql

@@ -23,6 +23,7 @@ CREATE TABLE item (
 	note VARCHAR(1000), 
 	quantity INTEGER, 
 	price DOUBLE PRECISION, 
+	currency VARCHAR(3), 
 	record_hash BYTEA, 
 	CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), 
 	CONSTRAINT item_xml2db_record_hash UNIQUE (record_hash)
@@ -54,6 +55,7 @@ CREATE TABLE shiporder_item (
 CREATE TABLE orders (
 	pk_orders SERIAL NOT NULL, 
 	batch_id VARCHAR(1000), 
+	version INTEGER, 
 	input_file_path VARCHAR(256), 
 	record_hash BYTEA, 
 	CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), 

tests/sample_models/orders/orders_ddl_postgresql_version1.sql

@@ -34,6 +34,7 @@ CREATE TABLE shiporder (
 CREATE TABLE orders (
 	pk_orders SERIAL NOT NULL, 
 	batch_id VARCHAR(1000), 
+	version INTEGER, 
 	xml2db_processed_at TIMESTAMP WITH TIME ZONE, 
 	input_file_path VARCHAR(256), 
 	record_hash BYTEA, 
@@ -60,6 +61,7 @@ CREATE TABLE item (
 	note VARCHAR(1000), 
 	quantity INTEGER, 
 	price DOUBLE PRECISION, 
+	currency VARCHAR(3), 
 	CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), 
 	FOREIGN KEY(fk_parent_shiporder) REFERENCES shiporder (pk_shiporder)
 )

tests/sample_models/orders/orders_ddl_postgresql_version2.sql

@@ -2,6 +2,7 @@
 CREATE TABLE orders (
 	pk_orders SERIAL NOT NULL, 
 	batch_id VARCHAR(1000), 
+	version INTEGER, 
 	input_file_path VARCHAR(256), 
 	xml2db_record_hash BYTEA, 
 	CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), 
@@ -42,6 +43,7 @@ CREATE TABLE item (
 	note VARCHAR(1000), 
 	quantity INTEGER, 
 	price DOUBLE PRECISION, 
+	currency VARCHAR(3), 
 	xml2db_record_hash BYTEA, 
 	CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), 
 	CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), 

tests/sample_models/orders/orders_erd_version0.md

@@ -3,6 +3,7 @@ erDiagram
     orders ||--o{ shiporder : "shiporder*"
     orders {
         string batch_id
+        int version
     }
     shiporder ||--|| orderperson : "orderperson"
     shiporder ||--o| orderperson : "shipto"
@@ -17,6 +18,7 @@ erDiagram
         string note
         integer quantity
         decimal price
+        string currency
     }
     orderperson {
         string name

tests/sample_models/orders/orders_erd_version1.md

@@ -6,10 +6,12 @@ erDiagram
         string note
         integer quantity
         decimal price
+        string currency
     }
     orders ||--o{ shiporder : "shiporder*"
     orders {
         string batch_id
+        int version
     }
     shiporder ||--|| orderperson : "orderperson"
     shiporder ||--o| orderperson : "shipto"

tests/sample_models/orders/orders_erd_version2.md

@@ -20,6 +20,7 @@ erDiagram
         string note
         integer quantity
         decimal price
+        string currency
     }
     product {
         string name
@@ -39,5 +40,6 @@ erDiagram
     orders ||--o{ shiporder : "shiporder"
     orders {
         string batch_id
+        int version
     }
 ```

tests/sample_models/orders/xml/order1.xml

@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
 <orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="5" xsi:noNamespaceSchemaLocation="../orders.xsd">
+  <version>2</version>
   <shiporder orderid="1" processed_at="2023-09-10T09:37:00.000+02:00">
     <orderperson>
       <name>Bob</name>
@@ -20,6 +21,7 @@
       </product>
       <quantity>13</quantity>
       <price>340.23</price>
+      <currency>EUR</currency>
     </item>
     <item>
       <product>
@@ -29,6 +31,7 @@
       <note>string</note>
       <quantity>8</quantity>
       <price>56.2</price>
+      <currency>EUR</currency>
     </item>
     <item>
       <product>
@@ -37,6 +40,7 @@
       </product>
       <quantity>25</quantity>
       <price>21.7</price>
+      <currency>EUR</currency>
     </item>
   </shiporder>
 </orders>

tests/sample_models/orders/xml/order2.xml

@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
 <orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="2" xsi:noNamespaceSchemaLocation="../orders.xsd">
+  <version>2</version>
   <shiporder orderid="2" processed_at="2023-09-10T09:39:45.000+02:00">
     <orderperson>
       <name>Alice</name>
@@ -18,6 +19,7 @@
       </product>
       <quantity>43</quantity>
       <price>30.3</price>
+      <currency>EUR</currency>
     </item>
     <item>
       <product>
@@ -27,6 +29,7 @@
       <note>string</note>
       <quantity>21</quantity>
       <price>70.4</price>
+      <currency>EUR</currency>
     </item>
     <item>
       <product>
@@ -35,6 +38,7 @@
       </product>
       <quantity>2</quantity>
       <price>12.2</price>
+      <currency>EUR</currency>
     </item>
   </shiporder>
 </orders>

tests/sample_models/orders/xml/order3.xml

@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
 <orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="4" xsi:noNamespaceSchemaLocation="../orders.xsd">
+  <version>2</version>
   <shiporder orderid="3" processed_at="2023-09-11T18:27:56.000+02:00">
     <orderperson>
       <name>Alice</name>
@@ -27,6 +28,7 @@
       </product>
       <quantity>10</quantity>
       <price>12.3</price>
+      <currency>EUR</currency>
     </item>
     <item>
       <product>
@@ -36,6 +38,7 @@
       <note>string</note>
       <quantity>21</quantity>
       <price>70.4</price>
+      <currency>EUR</currency>
     </item>
   </shiporder>
 </orders>