Merge pull request #31 from portfoliome/inject-nulls

Inject nulls

.travis.yml

@@ -17,7 +17,7 @@ install:
   - python setup.py install
 
 script:
-  - py.test tests -s --cov
+  - py.test tests --cov foil --cov tests -s
 
 after_success:
   - codecov

foil/_version.py

@@ -1,3 +1,3 @@
-version_info = (0, 2, 2)
+version_info = (0, 2, 3)
 
 __version__ = '.'.join(map(str, version_info))

foil/converters.py

@@ -1,19 +1,3 @@
-from typing import Mapping
 
-
-def rename_keys(record: Mapping, key_map: Mapping) -> dict:
-    """New record with same keys or renamed keys if key found in key_map."""
-
-    new_record = dict()
-
-    for k, v in record.items():
-        key = key_map[k] if k in key_map else k
-        new_record[key] = v
-
-    return new_record
-
-
-def replace_keys(record: Mapping, key_map: Mapping) -> dict:
-    """New record with renamed keys including keys only found in key_map."""
-
-    return {key_map[k]: v for k, v in record.items() if k in key_map}
+# moved to foil/records.py
+from foil.records import replace_keys, rename_keys

foil/dotenv.py

@@ -20,4 +20,5 @@ def read_dotenv(dotenv_path: str):
 
             env_key, env_value = line.split('=', 1)
             env_value = env_value.strip("'").strip('"')
+
             yield env_key, env_value

foil/parsers.py

@@ -24,6 +24,7 @@
 
 def passthrough(value):
     """Pass through value for no conversion."""
+
     return value
 
 

foil/records.py

@@ -0,0 +1,32 @@
+"""Functionality for manipulating dictionary records."""
+
+from typing import Mapping
+
+
+def rename_keys(record: Mapping, key_map: Mapping) -> dict:
+    """New record with same keys or renamed keys if key found in key_map."""
+
+    new_record = dict()
+
+    for k, v in record.items():
+        key = key_map[k] if k in key_map else k
+        new_record[key] = v
+
+    return new_record
+
+
+def replace_keys(record: Mapping, key_map: Mapping) -> dict:
+    """New record with renamed keys including keys only found in key_map."""
+
+    return {key_map[k]: v for k, v in record.items() if k in key_map}
+
+
+def inject_nulls(data: Mapping, field_names) -> dict:
+    """Insert None as value for missing fields."""
+
+    record = dict()
+
+    for field in field_names:
+        record[field] = data.get(field, None)
+
+    return record

tests/test_agent.py

@@ -0,0 +1,11 @@
+import unittest
+
+from foil.agent import UserAgent
+
+
+class TestAgent(unittest.TestCase):
+    def test_user_agent(self):
+        expected = 'Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)'
+        result = UserAgent.ie
+
+        self.assertEqual(expected, result)

tests/test_dotenv.py

@@ -0,0 +1,35 @@
+import os
+import textwrap
+import unittest
+from tempfile import NamedTemporaryFile
+
+from foil.dotenv import read_dotenv
+
+
+def mock_dotenv():
+    return textwrap.dedent("""\
+SECRET=quiet
+# comment this
+forgot equals
+DB='drawer'""")
+
+
+class TestDotEnv(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        with NamedTemporaryFile(suffix='.env', delete=False) as tmp:
+            with open(tmp.name, 'w', encoding='UTF-8') as file:
+                file.write(mock_dotenv())
+            cls.path = tmp.name
+
+    def test_read_dotenv(self):
+        expected = [('SECRET', 'quiet'), ('DB', 'drawer')]
+        result = list(read_dotenv(self.path))
+
+        self.assertEqual(expected, result)
+
+    @classmethod
+    def tearDown(cls):
+        if os.path.exists(cls.path):
+            os.unlink(cls.path)

tests/test_fileio.py

@@ -9,10 +9,7 @@
 from tempfile import NamedTemporaryFile
 
 from foil.fileio import (concatenate_streams, DelimitedReader,
-                         DelimitedSubsetReader, ZipReader)
-
-
-sample_path = os.path.join(os.path.dirname(__file__), 'sample_data')
+                         DelimitedSubsetReader, TextReader, ZipReader)
 
 
 class MockDialect(csv.Dialect):
@@ -68,6 +65,29 @@ def parse_date(date_str):
         return datetime.strptime(date_str, '%Y-%m-%d').date()
 
 
+class TestTextReader(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        file_content = 'hello\nworld\n'
+        with NamedTemporaryFile(prefix='text_', suffix='.txt', delete=False) as tmp:
+            with open(tmp.name, 'w', encoding='UTF-8') as text_file:
+                text_file.write(file_content)
+            cls.path = tmp.name
+
+    def test_text_reader(self):
+        reader = TextReader(self.path, 'UTF-8')
+
+        expected = ['hello', 'world']
+        result = list(reader)
+
+        self.assertEqual(expected, result)
+
+    @classmethod
+    def tearDownClass(cls):
+        if os.path.exists(cls.path):
+            os.unlink(cls.path)
+
+
 class TestDelimitedReader(unittest.TestCase):
 
     encoding = 'UTF-8'

tests/test_logger.py

@@ -0,0 +1,30 @@
+import json
+import unittest
+from logging import INFO, LogRecord
+
+from foil.logger import JSONFormatter
+
+
+class TestLogFormatter(unittest.TestCase):
+    def test_json_formatter(self):
+        name = 'name'
+        line = 42
+        module = 'some_module'
+        func = 'some_function'
+        msg = {'content': 'sample log'}
+
+        log_record = LogRecord(
+            name, INFO, module, line, msg, None, None, func=func
+        )
+        formatter = JSONFormatter()
+
+        log_result = formatter.format(log_record)
+        result = json.loads(log_result)
+
+        # check some of the fields to ensure json formatted correctly
+        self.assertEqual(name, result['name'])
+        self.assertEqual(line, result['lineNumber'])
+        self.assertEqual(func, result['functionName'])
+        self.assertEqual(module, result['module'])
+        self.assertEqual('INFO', result['level'])
+        self.assertEqual(msg, result['message'])

tests/test_parsers.py

@@ -1,9 +1,11 @@
 import unittest
+from datetime import date
 
 from foil.parsers import (make_converters, parse_bool, passthrough,
-                          parse_float, parse_int, parse_numeric,
-                          parse_quoted_bool, parse_quoted_float,
-                          parse_quoted_int, parse_quoted_numeric,
+                          parse_float, parse_int, parse_int_bool,
+                          parse_iso_date, parse_numeric, parse_quoted_bool,
+                          parse_quoted_float, parse_quoted_int,
+                          parse_quoted_string, parse_quoted_numeric,
                           parse_broken_json)
 
 
@@ -38,6 +40,31 @@ def test_numeric_nan_none(self):
             with self.subTest(val=val):
                 self.assertEqual(parse_numeric(float, val), None)
 
+    def test_int_bool(self):
+        mock_data = [(1, True), (0, False), (None, None)]
+
+        for input_expected in mock_data:
+            with self.subTest(input_expect=input_expected):
+                result = parse_int_bool(input_expected[0])
+                expected = input_expected[1]
+
+                self.assertEqual(expected, result)
+
+    def test_parse_iso_date(self):
+        mock_data = [('2014-04-04', date(2014, 4, 4)), ('', None), (None, None)]
+
+        for input_expected in mock_data:
+            with self.subTest(input_expect=input_expected):
+                result = parse_iso_date(input_expected[0])
+                expected = input_expected[1]
+
+                self.assertEqual(expected, result)
+
+    def test_pass_through(self):
+        expected = 123
+        result = passthrough(expected)
+
+        self.assertEqual(expected, result)
 
 class TestQuotedTextParsers(unittest.TestCase):
     def test_bool(self):
@@ -69,6 +96,16 @@ def test_numeric_nan(self):
             with self.subTest(val=val):
                 self.assertEqual(parse_quoted_numeric(float, val), None)
 
+    def test_parse_quoted_string(self):
+        mock_data = [('""', None), ('"Hello"', 'Hello')]
+
+        for input_expected in mock_data:
+            with self.subTest(input_expect=input_expected):
+                result = parse_quoted_string(input_expected[0])
+                expected = input_expected[1]
+
+                self.assertEqual(expected, result)
+
 
 class Klass:
     pass

tests/test_converters.py → tests/test_records.py

@@ -1,6 +1,7 @@
 import unittest
 
-from foil.converters import rename_keys, replace_keys
+
+from foil.records import inject_nulls, replace_keys, rename_keys
 
 
 class TestKeyConverters(unittest.TestCase):
@@ -19,3 +20,16 @@ def test_replace_keys(self):
         result = replace_keys(self.record, key_map=self.key_map)
 
         self.assertEqual(expected, result)
+
+
+class TestInjectNulls(unittest.TestCase):
+    def test_inject_nulls(self):
+        record = {'city': 'Chicago'}
+        record_copy = record.copy()
+        field_names = ['city', 'state']
+
+        expected = {'city': 'Chicago', 'state': None}
+        result = inject_nulls(record, field_names)
+
+        self.assertEqual(expected, result)
+        self.assertEqual(record_copy, record)

tests/test_serializers.py

@@ -51,5 +51,13 @@ def test_serialize_aware_datetime(self):
 
         self.assertEqual(expected, result)
 
+    def test_serialize_object(self):
+        data = {'hello': 'world'}
+
+        expected = json.dumps(data)
+        result = json_serializer(data)
+
+        self.assertEqual(expected, result)
+
     def _serialize(self, obj):
         return json.dumps(obj, default=json_serializer)

tests/test_util.py

@@ -0,0 +1,13 @@
+import unittest
+
+from foil.util import natural_sort
+
+
+class TestNaturalSort(unittest.TestCase):
+    def test_natural_sort(self):
+        entries = ['ab127b', 'ab123b']
+
+        expected = ['ab123b', 'ab127b']
+        result = natural_sort(entries)
+
+        self.assertEqual(expected, result)