diff --git a/.travis.yml b/.travis.yml index af622a7..82f875c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ install: - python setup.py install script: - - py.test tests -s --cov + - py.test tests --cov foil --cov tests -s after_success: - codecov diff --git a/foil/_version.py b/foil/_version.py index 3edc45f..33adb8a 100644 --- a/foil/_version.py +++ b/foil/_version.py @@ -1,3 +1,3 @@ -version_info = (0, 2, 2) +version_info = (0, 2, 3) __version__ = '.'.join(map(str, version_info)) diff --git a/foil/converters.py b/foil/converters.py index 3bc8055..5794977 100644 --- a/foil/converters.py +++ b/foil/converters.py @@ -1,19 +1,3 @@ -from typing import Mapping - -def rename_keys(record: Mapping, key_map: Mapping) -> dict: - """New record with same keys or renamed keys if key found in key_map.""" - - new_record = dict() - - for k, v in record.items(): - key = key_map[k] if k in key_map else k - new_record[key] = v - - return new_record - - -def replace_keys(record: Mapping, key_map: Mapping) -> dict: - """New record with renamed keys including keys only found in key_map.""" - - return {key_map[k]: v for k, v in record.items() if k in key_map} +# moved to foil/records.py +from foil.records import replace_keys, rename_keys diff --git a/foil/dotenv.py b/foil/dotenv.py index 0a2c839..36cd0a7 100644 --- a/foil/dotenv.py +++ b/foil/dotenv.py @@ -20,4 +20,5 @@ def read_dotenv(dotenv_path: str): env_key, env_value = line.split('=', 1) env_value = env_value.strip("'").strip('"') + yield env_key, env_value diff --git a/foil/parsers.py b/foil/parsers.py index c053b0e..103468c 100644 --- a/foil/parsers.py +++ b/foil/parsers.py @@ -24,6 +24,7 @@ def passthrough(value): """Pass through value for no conversion.""" + return value diff --git a/foil/records.py b/foil/records.py new file mode 100644 index 0000000..9385ae8 --- /dev/null +++ b/foil/records.py @@ -0,0 +1,32 @@ +"""Functionality for manipulating dictionary records.""" + +from typing import Mapping + + +def rename_keys(record: Mapping, key_map: Mapping) -> dict: + """New record with same keys or renamed keys if key found in key_map.""" + + new_record = dict() + + for k, v in record.items(): + key = key_map[k] if k in key_map else k + new_record[key] = v + + return new_record + + +def replace_keys(record: Mapping, key_map: Mapping) -> dict: + """New record with renamed keys including keys only found in key_map.""" + + return {key_map[k]: v for k, v in record.items() if k in key_map} + + +def inject_nulls(data: Mapping, field_names) -> dict: + """Insert None as value for missing fields.""" + + record = dict() + + for field in field_names: + record[field] = data.get(field, None) + + return record diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..8ea76ac --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,11 @@ +import unittest + +from foil.agent import UserAgent + + +class TestAgent(unittest.TestCase): + def test_user_agent(self): + expected = 'Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)' + result = UserAgent.ie + + self.assertEqual(expected, result) diff --git a/tests/test_dotenv.py b/tests/test_dotenv.py new file mode 100644 index 0000000..2d0e2d0 --- /dev/null +++ b/tests/test_dotenv.py @@ -0,0 +1,35 @@ +import os +import textwrap +import unittest +from tempfile import NamedTemporaryFile + +from foil.dotenv import read_dotenv + + +def mock_dotenv(): + return textwrap.dedent("""\ +SECRET=quiet +# comment this +forgot equals +DB='drawer'""") + + +class TestDotEnv(unittest.TestCase): + + @classmethod + def setUpClass(cls): + with NamedTemporaryFile(suffix='.env', delete=False) as tmp: + with open(tmp.name, 'w', encoding='UTF-8') as file: + file.write(mock_dotenv()) + cls.path = tmp.name + + def test_read_dotenv(self): + expected = [('SECRET', 'quiet'), ('DB', 'drawer')] + result = list(read_dotenv(self.path)) + + self.assertEqual(expected, result) + + @classmethod + def tearDown(cls): + if os.path.exists(cls.path): + os.unlink(cls.path) diff --git a/tests/test_fileio.py b/tests/test_fileio.py index 9a3db47..1805714 100644 --- a/tests/test_fileio.py +++ b/tests/test_fileio.py @@ -9,10 +9,7 @@ from tempfile import NamedTemporaryFile from foil.fileio import (concatenate_streams, DelimitedReader, - DelimitedSubsetReader, ZipReader) - - -sample_path = os.path.join(os.path.dirname(__file__), 'sample_data') + DelimitedSubsetReader, TextReader, ZipReader) class MockDialect(csv.Dialect): @@ -68,6 +65,29 @@ def parse_date(date_str): return datetime.strptime(date_str, '%Y-%m-%d').date() +class TestTextReader(unittest.TestCase): + @classmethod + def setUpClass(cls): + file_content = 'hello\nworld\n' + with NamedTemporaryFile(prefix='text_', suffix='.txt', delete=False) as tmp: + with open(tmp.name, 'w', encoding='UTF-8') as text_file: + text_file.write(file_content) + cls.path = tmp.name + + def test_text_reader(self): + reader = TextReader(self.path, 'UTF-8') + + expected = ['hello', 'world'] + result = list(reader) + + self.assertEqual(expected, result) + + @classmethod + def tearDownClass(cls): + if os.path.exists(cls.path): + os.unlink(cls.path) + + class TestDelimitedReader(unittest.TestCase): encoding = 'UTF-8' diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..fe7a111 --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,30 @@ +import json +import unittest +from logging import INFO, LogRecord + +from foil.logger import JSONFormatter + + +class TestLogFormatter(unittest.TestCase): + def test_json_formatter(self): + name = 'name' + line = 42 + module = 'some_module' + func = 'some_function' + msg = {'content': 'sample log'} + + log_record = LogRecord( + name, INFO, module, line, msg, None, None, func=func + ) + formatter = JSONFormatter() + + log_result = formatter.format(log_record) + result = json.loads(log_result) + + # check some of the fields to ensure json formatted correctly + self.assertEqual(name, result['name']) + self.assertEqual(line, result['lineNumber']) + self.assertEqual(func, result['functionName']) + self.assertEqual(module, result['module']) + self.assertEqual('INFO', result['level']) + self.assertEqual(msg, result['message']) diff --git a/tests/test_parsers.py b/tests/test_parsers.py index 214451e..c5c5027 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -1,9 +1,11 @@ import unittest +from datetime import date from foil.parsers import (make_converters, parse_bool, passthrough, - parse_float, parse_int, parse_numeric, - parse_quoted_bool, parse_quoted_float, - parse_quoted_int, parse_quoted_numeric, + parse_float, parse_int, parse_int_bool, + parse_iso_date, parse_numeric, parse_quoted_bool, + parse_quoted_float, parse_quoted_int, + parse_quoted_string, parse_quoted_numeric, parse_broken_json) @@ -38,6 +40,31 @@ def test_numeric_nan_none(self): with self.subTest(val=val): self.assertEqual(parse_numeric(float, val), None) + def test_int_bool(self): + mock_data = [(1, True), (0, False), (None, None)] + + for input_expected in mock_data: + with self.subTest(input_expect=input_expected): + result = parse_int_bool(input_expected[0]) + expected = input_expected[1] + + self.assertEqual(expected, result) + + def test_parse_iso_date(self): + mock_data = [('2014-04-04', date(2014, 4, 4)), ('', None), (None, None)] + + for input_expected in mock_data: + with self.subTest(input_expect=input_expected): + result = parse_iso_date(input_expected[0]) + expected = input_expected[1] + + self.assertEqual(expected, result) + + def test_pass_through(self): + expected = 123 + result = passthrough(expected) + + self.assertEqual(expected, result) class TestQuotedTextParsers(unittest.TestCase): def test_bool(self): @@ -69,6 +96,16 @@ def test_numeric_nan(self): with self.subTest(val=val): self.assertEqual(parse_quoted_numeric(float, val), None) + def test_parse_quoted_string(self): + mock_data = [('""', None), ('"Hello"', 'Hello')] + + for input_expected in mock_data: + with self.subTest(input_expect=input_expected): + result = parse_quoted_string(input_expected[0]) + expected = input_expected[1] + + self.assertEqual(expected, result) + class Klass: pass diff --git a/tests/test_converters.py b/tests/test_records.py similarity index 54% rename from tests/test_converters.py rename to tests/test_records.py index 1746cf2..5eee7a9 100644 --- a/tests/test_converters.py +++ b/tests/test_records.py @@ -1,6 +1,7 @@ import unittest -from foil.converters import rename_keys, replace_keys + +from foil.records import inject_nulls, replace_keys, rename_keys class TestKeyConverters(unittest.TestCase): @@ -19,3 +20,16 @@ def test_replace_keys(self): result = replace_keys(self.record, key_map=self.key_map) self.assertEqual(expected, result) + + +class TestInjectNulls(unittest.TestCase): + def test_inject_nulls(self): + record = {'city': 'Chicago'} + record_copy = record.copy() + field_names = ['city', 'state'] + + expected = {'city': 'Chicago', 'state': None} + result = inject_nulls(record, field_names) + + self.assertEqual(expected, result) + self.assertEqual(record_copy, record) diff --git a/tests/test_serializers.py b/tests/test_serializers.py index db1bce1..8ceebab 100644 --- a/tests/test_serializers.py +++ b/tests/test_serializers.py @@ -51,5 +51,13 @@ def test_serialize_aware_datetime(self): self.assertEqual(expected, result) + def test_serialize_object(self): + data = {'hello': 'world'} + + expected = json.dumps(data) + result = json_serializer(data) + + self.assertEqual(expected, result) + def _serialize(self, obj): return json.dumps(obj, default=json_serializer) diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..c4bc386 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,13 @@ +import unittest + +from foil.util import natural_sort + + +class TestNaturalSort(unittest.TestCase): + def test_natural_sort(self): + entries = ['ab127b', 'ab123b'] + + expected = ['ab123b', 'ab127b'] + result = natural_sort(entries) + + self.assertEqual(expected, result)