diff --git a/Fit b/Fit index e5a5eca..c1eb5d4 160000 --- a/Fit +++ b/Fit @@ -1 +1 @@ -Subproject commit e5a5ecad998d25dc34feff1fc49513b393e6195e +Subproject commit c1eb5d4952d46240b40bb6af39d5201c35619fef diff --git a/garmindb/GarminConnectConfig.json.example b/garmindb/GarminConnectConfig.json.example index 616024c..712189d 100644 --- a/garmindb/GarminConnectConfig.json.example +++ b/garmindb/GarminConnectConfig.json.example @@ -10,7 +10,8 @@ "rhr_start_date" : "01/01/2019", "monitoring_start_date" : "01/01/2019", "download_latest_activities" : 25, - "download_all_activities" : 1000 + "download_all_activities" : 1000, + "sleep_from_fit" : false }, "copy": { "mount_dir" : "/Volumes/GARMIN" diff --git a/garmindb/__init__.py b/garmindb/__init__.py index 5d9b9be..ff16e42 100644 --- a/garmindb/__init__.py +++ b/garmindb/__init__.py @@ -20,6 +20,7 @@ from .statistics import Statistics from .tcx import Tcx from .monitoring_fit_file_processor import MonitoringFitFileProcessor +from .sleep_fit_file_processor import SleepFitFileProcessor from .export_activities import ActivityExporter from .open_with_basecamp import OpenWithBaseCamp from .open_with_google_earth import OpenWithGoogleEarth @@ -34,7 +35,7 @@ from .plugin_manager import PluginManager from .version import format_version, log_version, python_version_check -from .import_monitoring import GarminMonitoringFitData, GarminSummaryData, GarminProfile, GarminWeightData, GarminSleepData, GarminRhrData, GarminSettingsFitData, GarminHydrationData +from .import_monitoring import GarminMonitoringFitData, GarminSleepFitData, GarminSummaryData, GarminProfile, GarminWeightData, GarminSleepData, GarminRhrData, GarminSettingsFitData, GarminHydrationData from .activities_fit_data import GarminActivitiesFitData from .garmin_tcx_data import GarminTcxData from .garmin_json_data import GarminJsonSummaryData, GarminJsonDetailsData diff --git a/garmindb/fit_file_processor.py b/garmindb/fit_file_processor.py index dfb172e..7d08349 100644 --- a/garmindb/fit_file_processor.py +++ b/garmindb/fit_file_processor.py @@ -21,7 +21,7 @@ class FitFileProcessor(): """Class that takes a parsed FIT file object and imports it into a database.""" - def __init__(self, db_params, plugin_manager, debug=0): + def __init__(self, db_params, plugin_manager=None, debug=0): """ Return a new FitFileProcessor instance. diff --git a/garmindb/garmin_connect_config_manager.py b/garmindb/garmin_connect_config_manager.py index 396d91b..b4d8ad2 100644 --- a/garmindb/garmin_connect_config_manager.py +++ b/garmindb/garmin_connect_config_manager.py @@ -85,6 +85,10 @@ def stat_start_date(self, stat_type): days = (datetime.datetime.now().date() - date).days return (date, days) + def sleep_from_fit(self): + """Return a boolean. True means sleep data comes from FIT files. False means sleep data comes from Garmin Connect downloads.""" + return self.__get_node_value('data', 'sleep_from_fit') + def device_mount_dir(self): """Return the directory where the Garmin USB device is mounted.""" return self.__get_node_value('copy', 'mount_dir') diff --git a/garmindb/import_monitoring.py b/garmindb/import_monitoring.py index b2c1241..eb73865 100755 --- a/garmindb/import_monitoring.py +++ b/garmindb/import_monitoring.py @@ -75,6 +75,24 @@ def __init__(self, input_dir, latest, measurement_system, debug): super().__init__(input_dir, debug, latest, True, [fitfile.FileType.monitoring_b], measurement_system) +class GarminSleepFitData(FitData): + """Class for importing sleep FIT files into a database.""" + + def __init__(self, input_dir, latest, measurement_system, debug): + """ + Return an instance of GarminSleepFitData. + + Parameters: + ---------- + input_dir (string): directory (full path) to check for monitoring data files + latest (Boolean): check for latest files only + measurement_system (enum): which measurement system to use when importing the files + debug (Boolean): enable debug logging + + """ + super().__init__(input_dir, debug, latest, True, [fitfile.FileType.sleep], measurement_system) + + class GarminSettingsFitData(FitData): """Class for importing settings FIT files into a database.""" @@ -133,6 +151,8 @@ def __init__(self, db_params, input_dir, latest, debug): 'sleepTimeSeconds': fitfile.conversions.secs_to_dt_time, 'sleepStartTimestampGMT': Conversions.epoch_ms_to_dt, 'sleepEndTimestampGMT': Conversions.epoch_ms_to_dt, + 'sleepStartTimestampLocal': Conversions.epoch_ms_to_dt, + 'sleepEndTimestampLocal': Conversions.epoch_ms_to_dt, 'deepSleepSeconds': fitfile.conversions.secs_to_dt_time, 'lightSleepSeconds': fitfile.conversions.secs_to_dt_time, 'remSleepSeconds': fitfile.conversions.secs_to_dt_time, @@ -149,11 +169,19 @@ def _process_json(self, json_data): if date is None: return 0 day = date.date() + # Find the UTC offset so we can convert times to local + start_utc = daily_sleep.get('sleepStartTimestampGMT') + start_local = daily_sleep.get('sleepStartTimestampLocal') + if start_utc and start_local: + utc_offset = (start_local - start_utc).total_seconds() + else: + utc_offset = 0 + self.local_tz = datetime.timezone(datetime.timedelta(seconds=utc_offset)) if json_data.get('remSleepData'): - root_logger.info("Importing %s with REM data", day) + root_logger.info("Importing %s with REM data and UTC offset %r", day, utc_offset) sleep_activity_levels = RemSleepActivityLevels else: - root_logger.info("Importing %s without REM data", day) + root_logger.info("Importing %s without REM data and UTC offset %r", day, utc_offset) sleep_activity_levels = SleepActivityLevels day_data = { 'day': day, @@ -165,23 +193,23 @@ def _process_json(self, json_data): 'rem_sleep': daily_sleep.get('remSleepSeconds'), 'awake': daily_sleep.get('awakeSleepSeconds') } - Sleep.insert_or_update( - self.garmin_db, day_data, ignore_none=True) + Sleep.insert_or_update(self.garmin_db, day_data, ignore_none=True) sleep_levels = json_data.get('sleepLevels') if sleep_levels is None: return 0 for sleep_level in sleep_levels: start = sleep_level['startGMT'] + start_local = start + datetime.timedelta(seconds=utc_offset) end = sleep_level['endGMT'] event = sleep_activity_levels(sleep_level['activityLevel']) duration = (datetime.datetime.min + (end - start)).time() + root_logger.info("Sleep event %r (%r) %r", start_local, start, event) level_data = { - 'timestamp': start, + 'timestamp': start_local, 'event': event.name, 'duration': duration } - SleepEvents.insert_or_update( - self.garmin_db, level_data, ignore_none=True) + SleepEvents.insert_or_update(self.garmin_db, level_data, ignore_none=True) return len(sleep_levels) diff --git a/garmindb/sleep_fit_file_processor.py b/garmindb/sleep_fit_file_processor.py new file mode 100644 index 0000000..22012ce --- /dev/null +++ b/garmindb/sleep_fit_file_processor.py @@ -0,0 +1,45 @@ +"""Class that takes a parsed monitoring FIT file object and imports it into a database.""" + +__author__ = "Tom Goetz" +__copyright__ = "Copyright Tom Goetz" +__license__ = "GPL" + +import logging +import sys + +import fitfile + +from .garmindb import SleepEvents +from .fit_file_processor import FitFileProcessor + + +logger = logging.getLogger(__file__) +logger.addHandler(logging.StreamHandler(stream=sys.stdout)) +root_logger = logging.getLogger() + + +class SleepFitFileProcessor(FitFileProcessor): + """Class that takes a parsed sleep FIT file object and imports it into a database.""" + + def write_file(self, fit_file): + """Given a Fit File object, write all of its messages to the DB.""" + self.last_sleep_event = None + self.last_sleep_level = None + with self.garmin_db.managed_session() as self.garmin_db_session: + self._write_message_types(fit_file, fit_file.message_types) + + def _write_sleep_level_entry(self, fit_file, message_fields): + logger.debug("sleep level message: %r", message_fields) + timestamp = fit_file.utc_datetime_to_local(message_fields.timestamp) + sleep_level = message_fields.get('sleep_level') + if sleep_level.value > fitfile.field_enums.SleepActivityLevel.unknown.value and self.last_sleep_event is not None and \ + (sleep_level is not fitfile.field_enums.SleepActivityLevel.awake or self.last_sleep_level is not fitfile.field_enums.SleepActivityLevel.awake): + sleep_event = { + 'timestamp' : fit_file.utc_datetime_to_local(self.last_sleep_event), + 'event' : sleep_level.name, + 'duration' : fitfile.conversions.timedelta_to_time(timestamp - self.last_sleep_event) + } + logger.debug("sleep level event: %r", sleep_event) + SleepEvents.s_insert_or_update(self.garmin_db_session, sleep_event) + self.last_sleep_event = timestamp + self.last_sleep_level = sleep_level diff --git a/garmindb/version_info.py b/garmindb/version_info.py index 153e5be..1887c13 100644 --- a/garmindb/version_info.py +++ b/garmindb/version_info.py @@ -7,7 +7,7 @@ python_required = (3, 0, 0) python_tested = (3, 9, 10) -version_info = (3, 2, 4) +version_info = (3, 2, 5) prerelease = True diff --git a/requirements.txt b/requirements.txt index 60a8d8a..72def9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,6 @@ matplotlib cloudscraper ipykernel ipyleaflet -fitfile>=1.1.2 +fitfile>=1.1.3 tcxfile>=1.0.4 idbutils>=1.0.6 diff --git a/scripts/garmindb_cli.py b/scripts/garmindb_cli.py index 8163a03..b4a7412 100755 --- a/scripts/garmindb_cli.py +++ b/scripts/garmindb_cli.py @@ -24,8 +24,9 @@ from garmindb.summarydb import SummaryDb from garmindb import Download, Copy, Analyze -from garmindb import FitFileProcessor, ActivityFitFileProcessor, MonitoringFitFileProcessor -from garmindb import GarminProfile, GarminWeightData, GarminSummaryData, GarminMonitoringFitData, GarminSleepData, GarminRhrData, GarminSettingsFitData, GarminHydrationData +from garmindb import FitFileProcessor, ActivityFitFileProcessor, MonitoringFitFileProcessor, SleepFitFileProcessor +from garmindb import GarminProfile, GarminWeightData, GarminSummaryData, GarminMonitoringFitData, GarminSleepFitData, GarminSleepData, GarminRhrData, GarminSettingsFitData, \ + GarminHydrationData from garmindb import GarminJsonSummaryData, GarminJsonDetailsData, GarminTcxData, GarminActivitiesFitData from garmindb import ActivityExporter @@ -179,8 +180,8 @@ def import_data(debug, latest, stats): if gwd.file_count() > 0: gwd.process() + monitoring_dir = ConfigManager.get_or_create_monitoring_base_dir() if Statistics.monitoring in stats: - monitoring_dir = ConfigManager.get_or_create_monitoring_base_dir() gsd = GarminSummaryData(db_params_dict, monitoring_dir, latest, measurement_system, debug) if gsd.file_count() > 0: gsd.process() @@ -194,10 +195,15 @@ def import_data(debug, latest, stats): gfd.process_files(MonitoringFitFileProcessor(db_params_dict, plugin_manager, debug)) if Statistics.sleep in stats: - sleep_dir = ConfigManager.get_or_create_sleep_dir() - gsd = GarminSleepData(db_params_dict, sleep_dir, latest, debug) - if gsd.file_count() > 0: - gsd.process() + if gc_config.sleep_from_fit(): + gsd = GarminSleepFitData(monitoring_dir, latest=False, measurement_system=measurement_system, debug=2) + if gsd.file_count() > 0: + gsd.process_files(SleepFitFileProcessor(db_params_dict)) + else: + sleep_dir = ConfigManager.get_or_create_sleep_dir() + gsd = GarminSleepData(db_params_dict, sleep_dir, latest, debug) + if gsd.file_count() > 0: + gsd.process() if Statistics.rhr in stats: rhr_dir = ConfigManager.get_or_create_rhr_dir() diff --git a/test/test_fit_file.py b/test/test_fit_file.py index e304f59..1e60e53 100644 --- a/test/test_fit_file.py +++ b/test/test_fit_file.py @@ -112,8 +112,6 @@ def check_file_id(self, fit_file, file_type): self.assertLessEqual(fit_file.utc_offset, +46800, 'Is not a valid time zone offset') # file contains less than a day span of time (self.start_time, self.end_time) = fit_file.date_span() - if fit_file.product != fitfile.GarminProduct.connect: - self.check_timestamp_delta(fit_file, self.start_time, self.end_time, (0, 86400)) for message in fit_file.file_id: self.check_value(fit_file, message, 'manufacturer', fitfile.Manufacturer.Garmin) self.check_value(fit_file, message, 'type', file_type) diff --git a/test/test_garmin_db.py b/test/test_garmin_db.py index 73f3de4..d8d58b8 100644 --- a/test/test_garmin_db.py +++ b/test/test_garmin_db.py @@ -10,7 +10,7 @@ import fitfile -from garmindb import ConfigManager +from garmindb import ConfigManager, GarminSleepFitData, SleepFitFileProcessor from garmindb.garmindb import GarminDb, Attributes, Device, DeviceInfo, File, Weight, Stress, Sleep, SleepEvents, RestingHeartRate from test_db_base import TestDBBase @@ -29,7 +29,10 @@ class TestGarminDb(TestDBBase, unittest.TestCase): @classmethod def setUpClass(cls): db_params = ConfigManager.get_db_params() + cls.test_db_params = ConfigManager.get_db_params(test_db=True) + print(f"db params {repr(cls.test_db_params)}") cls.garmin_db = GarminDb(db_params) + cls.measurement_system = fitfile.field_enums.DisplayMeasure.statute table_dict = { 'attributes_table': Attributes, 'device_table': Device, @@ -118,6 +121,12 @@ def test_measurement_system(self): measurement_system = Attributes.measurements_type(self.garmin_db, fitfile.field_enums.DisplayMeasure.metric) self.assertIn(measurement_system, fitfile.field_enums.DisplayMeasure) + def test_sleep_import(self): + gfd = GarminSleepFitData('test_files/fit/sleep', latest=False, measurement_system=self.measurement_system, debug=2) + self.gfd_file_count = gfd.file_count() + if gfd.file_count() > 0: + gfd.process_files(SleepFitFileProcessor(self.test_db_params)) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/utilities b/utilities index 6962a5e..4960a07 160000 --- a/utilities +++ b/utilities @@ -1 +1 @@ -Subproject commit 6962a5e28d857fc0b6ffcb1a7755bcabc4ac1542 +Subproject commit 4960a078f1ea1acae936a902a072350688f812a7