Issue #150: add config variable sleep_from_fit which if set to true c…

…auses sleep_events to be populated from FIT files rather than Garmin Connect JSON data

garmindb/GarminConnectConfig.json.example

@@ -10,7 +10,8 @@
         "rhr_start_date"                : "01/01/2019",
         "monitoring_start_date"         : "01/01/2019",
         "download_latest_activities"    : 25,
-        "download_all_activities"       : 1000
+        "download_all_activities"       : 1000,
+        "sleep_from_fit"                : false
     },
     "copy": {
         "mount_dir"                     : "/Volumes/GARMIN"

garmindb/__init__.py

@@ -20,6 +20,7 @@
 from .statistics import Statistics
 from .tcx import Tcx
 from .monitoring_fit_file_processor import MonitoringFitFileProcessor
+from .sleep_fit_file_processor import SleepFitFileProcessor
 from .export_activities import ActivityExporter
 from .open_with_basecamp import OpenWithBaseCamp
 from .open_with_google_earth import OpenWithGoogleEarth
@@ -34,7 +35,7 @@
 from .plugin_manager import PluginManager
 from .version import format_version, log_version, python_version_check
 
-from .import_monitoring import GarminMonitoringFitData, GarminSummaryData, GarminProfile, GarminWeightData, GarminSleepData, GarminRhrData, GarminSettingsFitData, GarminHydrationData
+from .import_monitoring import GarminMonitoringFitData, GarminSleepFitData, GarminSummaryData, GarminProfile, GarminWeightData, GarminSleepData, GarminRhrData, GarminSettingsFitData, GarminHydrationData
 from .activities_fit_data import GarminActivitiesFitData
 from .garmin_tcx_data import GarminTcxData
 from .garmin_json_data import GarminJsonSummaryData, GarminJsonDetailsData

garmindb/fit_file_processor.py

@@ -21,7 +21,7 @@
 class FitFileProcessor():
     """Class that takes a parsed FIT file object and imports it into a database."""
 
-    def __init__(self, db_params, plugin_manager, debug=0):
+    def __init__(self, db_params, plugin_manager=None, debug=0):
         """
         Return a new FitFileProcessor instance.
 

garmindb/garmin_connect_config_manager.py

@@ -85,6 +85,10 @@ def stat_start_date(self, stat_type):
         days = (datetime.datetime.now().date() - date).days
         return (date, days)
 
+    def sleep_from_fit(self):
+        """Return a boolean. True means sleep data comes from FIT files. False means sleep data comes from Garmin Connect downloads."""
+        return self.__get_node_value('data', 'sleep_from_fit')
+
     def device_mount_dir(self):
         """Return the directory where the Garmin USB device is mounted."""
         return self.__get_node_value('copy', 'mount_dir')

garmindb/import_monitoring.py

@@ -75,6 +75,24 @@ def __init__(self, input_dir, latest, measurement_system, debug):
         super().__init__(input_dir, debug, latest, True, [fitfile.FileType.monitoring_b], measurement_system)
 
 
+class GarminSleepFitData(FitData):
+    """Class for importing sleep FIT files into a database."""
+
+    def __init__(self, input_dir, latest, measurement_system, debug):
+        """
+        Return an instance of GarminSleepFitData.
+
+        Parameters:
+        ----------
+        input_dir (string): directory (full path) to check for monitoring data files
+        latest (Boolean): check for latest files only
+        measurement_system (enum): which measurement system to use when importing the files
+        debug (Boolean): enable debug logging
+
+        """
+        super().__init__(input_dir, debug, latest, True, [fitfile.FileType.sleep], measurement_system)
+
+
 class GarminSettingsFitData(FitData):
     """Class for importing settings FIT files into a database."""
 
@@ -133,6 +151,8 @@ def __init__(self, db_params, input_dir, latest, debug):
             'sleepTimeSeconds': fitfile.conversions.secs_to_dt_time,
             'sleepStartTimestampGMT': Conversions.epoch_ms_to_dt,
             'sleepEndTimestampGMT': Conversions.epoch_ms_to_dt,
+            'sleepStartTimestampLocal': Conversions.epoch_ms_to_dt,
+            'sleepEndTimestampLocal': Conversions.epoch_ms_to_dt,
             'deepSleepSeconds': fitfile.conversions.secs_to_dt_time,
             'lightSleepSeconds': fitfile.conversions.secs_to_dt_time,
             'remSleepSeconds': fitfile.conversions.secs_to_dt_time,
@@ -149,11 +169,19 @@ def _process_json(self, json_data):
         if date is None:
             return 0
         day = date.date()
+        # Find the UTC offset so we can convert times to local
+        start_utc = daily_sleep.get('sleepStartTimestampGMT')
+        start_local = daily_sleep.get('sleepStartTimestampLocal')
+        if start_utc and start_local:
+            utc_offset = (start_local - start_utc).total_seconds()
+        else:
+            utc_offset = 0
+        self.local_tz = datetime.timezone(datetime.timedelta(seconds=utc_offset))
         if json_data.get('remSleepData'):
-            root_logger.info("Importing %s with REM data", day)
+            root_logger.info("Importing %s with REM data and UTC offset %r", day, utc_offset)
             sleep_activity_levels = RemSleepActivityLevels
         else:
-            root_logger.info("Importing %s without REM data", day)
+            root_logger.info("Importing %s without REM data and UTC offset %r", day, utc_offset)
             sleep_activity_levels = SleepActivityLevels
         day_data = {
             'day': day,
@@ -165,23 +193,23 @@ def _process_json(self, json_data):
             'rem_sleep': daily_sleep.get('remSleepSeconds'),
             'awake': daily_sleep.get('awakeSleepSeconds')
         }
-        Sleep.insert_or_update(
-            self.garmin_db, day_data, ignore_none=True)
+        Sleep.insert_or_update(self.garmin_db, day_data, ignore_none=True)
         sleep_levels = json_data.get('sleepLevels')
         if sleep_levels is None:
             return 0
         for sleep_level in sleep_levels:
             start = sleep_level['startGMT']
+            start_local = start + datetime.timedelta(seconds=utc_offset)
             end = sleep_level['endGMT']
             event = sleep_activity_levels(sleep_level['activityLevel'])
             duration = (datetime.datetime.min + (end - start)).time()
+            root_logger.info("Sleep event %r (%r) %r", start_local, start, event)
             level_data = {
-                'timestamp': start,
+                'timestamp': start_local,
                 'event': event.name,
                 'duration': duration
             }
-            SleepEvents.insert_or_update(
-                self.garmin_db, level_data, ignore_none=True)
+            SleepEvents.insert_or_update(self.garmin_db, level_data, ignore_none=True)
         return len(sleep_levels)
 
 

garmindb/sleep_fit_file_processor.py

@@ -0,0 +1,45 @@
+"""Class that takes a parsed monitoring FIT file object and imports it into a database."""
+
+__author__ = "Tom Goetz"
+__copyright__ = "Copyright Tom Goetz"
+__license__ = "GPL"
+
+import logging
+import sys
+
+import fitfile
+
+from .garmindb import SleepEvents
+from .fit_file_processor import FitFileProcessor
+
+
+logger = logging.getLogger(__file__)
+logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+root_logger = logging.getLogger()
+
+
+class SleepFitFileProcessor(FitFileProcessor):
+    """Class that takes a parsed sleep FIT file object and imports it into a database."""
+
+    def write_file(self, fit_file):
+        """Given a Fit File object, write all of its messages to the DB."""
+        self.last_sleep_event = None
+        self.last_sleep_level = None
+        with self.garmin_db.managed_session() as self.garmin_db_session:
+            self._write_message_types(fit_file, fit_file.message_types)
+
+    def _write_sleep_level_entry(self, fit_file, message_fields):
+        logger.debug("sleep level message: %r", message_fields)
+        timestamp = fit_file.utc_datetime_to_local(message_fields.timestamp)
+        sleep_level = message_fields.get('sleep_level')
+        if sleep_level.value > fitfile.field_enums.SleepActivityLevel.unknown.value and self.last_sleep_event is not None and \
+           (sleep_level is not fitfile.field_enums.SleepActivityLevel.awake or self.last_sleep_level is not fitfile.field_enums.SleepActivityLevel.awake):
+            sleep_event = {
+                'timestamp' : fit_file.utc_datetime_to_local(self.last_sleep_event),
+                'event'     : sleep_level.name,
+                'duration'  : fitfile.conversions.timedelta_to_time(timestamp - self.last_sleep_event)
+            }
+            logger.debug("sleep level event: %r", sleep_event)
+            SleepEvents.s_insert_or_update(self.garmin_db_session, sleep_event)
+        self.last_sleep_event = timestamp
+        self.last_sleep_level = sleep_level

garmindb/version_info.py

@@ -7,7 +7,7 @@
 
 python_required = (3, 0, 0)
 python_tested = (3, 9, 10)
-version_info = (3, 2, 4)
+version_info = (3, 2, 5)
 prerelease = True
 
 

requirements.txt

@@ -7,6 +7,6 @@ matplotlib
 cloudscraper
 ipykernel
 ipyleaflet
-fitfile>=1.1.2
+fitfile>=1.1.3
 tcxfile>=1.0.4
 idbutils>=1.0.6

scripts/garmindb_cli.py

@@ -24,8 +24,9 @@
 from garmindb.summarydb import SummaryDb
 
 from garmindb import Download, Copy, Analyze
-from garmindb import FitFileProcessor, ActivityFitFileProcessor, MonitoringFitFileProcessor
-from garmindb import GarminProfile, GarminWeightData, GarminSummaryData, GarminMonitoringFitData, GarminSleepData, GarminRhrData, GarminSettingsFitData, GarminHydrationData
+from garmindb import FitFileProcessor, ActivityFitFileProcessor, MonitoringFitFileProcessor, SleepFitFileProcessor
+from garmindb import GarminProfile, GarminWeightData, GarminSummaryData, GarminMonitoringFitData, GarminSleepFitData, GarminSleepData, GarminRhrData, GarminSettingsFitData, \
+    GarminHydrationData
 from garmindb import GarminJsonSummaryData, GarminJsonDetailsData, GarminTcxData, GarminActivitiesFitData
 from garmindb import ActivityExporter
 
@@ -179,8 +180,8 @@ def import_data(debug, latest, stats):
         if gwd.file_count() > 0:
             gwd.process()
 
+    monitoring_dir = ConfigManager.get_or_create_monitoring_base_dir()
     if Statistics.monitoring in stats:
-        monitoring_dir = ConfigManager.get_or_create_monitoring_base_dir()
         gsd = GarminSummaryData(db_params_dict, monitoring_dir, latest, measurement_system, debug)
         if gsd.file_count() > 0:
             gsd.process()
@@ -194,10 +195,15 @@ def import_data(debug, latest, stats):
             gfd.process_files(MonitoringFitFileProcessor(db_params_dict, plugin_manager, debug))
 
     if Statistics.sleep in stats:
-        sleep_dir = ConfigManager.get_or_create_sleep_dir()
-        gsd = GarminSleepData(db_params_dict, sleep_dir, latest, debug)
-        if gsd.file_count() > 0:
-            gsd.process()
+        if gc_config.sleep_from_fit():
+            gsd = GarminSleepFitData(monitoring_dir, latest=False, measurement_system=measurement_system, debug=2)
+            if gsd.file_count() > 0:
+                gsd.process_files(SleepFitFileProcessor(db_params_dict))
+        else:
+            sleep_dir = ConfigManager.get_or_create_sleep_dir()
+            gsd = GarminSleepData(db_params_dict, sleep_dir, latest, debug)
+            if gsd.file_count() > 0:
+                gsd.process()
 
     if Statistics.rhr in stats:
         rhr_dir = ConfigManager.get_or_create_rhr_dir()

test/test_fit_file.py

@@ -112,8 +112,6 @@ def check_file_id(self, fit_file, file_type):
         self.assertLessEqual(fit_file.utc_offset, +46800, 'Is not a valid time zone offset')
         # file contains less than a day span of time
         (self.start_time, self.end_time) = fit_file.date_span()
-        if fit_file.product != fitfile.GarminProduct.connect:
-            self.check_timestamp_delta(fit_file, self.start_time, self.end_time, (0, 86400))
         for message in fit_file.file_id:
             self.check_value(fit_file, message, 'manufacturer', fitfile.Manufacturer.Garmin)
             self.check_value(fit_file, message, 'type', file_type)

test/test_garmin_db.py

@@ -10,7 +10,7 @@
 
 import fitfile
 
-from garmindb import ConfigManager
+from garmindb import ConfigManager, GarminSleepFitData, SleepFitFileProcessor
 from garmindb.garmindb import GarminDb, Attributes, Device, DeviceInfo, File, Weight, Stress, Sleep, SleepEvents, RestingHeartRate
 
 from test_db_base import TestDBBase
@@ -29,7 +29,10 @@ class TestGarminDb(TestDBBase, unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         db_params = ConfigManager.get_db_params()
+        cls.test_db_params = ConfigManager.get_db_params(test_db=True)
+        print(f"db params {repr(cls.test_db_params)}")
         cls.garmin_db = GarminDb(db_params)
+        cls.measurement_system = fitfile.field_enums.DisplayMeasure.statute
         table_dict = {
             'attributes_table': Attributes,
             'device_table': Device,
@@ -118,6 +121,12 @@ def test_measurement_system(self):
         measurement_system = Attributes.measurements_type(self.garmin_db, fitfile.field_enums.DisplayMeasure.metric)
         self.assertIn(measurement_system, fitfile.field_enums.DisplayMeasure)
 
+    def test_sleep_import(self):
+        gfd = GarminSleepFitData('test_files/fit/sleep', latest=False, measurement_system=self.measurement_system, debug=2)
+        self.gfd_file_count = gfd.file_count()
+        if gfd.file_count() > 0:
+            gfd.process_files(SleepFitFileProcessor(self.test_db_params))
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)