diff --git a/openaq_api/openaq_api/v3/routers/measurements.py b/openaq_api/openaq_api/v3/routers/measurements.py index 1b3b00c..5ec281b 100644 --- a/openaq_api/openaq_api/v3/routers/measurements.py +++ b/openaq_api/openaq_api/v3/routers/measurements.py @@ -402,6 +402,8 @@ async def fetch_measurements_aggregated(query, aggregate_to, db): else: raise Exception(f"{aggregate_to} is not supported") + query.set_column_map({"timezone": "tz.tzid"}) + sql = f""" WITH meas AS ( SELECT @@ -413,8 +415,8 @@ async def fetch_measurements_aggregated(query, aggregate_to, db): , AVG(s.data_logging_period_seconds) as log_seconds , MAX(truncate_timestamp(datetime, '{aggregate_to}', tz.tzid, '1{aggregate_to}'::interval)) as last_period - , MIN(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_first - , MAX(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_last + , MIN(timezone(tz.tzid, datetime)) as datetime_first + , MAX(timezone(tz.tzid, datetime)) as datetime_last , COUNT(1) as value_count , AVG(value) as value_avg , STDDEV(value) as value_sd @@ -464,11 +466,11 @@ async def fetch_measurements_aggregated(query, aggregate_to, db): -------- , calculate_coverage( value_count::int - , {interval_seconds} - , {interval_seconds} + , avg_seconds + , log_seconds , EXTRACT(EPOCH FROM last_period - datetime) )||jsonb_build_object( - 'datetime_from', get_datetime_object(datetime_first, t.timezone) + 'datetime_from', get_datetime_object(datetime_first - make_interval(secs=>log_seconds), t.timezone) , 'datetime_to', get_datetime_object(datetime_last, t.timezone) ) as coverage , sensor_flags_exist(t.sensors_id, t.datetime, '-{dur}'::interval) as flag_info @@ -477,6 +479,7 @@ async def fetch_measurements_aggregated(query, aggregate_to, db): JOIN measurands m ON (t.measurands_id = m.measurands_id) {query.pagination()} """ + params = query.params() params["aggregate_to"] = aggregate_to return await db.fetchPage(sql, params) @@ -514,7 +517,7 @@ async def fetch_hours(query, db): , s.data_logging_period_seconds , 1 * 3600 )||jsonb_build_object( - 'datetime_from', get_datetime_object(h.datetime_first, sn.timezone) + 'datetime_from', get_datetime_object(h.datetime_first - '1h'::interval, sn.timezone) , 'datetime_to', get_datetime_object(h.datetime_last, sn.timezone) ) as coverage , sensor_flags_exist(h.sensors_id, h.datetime) as flag_info @@ -554,8 +557,8 @@ async def fetch_hours_aggregated(query, aggregate_to, db): , AVG(s.data_logging_period_seconds) as log_seconds , MAX(truncate_timestamp(datetime, '{aggregate_to}', tz.tzid, '{dur}'::interval)) as last_period - , MIN(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_first - , MAX(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_last + , MIN(timezone(tz.tzid, datetime)) as datetime_first + , MAX(timezone(tz.tzid, datetime)) as datetime_last , COUNT(1) as value_count , AVG(value_avg) as value_avg , STDDEV(value_avg) as value_sd @@ -609,7 +612,7 @@ async def fetch_hours_aggregated(query, aggregate_to, db): , 3600 , EXTRACT(EPOCH FROM last_period - datetime) )||jsonb_build_object( - 'datetime_from', get_datetime_object(datetime_first, t.timezone) + 'datetime_from', get_datetime_object(datetime_first - '1h'::interval, t.timezone) , 'datetime_to', get_datetime_object(datetime_last, t.timezone) ) as coverage , sensor_flags_exist(t.sensors_id, t.datetime, '-{dur}'::interval) as flag_info @@ -906,7 +909,7 @@ async def fetch_hours_trends(aggregate_to, query, db): , e.n * {interval_seconds} )|| jsonb_build_object( - 'datetime_from', get_datetime_object(o.coverage_first, o.timezone) + 'datetime_from', get_datetime_object(o.coverage_first - make_interval(secs=>{interval_seconds}), o.timezone) , 'datetime_to', get_datetime_object(o.coverage_last, o.timezone) ) as coverage , sensor_flags_exist(o.sensors_id, o.coverage_first, '{dur}'::interval) as flag_info @@ -914,7 +917,7 @@ async def fetch_hours_trends(aggregate_to, query, db): JOIN observed o ON (e.factor = o.factor) ORDER BY e.factor """ - + logger.debug(params) return await db.fetchPage(sql, params) @@ -1003,6 +1006,7 @@ async def fetch_days_aggregated(query, aggregate_to, db): {query.total()} FROM meas t JOIN measurands m ON (t.measurands_id = m.measurands_id) + ORDER BY datetime {query.pagination()} """ params = query.params() diff --git a/openaq_api/tests/test_sensor_measurements.py b/openaq_api/tests/test_sensor_measurements.py index 9a231cd..e91b7d0 100644 --- a/openaq_api/tests/test_sensor_measurements.py +++ b/openaq_api/tests/test_sensor_measurements.py @@ -21,8 +21,19 @@ def test_default_good(self, client): assert len(data) > 0, "response did not have at least one record" def test_date_filter_good(self, client): - response = client.get(f"/v3/sensors/{sensors_id}/measurements?datetime_from=2023-03-06") + ## 7 is the only hourly sensor + response = client.get(f"/v3/sensors/7/measurements?datetime_from=2023-03-05&datetime_to=2023-03-06") assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + row = data[0] + assert len(data) == 24 + assert row['coverage']['expectedCount'] == 1 + assert row['coverage']['observedCount'] == 1 + assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00' + assert row['coverage']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00' + assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00' + assert row['period']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00' + def test_aggregated_hourly_good(self, client): response = client.get(f"/v3/sensors/{sensors_id}/measurements/hourly") @@ -30,6 +41,26 @@ def test_aggregated_hourly_good(self, client): data = json.loads(response.content).get('results', []) assert len(data) > 0 + def test_date_filter_aggregated_hourly_good(self, client): + response = client.get(f"/v3/sensors/{sensors_id}/measurements/hourly?datetime_from=2023-03-05&datetime_to=2023-03-06") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 24 + + row = data[0] + period = row['period']['label'] + + assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00' + assert row['coverage']['datetimeTo']['local'] == '2023-03-05T01:00:00-10:00' + assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00' + assert row['period']['datetimeTo']['local'] == '2023-03-05T01:00:00-10:00' + + assert row['coverage']['expectedCount'] == 2 + assert row['coverage']['observedCount'] == 2 + assert row['coverage']['percentComplete'] == 100 + assert row['coverage']['percentComplete'] == row['coverage']['percentCoverage'] + + def test_aggregated_daily_good(self, client): response = client.get(f"/v3/sensors/{sensors_id}/measurements/daily") assert response.status_code == 200 @@ -44,6 +75,20 @@ def test_default_good(self, client): data = json.loads(response.content).get('results', []) assert len(data) > 0 + def test_date_filter_good(self, client): + ## 7 is the only hourly sensor + response = client.get(f"/v3/sensors/7/hours?datetime_from=2023-03-05T00:00:00&datetime_to=2023-03-06T00:00:00") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + row = data[0] + assert len(data) == 24 + assert row['coverage']['expectedCount'] == 1 + assert row['coverage']['observedCount'] == 1 + assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00' + assert row['coverage']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00' + assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00' + assert row['period']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00' + def test_aggregated_daily_good(self, client): response = client.get(f"/v3/sensors/{sensors_id}/hours/daily") assert response.status_code == 200 @@ -72,6 +117,24 @@ def test_aggregated_yearly_good(self, client): data = json.loads(response.content).get('results', []) assert len(data) > 0 + def test_aggregated_daily_good_with_dates(self, client): + response = client.get(f"/v3/sensors/{sensors_id}/hours/daily?datetime_from=2023-03-05&datetime_to=2023-03-06") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 + row = data[0] + + assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00' + assert row['coverage']['datetimeTo']['local'] == '2023-03-06T00:00:00-10:00' + assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00' + assert row['period']['datetimeTo']['local'] == '2023-03-06T00:00:00-10:00' + + assert row['coverage']['expectedCount'] == 24 + assert row['coverage']['observedCount'] == 24 + assert row['coverage']['percentComplete'] == 100 + assert row['coverage']['percentComplete'] == row['coverage']['percentCoverage'] + + def test_aggregated_yearly_good_with_dates(self, client): response = client.get(f"/v3/sensors/{sensors_id}/hours/yearly?datetime_from=2022-01-01&datetime_to=2023-01-01") assert response.status_code == 200 @@ -79,7 +142,7 @@ def test_aggregated_yearly_good_with_dates(self, client): assert len(data) == 1 row = data[0] assert row.get('coverage', {}).get('expectedCount') == (365 * 24) - assert row.get('coverage', {}).get('observedCount') == 365 + assert row.get('coverage', {}).get('observedCount') == 365 * 24 def test_aggregated_hod_good(self, client): response = client.get(f"/v3/sensors/{sensors_id}/hours/hourofday") @@ -100,7 +163,7 @@ def test_aggregated_hod_timestamps_good(self, client): assert len(data) == 24 def test_aggregated_hod_timestamptzs_good(self, client): - response = client.get(f"/v3/sensors/{sensors_id}/hours/hourofday?datetime_from=2023-03-01T00:00:01Z&datetime_to=2023-04-01T00:00:01Z") + response = client.get(f"/v3/sensors/{sensors_id}/hours/hourofday?datetime_from=2023-03-01T00:00:00Z&datetime_to=2023-04-01T00:00:00Z") assert response.status_code == 200 data = json.loads(response.content).get('results', []) assert len(data) == 24 @@ -113,14 +176,14 @@ def test_aggregated_dow_good(self, client): assert len(data) == 7 def test_aggregated_moy_good(self, client): - response = client.get(f"/v3/sensors/{sensors_id}/hours/monthofyear?datetime_from=2022-01-01&datetime_to=2023-01-01") + response = client.get(f"/v3/sensors/{sensors_id}/hours/monthofyear?datetime_from=2022-01-01T00:00:00Z&datetime_to=2023-01-01T00:00:00Z") assert response.status_code == 200 data = json.loads(response.content).get('results', []) assert len(data) == 12 row = data[0] # hours are time ending - assert row['coverage']['datetimeFrom']['local'] == '2022-01-02T00:00:00-10:00' + assert row['coverage']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00' assert row['coverage']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00' assert row['period']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00' assert row['period']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00' @@ -150,6 +213,11 @@ def test_aggregated_monthly_good(self, client): data = json.loads(response.content).get('results', []) assert len(data) == 12 row = data[0] + assert row['coverage']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00' + assert row['coverage']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00' + assert row['period']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00' + assert row['period']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00' + assert row['coverage']['expectedCount'] == 31 assert row['coverage']['observedCount'] == 31 assert row['coverage']['percentComplete'] == 100 @@ -223,6 +291,6 @@ def test_good_with_dates(self, client): assert len(data) == 1 row = data[0] assert row['coverage']['expectedCount'] == 8760 - assert row['coverage']['observedCount'] == 365 - assert row['coverage']['percentComplete'] == 4 + assert row['coverage']['observedCount'] == 365*24 + assert row['coverage']['percentComplete'] == 100 assert row['coverage']['percentComplete'] == row['coverage']['percentCoverage']