Skip to content

Commit

Permalink
Fixed some timestamp issues in the hours and hourly endpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
@caparker
caparker committed Dec 17, 2024
1 parent fecd568 commit 019c711
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 18 deletions.
26 changes: 15 additions & 11 deletions openaq_api/openaq_api/v3/routers/measurements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,8 @@ async def fetch_measurements_aggregated(query, aggregate_to, db):
else:
raise Exception(f"{aggregate_to} is not supported")

query.set_column_map({"timezone": "tz.tzid"})

sql = f"""
WITH meas AS (
SELECT
Expand All @@ -413,8 +415,8 @@ async def fetch_measurements_aggregated(query, aggregate_to, db):
, AVG(s.data_logging_period_seconds) as log_seconds
, MAX(truncate_timestamp(datetime, '{aggregate_to}', tz.tzid, '1{aggregate_to}'::interval))
as last_period
, MIN(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_first
, MAX(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_last
, MIN(timezone(tz.tzid, datetime)) as datetime_first
, MAX(timezone(tz.tzid, datetime)) as datetime_last
, COUNT(1) as value_count
, AVG(value) as value_avg
, STDDEV(value) as value_sd
Expand Down Expand Up @@ -464,11 +466,11 @@ async def fetch_measurements_aggregated(query, aggregate_to, db):
--------
, calculate_coverage(
value_count::int
, {interval_seconds}
, {interval_seconds}
, avg_seconds
, log_seconds
, EXTRACT(EPOCH FROM last_period - datetime)
)||jsonb_build_object(
'datetime_from', get_datetime_object(datetime_first, t.timezone)
'datetime_from', get_datetime_object(datetime_first - make_interval(secs=>log_seconds), t.timezone)
, 'datetime_to', get_datetime_object(datetime_last, t.timezone)
) as coverage
, sensor_flags_exist(t.sensors_id, t.datetime, '-{dur}'::interval) as flag_info
Expand All @@ -477,6 +479,7 @@ async def fetch_measurements_aggregated(query, aggregate_to, db):
JOIN measurands m ON (t.measurands_id = m.measurands_id)
{query.pagination()}
"""

params = query.params()
params["aggregate_to"] = aggregate_to
return await db.fetchPage(sql, params)
Expand Down Expand Up @@ -514,7 +517,7 @@ async def fetch_hours(query, db):
, s.data_logging_period_seconds
, 1 * 3600
)||jsonb_build_object(
'datetime_from', get_datetime_object(h.datetime_first, sn.timezone)
'datetime_from', get_datetime_object(h.datetime_first - '1h'::interval, sn.timezone)
, 'datetime_to', get_datetime_object(h.datetime_last, sn.timezone)
) as coverage
, sensor_flags_exist(h.sensors_id, h.datetime) as flag_info
Expand Down Expand Up @@ -554,8 +557,8 @@ async def fetch_hours_aggregated(query, aggregate_to, db):
, AVG(s.data_logging_period_seconds) as log_seconds
, MAX(truncate_timestamp(datetime, '{aggregate_to}', tz.tzid, '{dur}'::interval))
as last_period
, MIN(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_first
, MAX(timezone(tz.tzid, datetime - '1sec'::interval)) as datetime_last
, MIN(timezone(tz.tzid, datetime)) as datetime_first
, MAX(timezone(tz.tzid, datetime)) as datetime_last
, COUNT(1) as value_count
, AVG(value_avg) as value_avg
, STDDEV(value_avg) as value_sd
Expand Down Expand Up @@ -609,7 +612,7 @@ async def fetch_hours_aggregated(query, aggregate_to, db):
, 3600
, EXTRACT(EPOCH FROM last_period - datetime)
)||jsonb_build_object(
'datetime_from', get_datetime_object(datetime_first, t.timezone)
'datetime_from', get_datetime_object(datetime_first - '1h'::interval, t.timezone)
, 'datetime_to', get_datetime_object(datetime_last, t.timezone)
) as coverage
, sensor_flags_exist(t.sensors_id, t.datetime, '-{dur}'::interval) as flag_info
Expand Down Expand Up @@ -906,15 +909,15 @@ async def fetch_hours_trends(aggregate_to, query, db):
, e.n * {interval_seconds}
)||
jsonb_build_object(
'datetime_from', get_datetime_object(o.coverage_first, o.timezone)
'datetime_from', get_datetime_object(o.coverage_first - make_interval(secs=>{interval_seconds}), o.timezone)
, 'datetime_to', get_datetime_object(o.coverage_last, o.timezone)
) as coverage
, sensor_flags_exist(o.sensors_id, o.coverage_first, '{dur}'::interval) as flag_info
FROM expected e
JOIN observed o ON (e.factor = o.factor)
ORDER BY e.factor
"""

logger.debug(params)

return await db.fetchPage(sql, params)

Expand Down Expand Up @@ -1003,6 +1006,7 @@ async def fetch_days_aggregated(query, aggregate_to, db):
{query.total()}
FROM meas t
JOIN measurands m ON (t.measurands_id = m.measurands_id)
ORDER BY datetime
{query.pagination()}
"""
params = query.params()
Expand Down
82 changes: 75 additions & 7 deletions openaq_api/tests/test_sensor_measurements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,46 @@ def test_default_good(self, client):
assert len(data) > 0, "response did not have at least one record"

def test_date_filter_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/measurements?datetime_from=2023-03-06")
## 7 is the only hourly sensor
response = client.get(f"/v3/sensors/7/measurements?datetime_from=2023-03-05&datetime_to=2023-03-06")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
row = data[0]
assert len(data) == 24
assert row['coverage']['expectedCount'] == 1
assert row['coverage']['observedCount'] == 1
assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00'
assert row['coverage']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00'
assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00'
assert row['period']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00'


def test_aggregated_hourly_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/measurements/hourly")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
assert len(data) > 0

def test_date_filter_aggregated_hourly_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/measurements/hourly?datetime_from=2023-03-05&datetime_to=2023-03-06")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
assert len(data) == 24

row = data[0]
period = row['period']['label']

assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00'
assert row['coverage']['datetimeTo']['local'] == '2023-03-05T01:00:00-10:00'
assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00'
assert row['period']['datetimeTo']['local'] == '2023-03-05T01:00:00-10:00'

assert row['coverage']['expectedCount'] == 2
assert row['coverage']['observedCount'] == 2
assert row['coverage']['percentComplete'] == 100
assert row['coverage']['percentComplete'] == row['coverage']['percentCoverage']


def test_aggregated_daily_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/measurements/daily")
assert response.status_code == 200
Expand All @@ -44,6 +75,20 @@ def test_default_good(self, client):
data = json.loads(response.content).get('results', [])
assert len(data) > 0

def test_date_filter_good(self, client):
## 7 is the only hourly sensor
response = client.get(f"/v3/sensors/7/hours?datetime_from=2023-03-05T00:00:00&datetime_to=2023-03-06T00:00:00")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
row = data[0]
assert len(data) == 24
assert row['coverage']['expectedCount'] == 1
assert row['coverage']['observedCount'] == 1
assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00'
assert row['coverage']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00'
assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-08:00'
assert row['period']['datetimeTo']['local'] == '2023-03-05T01:00:00-08:00'

def test_aggregated_daily_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/hours/daily")
assert response.status_code == 200
Expand Down Expand Up @@ -72,14 +117,32 @@ def test_aggregated_yearly_good(self, client):
data = json.loads(response.content).get('results', [])
assert len(data) > 0

def test_aggregated_daily_good_with_dates(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/hours/daily?datetime_from=2023-03-05&datetime_to=2023-03-06")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
assert len(data) == 1
row = data[0]

assert row['coverage']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00'
assert row['coverage']['datetimeTo']['local'] == '2023-03-06T00:00:00-10:00'
assert row['period']['datetimeFrom']['local'] == '2023-03-05T00:00:00-10:00'
assert row['period']['datetimeTo']['local'] == '2023-03-06T00:00:00-10:00'

assert row['coverage']['expectedCount'] == 24
assert row['coverage']['observedCount'] == 24
assert row['coverage']['percentComplete'] == 100
assert row['coverage']['percentComplete'] == row['coverage']['percentCoverage']


def test_aggregated_yearly_good_with_dates(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/hours/yearly?datetime_from=2022-01-01&datetime_to=2023-01-01")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
assert len(data) == 1
row = data[0]
assert row.get('coverage', {}).get('expectedCount') == (365 * 24)
assert row.get('coverage', {}).get('observedCount') == 365
assert row.get('coverage', {}).get('observedCount') == 365 * 24

def test_aggregated_hod_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/hours/hourofday")
Expand All @@ -100,7 +163,7 @@ def test_aggregated_hod_timestamps_good(self, client):
assert len(data) == 24

def test_aggregated_hod_timestamptzs_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/hours/hourofday?datetime_from=2023-03-01T00:00:01Z&datetime_to=2023-04-01T00:00:01Z")
response = client.get(f"/v3/sensors/{sensors_id}/hours/hourofday?datetime_from=2023-03-01T00:00:00Z&datetime_to=2023-04-01T00:00:00Z")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
assert len(data) == 24
Expand All @@ -113,14 +176,14 @@ def test_aggregated_dow_good(self, client):
assert len(data) == 7

def test_aggregated_moy_good(self, client):
response = client.get(f"/v3/sensors/{sensors_id}/hours/monthofyear?datetime_from=2022-01-01&datetime_to=2023-01-01")
response = client.get(f"/v3/sensors/{sensors_id}/hours/monthofyear?datetime_from=2022-01-01T00:00:00Z&datetime_to=2023-01-01T00:00:00Z")
assert response.status_code == 200
data = json.loads(response.content).get('results', [])
assert len(data) == 12

row = data[0]
# hours are time ending
assert row['coverage']['datetimeFrom']['local'] == '2022-01-02T00:00:00-10:00'
assert row['coverage']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00'
assert row['coverage']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00'
assert row['period']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00'
assert row['period']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00'
Expand Down Expand Up @@ -150,6 +213,11 @@ def test_aggregated_monthly_good(self, client):
data = json.loads(response.content).get('results', [])
assert len(data) == 12
row = data[0]
assert row['coverage']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00'
assert row['coverage']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00'
assert row['period']['datetimeFrom']['local'] == '2022-01-01T00:00:00-10:00'
assert row['period']['datetimeTo']['local'] == '2022-02-01T00:00:00-10:00'

assert row['coverage']['expectedCount'] == 31
assert row['coverage']['observedCount'] == 31
assert row['coverage']['percentComplete'] == 100
Expand Down Expand Up @@ -223,6 +291,6 @@ def test_good_with_dates(self, client):
assert len(data) == 1
row = data[0]
assert row['coverage']['expectedCount'] == 8760
assert row['coverage']['observedCount'] == 365
assert row['coverage']['percentComplete'] == 4
assert row['coverage']['observedCount'] == 365*24
assert row['coverage']['percentComplete'] == 100
assert row['coverage']['percentComplete'] == row['coverage']['percentCoverage']

0 comments on commit 019c711

Please sign in to comment.