diff --git a/openaqdb/idempotent/exports_views.sql b/openaqdb/idempotent/exports_views.sql index 239747c..3adbb1e 100644 --- a/openaqdb/idempotent/exports_views.sql +++ b/openaqdb/idempotent/exports_views.sql @@ -43,8 +43,8 @@ JOIN measurands p ON (s.measurands_id = p.measurands_id) JOIN sensor_systems ss ON (s.sensor_systems_id = ss.sensor_systems_id) JOIN sensor_nodes sn ON (ss.sensor_nodes_id = sn.sensor_nodes_id) JOIN providers pr ON (sn.source_name = pr.source_name) -JOIN timezones t ON (sn.timezones_id = t.gid) -WHERE t.gid IS NOT NULL +JOIN timezones t ON (sn.timezones_id = t.timezones_id) +WHERE t.timezones_id IS NOT NULL -- once we have versioning we can uncomment this line --AND s.sensors_id NOT IN (SELECT sensors_id FROM versions) ; @@ -121,7 +121,7 @@ SELECT l.sensor_nodes_id , utc_offset(tz.tzid) as utc_offset FROM public.open_data_export_logs l JOIN public.sensor_nodes sn ON (l.sensor_nodes_id = sn.sensor_nodes_id) -JOIN public.timezones tz ON (sn.timezones_id = tz.gid) +JOIN public.timezones tz ON (sn.timezones_id = tz.timezones_id) WHERE -- older than 72 hours to give us time to collect data day < (now() AT TIME ZONE tz.tzid - '72hours'::interval)::date @@ -142,7 +142,7 @@ SELECT , COUNT(1) as n FROM public.open_data_export_logs l JOIN public.sensor_nodes sn ON (l.sensor_nodes_id = sn.sensor_nodes_id) -JOIN public.timezones tz ON (sn.timezones_id = tz.gid) +JOIN public.timezones tz ON (sn.timezones_id = tz.timezones_id) GROUP BY 1,2,3,4 ORDER BY 1,2,3,4; @@ -312,8 +312,8 @@ COMMIT; -- JOIN sensor_systems ss ON (s.sensor_systems_id = ss.sensor_systems_id) -- JOIN sensor_nodes sn ON (ss.sensor_nodes_id = sn.sensor_nodes_id) -- JOIN providers pr ON (sn.source_name = pr.source_name) --- JOIN timezones t ON (sn.timezones_id = t.gid) --- WHERE t.gid IS NOT NULL; +-- JOIN timezones t ON (sn.timezones_id = t.timezones_id) +-- WHERE t.timezones_id IS NOT NULL; -- \timing on diff --git a/openaqdb/idempotent/util_functions.sql b/openaqdb/idempotent/util_functions.sql index 0d26dea..4b2394e 100644 --- a/openaqdb/idempotent/util_functions.sql +++ b/openaqdb/idempotent/util_functions.sql @@ -108,14 +108,14 @@ $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION utc_offset(dt timestamptz, sn int) RETURNS interval AS $$ SELECT utc_offset(dt, t.tzid) FROM sensor_nodes n - JOIN timezones t ON (t.gid = n.timezones_id) + JOIN timezones t ON (t.timezones_id = n.timezones_id) WHERE sensor_nodes_id = sn; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION utc_offset(sn int) RETURNS interval AS $$ SELECT utc_offset(t.tzid) FROM sensor_nodes n - JOIN timezones t ON (t.gid = n.timezones_id) + JOIN timezones t ON (t.timezones_id = n.timezones_id) WHERE sensor_nodes_id = sn; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; diff --git a/openaqdb/idempotent/views.sql b/openaqdb/idempotent/views.sql index 69c3f7b..a5fbb56 100644 --- a/openaqdb/idempotent/views.sql +++ b/openaqdb/idempotent/views.sql @@ -643,7 +643,7 @@ SELECT sn.sensor_nodes_id , s.added_on as sensor_added_on , sn.added_on as node_added_on FROM sensor_nodes sn -LEFT JOIN timezones tz ON (sn.timezones_id = tz.gid) +LEFT JOIN timezones tz ON (sn.timezones_id = tz.timezones_id) LEFT JOIN providers p ON (sn.providers_id = p.providers_id) LEFT JOIN sensor_systems sy USING (sensor_nodes_id) LEFT JOIN sensors s USING (sensor_systems_id) diff --git a/openaqdb/import_sensors.sql b/openaqdb/import_sensors.sql index 39612d2..2c4220a 100644 --- a/openaqdb/import_sensors.sql +++ b/openaqdb/import_sensors.sql @@ -41,7 +41,7 @@ GROUP BY 1; CREATE TABLE IF NOT EXISTS sensor_nodes_migrate ( sensor_nodes_id int primary key, - timezones_id int REFERENCES timezones(gid), + timezones_id int REFERENCES timezones(timezones_id), ismobile boolean, geom geometry, site_name text, @@ -217,7 +217,7 @@ SELECT n.sensor_nodes_id , s.source_id , n.site_name , st_setsrid(st_makepoint(lon,lat), 4326) -, t.gid +, t.timezones_id , n.origin , CASE WHEN lon IS NULL THEN true ELSE false END , json_build_object('imported', 'production') diff --git a/openaqdb/init.sh b/openaqdb/init.sh index 8cc6282..cdf42fc 100755 --- a/openaqdb/init.sh +++ b/openaqdb/init.sh @@ -43,7 +43,7 @@ psql --single-transaction -v ON_ERROR_STOP=1 -f lookups/measurands_map.sql psql --single-transaction -v ON_ERROR_STOP=1 -f lookups/thresholds.sql psql --single-transaction -v ON_ERROR_STOP=1 -f lookups/thresholds.sql gunzip -c lookups/countries.tsv.gz | psql --single-transaction -v ON_ERROR_STOP=1 -c "copy countries from stdin" -gunzip -c lookups/timezones.tsv.gz | psql --single-transaction -v ON_ERROR_STOP=1 -c "copy timezones from stdin" +gunzip -c lookups/timezones.csv.gz | psql --single-transaction -v ON_ERROR_STOP=1 -c "copy timezones (tzid, geog) from stdin DELIMITER ',' CSV HEADER" gunzip -c lookups/providers_data.csv.gz | psql --single-transaction -v ON_ERROR_STOP=1 -c "COPY providers (providers_id,label,description,source_name,export_prefix,license,metadata,owner_entities_id) FROM stdin DELIMITER ',' CSV HEADER" gunzip -c lookups/sources_from_openaq.tsv.gz | psql --single-transaction -v ON_ERROR_STOP=1 -c "copy sources_from_openaq from stdin" diff --git a/openaqdb/locations/locations.sql b/openaqdb/locations/locations.sql index ee1b59a..5564bde 100644 --- a/openaqdb/locations/locations.sql +++ b/openaqdb/locations/locations.sql @@ -109,7 +109,7 @@ SELECT , pl.licenses , l.providers_id FROM sensor_nodes l -JOIN timezones t ON (l.timezones_id = t.gid) +JOIN timezones t ON (l.timezones_id = t.timezones_id) JOIN countries c ON (c.countries_id = l.countries_id) JOIN entities oc ON (oc.entities_id = l.owner_entities_id) JOIN providers p ON (p.providers_id = l.providers_id) diff --git a/openaqdb/lookups/timezones.tsv.gz b/openaqdb/lookups/timezones.csv.gz similarity index 56% rename from openaqdb/lookups/timezones.tsv.gz rename to openaqdb/lookups/timezones.csv.gz index 56d1509..8febc03 100644 Binary files a/openaqdb/lookups/timezones.tsv.gz and b/openaqdb/lookups/timezones.csv.gz differ diff --git a/openaqdb/metadata_transfer.sql b/openaqdb/metadata_transfer.sql index fc35a46..430e9a7 100644 --- a/openaqdb/metadata_transfer.sql +++ b/openaqdb/metadata_transfer.sql @@ -107,7 +107,7 @@ LIMIT 10; -- move the timezone data from the metadata to the timezones_id field UPDATE sensor_nodes -SET timezones_id = t.gid +SET timezones_id = t.timezones_id FROM timezones t WHERE sensor_nodes.metadata->>'timezone' IS NOT NULL AND sensor_nodes.metadata->>'timezone' = t.tzid diff --git a/openaqdb/tables/daily_data_rollups.sql b/openaqdb/tables/daily_data_rollups.sql index 89f44f3..36b4d75 100644 --- a/openaqdb/tables/daily_data_rollups.sql +++ b/openaqdb/tables/daily_data_rollups.sql @@ -103,7 +103,7 @@ FROM hourly_data m JOIN sensors s ON (m.sensors_id = s.sensors_id) JOIN sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) JOIN sensor_nodes sn ON (sy.sensor_nodes_id = sn.sensor_nodes_id) -JOIN timezones t ON (sn.timezones_id = t.gid) +JOIN timezones t ON (sn.timezones_id = t.timezones_id) WHERE value_count > 0 AND datetime > as_utc(sd, t.tzid) AND datetime <= as_utc(ed, t.tzid) @@ -153,7 +153,7 @@ FROM hourly_data m JOIN sensors s ON (m.sensors_id = s.sensors_id) JOIN sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) JOIN sensor_nodes sn ON (sy.sensor_nodes_id = sn.sensor_nodes_id) -JOIN timezones t ON (sn.timezones_id = t.gid) +JOIN timezones t ON (sn.timezones_id = t.timezones_id) WHERE value_count > 0 AND datetime > as_utc(dy, t.tzid) AND datetime <= as_utc(dy + 1, t.tzid) @@ -271,7 +271,7 @@ FROM hourly_data m JOIN sensors s ON (m.sensors_id = s.sensors_id) JOIN sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) JOIN sensor_nodes sn ON (sy.sensor_nodes_id = sn.sensor_nodes_id) -JOIN timezones t ON (sn.timezones_id = t.gid) +JOIN timezones t ON (sn.timezones_id = t.timezones_id) WHERE value_count > 0 AND datetime > as_utc(sd, t.tzid) AND datetime <= as_utc(ed, t.tzid) diff --git a/openaqdb/tables/daily_data_rollups_check.sql b/openaqdb/tables/daily_data_rollups_check.sql index 37a41a7..955da79 100644 --- a/openaqdb/tables/daily_data_rollups_check.sql +++ b/openaqdb/tables/daily_data_rollups_check.sql @@ -51,7 +51,7 @@ SELECT m.sensors_id JOIN sensors s ON (m.sensors_id = s.sensors_id) JOIN sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) JOIN sensor_nodes sn ON (sy.sensor_nodes_id = sn.sensor_nodes_id) - JOIN timezones t ON (sn.timezones_id = t.gid) + JOIN timezones t ON (sn.timezones_id = t.timezones_id) WHERE m.sensors_id = :sensor AND datetime > :day::timestamptz - :hr::interval AND datetime <= (:day::date + 1)::timestamptz - :hr::interval diff --git a/openaqdb/tables/metadata.sql b/openaqdb/tables/metadata.sql index 00074bb..fa10cc6 100644 --- a/openaqdb/tables/metadata.sql +++ b/openaqdb/tables/metadata.sql @@ -33,7 +33,7 @@ BEGIN ALTER TABLE sensor_nodes ADD COLUMN added_on timestamptz DEFAULT now(), ADD COLUMN modified_on timestamptz, - ADD COLUMN timezones_id int REFERENCES timezones(gid); + ADD COLUMN timezones_id int REFERENCES timezones(timezones_id); EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'sensors alter error'; --END; diff --git a/openaqdb/tables/timezones.sql b/openaqdb/tables/timezones.sql index bc8b209..2c13172 100644 --- a/openaqdb/tables/timezones.sql +++ b/openaqdb/tables/timezones.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS timezones ( - gid integer primary key, + timezones_id int generated always as identity primary key, tzid text, geog geography ); diff --git a/openaqdb/tables/timezones_views.sql b/openaqdb/tables/timezones_views.sql index ce6f2cc..da4fa65 100644 --- a/openaqdb/tables/timezones_views.sql +++ b/openaqdb/tables/timezones_views.sql @@ -3,7 +3,7 @@ RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT tzid FROM timezones WHERE st_intersects(g, geog) - ORDER BY gid ASC + ORDER BY timezones_id ASC LIMIT 1; $$; @@ -12,22 +12,22 @@ RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT tzid FROM timezones WHERE st_intersects(g::geography, geog) - ORDER BY gid ASC + ORDER BY timezones_id ASC LIMIT 1; $$; CREATE OR REPLACE FUNCTION get_timezones_id(g geometry) RETURNS int LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT gid + SELECT timezones_id FROM timezones WHERE st_intersects(g::geography, geog) - ORDER BY gid ASC + ORDER BY timezones_id ASC LIMIT 1; $$; CREATE OR REPLACE FUNCTION get_timezones_id(tz text) RETURNS int LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT gid + SELECT timezones_id FROM timezones WHERE lower(tzid) = lower(tz) LIMIT 1; @@ -44,9 +44,9 @@ WHERE added_on > current_date LIMIT 2) SELECT n.sensor_nodes_id , n.geom - , gid + , n.timezones_id , tzid - , timezones_id + , timezones.timezones_id , st_area(geog::geometry) , st_xmin(geog::geometry) , st_xmax(geog::geometry) @@ -56,7 +56,7 @@ LIMIT 2) WHERE st_intersects(geom::geography, geog); - SELECT gid + SELECT timezones_id , tzid , substring(st_astext(geog) from 0 for 100) , substring(st_astext(geog::geometry) from 0 for 100) diff --git a/scripts/tz_data.sh b/scripts/tz_data.sh new file mode 100644 index 0000000..83258d9 --- /dev/null +++ b/scripts/tz_data.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +wget https://github.com/evansiroky/timezone-boundary-builder/releases/download/2024a/timezones-with-oceans.geojson.zip +unzip timezones-with-oceans.geojson.zip +ogr2ogr -f CSV timezones.csv combined-with-oceans.json -dialect SQLite -sql "SELECT tzid, 'SRID=4326;' || ST_AsText(geometry) AS geog FROM combined-with-oceans" +gzip -c timezones.csv > timezones.csv.gz + +