release: bump version to 0.66.0
This commit is contained in:
rayd1o
@@ -203,6 +203,7 @@ async def init_db():
|
||||
import app.models.vessel_enrichment # noqa: F401
|
||||
import app.models.datasource_mapping # noqa: F401
|
||||
import app.models.earth_news # noqa: F401
|
||||
import app.models.earth_interactable # noqa: F401
|
||||
|
||||
logger.warning_event(
|
||||
"Database pool settings active",
|
||||
@@ -258,6 +259,406 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS earth_data_change_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
table_name VARCHAR(128) NOT NULL,
|
||||
operation VARCHAR(16) NOT NULL,
|
||||
source VARCHAR(128),
|
||||
entity_key VARCHAR(255),
|
||||
payload JSONB NOT NULL,
|
||||
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
consumed_at TIMESTAMPTZ
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_data_change_events_unconsumed
|
||||
ON earth_data_change_events (consumed_at, id)
|
||||
WHERE consumed_at IS NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_emit_earth_data_changed_statement(
|
||||
change_table TEXT,
|
||||
change_operation TEXT,
|
||||
change_source TEXT,
|
||||
source_record_count INTEGER,
|
||||
source_entity_keys TEXT[]
|
||||
)
|
||||
RETURNS VOID AS $$
|
||||
DECLARE
|
||||
change_event_id BIGINT;
|
||||
change_payload JSONB;
|
||||
BEGIN
|
||||
change_payload := jsonb_build_object(
|
||||
'event', 'earth.layer.changed',
|
||||
'table', change_table,
|
||||
'operation', change_operation,
|
||||
'source', change_source,
|
||||
'entity_key', NULL,
|
||||
'entity_keys', COALESCE(to_jsonb(source_entity_keys), '[]'::jsonb),
|
||||
'records_processed', COALESCE(source_record_count, 0),
|
||||
'occurred_at', NOW()
|
||||
);
|
||||
|
||||
INSERT INTO earth_data_change_events (
|
||||
table_name,
|
||||
operation,
|
||||
source,
|
||||
entity_key,
|
||||
payload,
|
||||
occurred_at
|
||||
) VALUES (
|
||||
change_table,
|
||||
change_operation,
|
||||
change_source,
|
||||
NULL,
|
||||
change_payload,
|
||||
NOW()
|
||||
)
|
||||
RETURNING id INTO change_event_id;
|
||||
|
||||
change_payload := change_payload || jsonb_build_object(
|
||||
'event_id', change_event_id
|
||||
);
|
||||
|
||||
UPDATE earth_data_change_events
|
||||
SET payload = change_payload
|
||||
WHERE id = change_event_id;
|
||||
|
||||
PERFORM pg_notify(
|
||||
'planet_earth_data_changes',
|
||||
change_payload::text
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_emit_collected_data_changed_statement(
|
||||
change_operation TEXT,
|
||||
change_source TEXT,
|
||||
source_record_count INTEGER,
|
||||
source_entity_keys TEXT[]
|
||||
)
|
||||
RETURNS VOID AS $$
|
||||
BEGIN
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
'collected_data',
|
||||
change_operation,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
RETURNS trigger AS $$
|
||||
DECLARE
|
||||
change_source TEXT;
|
||||
source_record_count INTEGER;
|
||||
source_entity_keys TEXT[];
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
RETURNS trigger AS $$
|
||||
DECLARE
|
||||
change_source TEXT;
|
||||
source_record_count INTEGER;
|
||||
source_entity_keys TEXT[];
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM new_rows WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM new_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM new_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM old_rows WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM old_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM old_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM (
|
||||
SELECT source FROM new_rows
|
||||
UNION
|
||||
SELECT source FROM old_rows
|
||||
) changed_sources
|
||||
WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM (
|
||||
SELECT id, source_id, entity_key, source FROM new_rows
|
||||
UNION ALL
|
||||
SELECT id, source_id, entity_key, source FROM old_rows
|
||||
) changed_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (
|
||||
SELECT id, source_id, entity_key, source FROM new_rows
|
||||
UNION ALL
|
||||
SELECT id, source_id, entity_key, source FROM old_rows
|
||||
) changed_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
for statement in (
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_insert ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_update ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_delete ON collected_data",
|
||||
"DROP FUNCTION IF EXISTS planet_notify_collected_data_changed()",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_insert
|
||||
AFTER INSERT ON collected_data
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_update
|
||||
AFTER UPDATE ON collected_data
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_delete
|
||||
AFTER DELETE ON collected_data
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
):
|
||||
await conn.execute(text(statement))
|
||||
for table_name in (
|
||||
"bgp_observations",
|
||||
"bgp_anomalies",
|
||||
"bgp_incidents",
|
||||
"bgp_collector_locations",
|
||||
"vessel_static",
|
||||
"vessel_position",
|
||||
"ais_raw_observations",
|
||||
"ais_source_health",
|
||||
"compute_center_locations",
|
||||
"earth_interactables",
|
||||
"earth_news_items",
|
||||
):
|
||||
for statement in (
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_insert ON {table_name}",
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_update ON {table_name}",
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_delete ON {table_name}",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_insert
|
||||
AFTER INSERT ON {table_name}
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_update
|
||||
AFTER UPDATE ON {table_name}
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_delete
|
||||
AFTER DELETE ON {table_name}
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
):
|
||||
await conn.execute(text(statement))
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -267,7 +668,16 @@ async def init_db():
|
||||
ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
|
||||
ADD COLUMN IF NOT EXISTS phase_current BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS phase_total BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30)
|
||||
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30),
|
||||
ADD COLUMN IF NOT EXISTS source VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS task_type VARCHAR(30) NOT NULL DEFAULT 'collect',
|
||||
ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS rollback_policy VARCHAR(40) NOT NULL DEFAULT 'keep_committed_batches',
|
||||
ADD COLUMN IF NOT EXISTS dedupe_key VARCHAR(180),
|
||||
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(120),
|
||||
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS requested_cancel_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS cancel_reason TEXT
|
||||
"""
|
||||
)
|
||||
)
|
||||
@@ -283,6 +693,17 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
ALTER TABLE earth_interactables
|
||||
ADD COLUMN IF NOT EXISTS altitude DOUBLE PRECISION,
|
||||
ADD COLUMN IF NOT EXISTS revision INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -307,6 +728,48 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_interactables_layer_deleted
|
||||
ON earth_interactables (layer, is_deleted)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_interactables_updated_at
|
||||
ON earth_interactables (updated_at)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_source_status
|
||||
ON collection_tasks (source, status)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_queue
|
||||
ON collection_tasks (status, created_at, id)
|
||||
WHERE status = 'queued'
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_dedupe
|
||||
ON collection_tasks (dedupe_key)
|
||||
WHERE dedupe_key IS NOT NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user