849 lines
34 KiB
Python
849 lines
34 KiB
Python
from typing import AsyncGenerator
|
|
|
|
from sqlalchemy import bindparam, text
|
|
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
|
from sqlalchemy.orm import declarative_base
|
|
|
|
from app.core.config import settings
|
|
from app.core.logging import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
DB_POOL_CONFIG = {
|
|
"pool_pre_ping": True,
|
|
"pool_recycle": 1800,
|
|
"pool_size": 10,
|
|
"max_overflow": 20,
|
|
"pool_timeout": 30,
|
|
}
|
|
|
|
engine = create_async_engine(
|
|
settings.DATABASE_URL,
|
|
echo=settings.DEBUG if hasattr(settings, "DEBUG") else False,
|
|
**DB_POOL_CONFIG,
|
|
)
|
|
|
|
async_session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
Base = declarative_base()
|
|
|
|
|
|
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
|
async with async_session_factory() as session:
|
|
try:
|
|
yield session
|
|
await session.commit()
|
|
except Exception:
|
|
await session.rollback()
|
|
raise
|
|
|
|
|
|
async def seed_default_datasources(session: AsyncSession):
|
|
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
|
from app.models.datasource import DataSource
|
|
|
|
for source, info in DEFAULT_DATASOURCES.items():
|
|
existing = await session.get(DataSource, info["id"])
|
|
if existing:
|
|
existing.name = info["name"]
|
|
existing.source = source
|
|
existing.module = info["module"]
|
|
existing.priority = info["priority"]
|
|
existing.frequency_minutes = info["frequency_minutes"]
|
|
existing.collector_class = source
|
|
if existing.config is None:
|
|
existing.config = "{}"
|
|
continue
|
|
|
|
session.add(
|
|
DataSource(
|
|
id=info["id"],
|
|
name=info["name"],
|
|
source=source,
|
|
module=info["module"],
|
|
priority=info["priority"],
|
|
frequency_minutes=info["frequency_minutes"],
|
|
collector_class=source,
|
|
config="{}",
|
|
is_active=True,
|
|
)
|
|
)
|
|
|
|
await session.commit()
|
|
|
|
|
|
LEGACY_EARTH_BOUNDARY_SOURCES = (
|
|
"earth_admin0_boundaries",
|
|
"earth_coastline",
|
|
"earth_claim_lines",
|
|
"earth_boundary_tiles",
|
|
)
|
|
LEGACY_EARTH_BOUNDARY_DATATYPES = (
|
|
"earth_boundary_source",
|
|
"earth_boundary_tiles",
|
|
)
|
|
LEGACY_EARTH_BOUNDARY_IDS = (29, 30, 31, 32)
|
|
|
|
|
|
async def purge_legacy_earth_boundary_datasources(session: AsyncSession) -> None:
|
|
source_names = tuple(LEGACY_EARTH_BOUNDARY_SOURCES)
|
|
source_ids = tuple(LEGACY_EARTH_BOUNDARY_IDS)
|
|
data_types = tuple(LEGACY_EARTH_BOUNDARY_DATATYPES)
|
|
await session.execute(
|
|
text(
|
|
"""
|
|
DELETE FROM datasource_mapping_templates
|
|
WHERE target_schema IN :data_types
|
|
OR datasource_config_id IN (
|
|
SELECT id FROM datasource_configs WHERE name IN :source_names
|
|
)
|
|
"""
|
|
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
|
|
{"source_names": list(source_names), "data_types": list(data_types)},
|
|
)
|
|
await session.execute(
|
|
text("DELETE FROM datasource_configs WHERE name IN :source_names").bindparams(
|
|
bindparam("source_names", expanding=True)
|
|
),
|
|
{"source_names": list(source_names)},
|
|
)
|
|
await session.execute(
|
|
text(
|
|
"""
|
|
DELETE FROM collected_data
|
|
WHERE source IN :source_names OR data_type IN :data_types
|
|
"""
|
|
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
|
|
{"source_names": list(source_names), "data_types": list(data_types)},
|
|
)
|
|
await session.execute(
|
|
text(
|
|
"""
|
|
DELETE FROM data_snapshots
|
|
WHERE source IN :source_names OR datasource_id IN :source_ids
|
|
"""
|
|
).bindparams(bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)),
|
|
{"source_names": list(source_names), "source_ids": list(source_ids)},
|
|
)
|
|
await session.execute(
|
|
text("DELETE FROM collection_tasks WHERE datasource_id IN :source_ids").bindparams(
|
|
bindparam("source_ids", expanding=True)
|
|
),
|
|
{"source_ids": list(source_ids)},
|
|
)
|
|
await session.execute(
|
|
text("DELETE FROM data_sources WHERE source IN :source_names OR id IN :source_ids").bindparams(
|
|
bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)
|
|
),
|
|
{"source_names": list(source_names), "source_ids": list(source_ids)},
|
|
)
|
|
await session.commit()
|
|
|
|
|
|
DEFAULT_LOGIN_USERS = (
|
|
{
|
|
"username": "admin",
|
|
"email": "admin@planet.local",
|
|
"password": "admin123",
|
|
"role": "super_admin",
|
|
},
|
|
{
|
|
"username": "linkong",
|
|
"email": "linkong@planet.local",
|
|
"password": "LK12345678",
|
|
"role": "super_admin",
|
|
},
|
|
)
|
|
|
|
|
|
async def ensure_default_admin_user(session: AsyncSession):
|
|
from app.core.security import get_password_hash
|
|
from app.models.user import User
|
|
|
|
for default_user in DEFAULT_LOGIN_USERS:
|
|
result = await session.execute(
|
|
text("SELECT id FROM users WHERE username = :username"),
|
|
{"username": default_user["username"]},
|
|
)
|
|
if result.fetchone():
|
|
continue
|
|
|
|
session.add(
|
|
User(
|
|
username=default_user["username"],
|
|
email=default_user["email"],
|
|
password_hash=get_password_hash(default_user["password"]),
|
|
role=default_user["role"],
|
|
is_active=True,
|
|
email_verified=True,
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
|
|
async def init_db():
|
|
import app.models.user # noqa: F401
|
|
import app.models.gpu_cluster # noqa: F401
|
|
import app.models.task # noqa: F401
|
|
import app.models.data_snapshot # noqa: F401
|
|
import app.models.datasource # noqa: F401
|
|
import app.models.datasource_config # noqa: F401
|
|
import app.models.alert # noqa: F401
|
|
import app.models.bgp_anomaly # noqa: F401
|
|
import app.models.bgp_collector_location # noqa: F401
|
|
import app.models.bgp_incident # noqa: F401
|
|
import app.models.bgp_observation # noqa: F401
|
|
import app.models.collected_data # noqa: F401
|
|
import app.models.compute_center_location # noqa: F401
|
|
import app.models.system_setting # noqa: F401
|
|
import app.models.playground_session # noqa: F401
|
|
import app.models.playground_message # noqa: F401
|
|
import app.models.system_log # noqa: F401
|
|
import app.models.vessel # noqa: F401
|
|
import app.models.vessel_enrichment # noqa: F401
|
|
import app.models.datasource_mapping # noqa: F401
|
|
import app.models.earth_news # noqa: F401
|
|
import app.models.earth_interactable # noqa: F401
|
|
|
|
logger.warning_event(
|
|
"Database pool settings active",
|
|
event="database.pool.initialized",
|
|
context={
|
|
"pool_pre_ping": DB_POOL_CONFIG["pool_pre_ping"],
|
|
"pool_recycle": DB_POOL_CONFIG["pool_recycle"],
|
|
"pool_size": DB_POOL_CONFIG["pool_size"],
|
|
"max_overflow": DB_POOL_CONFIG["max_overflow"],
|
|
"pool_timeout": DB_POOL_CONFIG["pool_timeout"],
|
|
},
|
|
)
|
|
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
users_email_verified_existed = (
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
SELECT 1
|
|
FROM information_schema.columns
|
|
WHERE table_name = 'users' AND column_name = 'email_verified'
|
|
"""
|
|
)
|
|
)
|
|
).fetchone() is not None
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
ALTER TABLE users
|
|
ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb,
|
|
ADD COLUMN IF NOT EXISTS email_verified BOOLEAN NOT NULL DEFAULT FALSE,
|
|
ADD COLUMN IF NOT EXISTS pending_email VARCHAR(255)
|
|
"""
|
|
)
|
|
)
|
|
if not users_email_verified_existed:
|
|
await conn.execute(
|
|
text("UPDATE users SET email_verified = TRUE WHERE email_verified = FALSE")
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
ALTER TABLE collected_data
|
|
ADD COLUMN IF NOT EXISTS snapshot_id INTEGER,
|
|
ADD COLUMN IF NOT EXISTS task_id INTEGER,
|
|
ADD COLUMN IF NOT EXISTS entity_key VARCHAR(255),
|
|
ADD COLUMN IF NOT EXISTS is_current BOOLEAN DEFAULT TRUE,
|
|
ADD COLUMN IF NOT EXISTS previous_record_id INTEGER,
|
|
ADD COLUMN IF NOT EXISTS change_type VARCHAR(20),
|
|
ADD COLUMN IF NOT EXISTS change_summary JSONB DEFAULT '{}'::jsonb,
|
|
ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS earth_data_change_events (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
table_name VARCHAR(128) NOT NULL,
|
|
operation VARCHAR(16) NOT NULL,
|
|
source VARCHAR(128),
|
|
entity_key VARCHAR(255),
|
|
payload JSONB NOT NULL,
|
|
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
consumed_at TIMESTAMPTZ
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_earth_data_change_events_unconsumed
|
|
ON earth_data_change_events (consumed_at, id)
|
|
WHERE consumed_at IS NULL
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE OR REPLACE FUNCTION planet_emit_earth_data_changed_statement(
|
|
change_table TEXT,
|
|
change_operation TEXT,
|
|
change_source TEXT,
|
|
source_record_count INTEGER,
|
|
source_entity_keys TEXT[]
|
|
)
|
|
RETURNS VOID AS $$
|
|
DECLARE
|
|
change_event_id BIGINT;
|
|
change_payload JSONB;
|
|
BEGIN
|
|
change_payload := jsonb_build_object(
|
|
'event', 'earth.layer.changed',
|
|
'table', change_table,
|
|
'operation', change_operation,
|
|
'source', change_source,
|
|
'entity_key', NULL,
|
|
'entity_keys', COALESCE(to_jsonb(source_entity_keys), '[]'::jsonb),
|
|
'records_processed', COALESCE(source_record_count, 0),
|
|
'occurred_at', NOW()
|
|
);
|
|
|
|
INSERT INTO earth_data_change_events (
|
|
table_name,
|
|
operation,
|
|
source,
|
|
entity_key,
|
|
payload,
|
|
occurred_at
|
|
) VALUES (
|
|
change_table,
|
|
change_operation,
|
|
change_source,
|
|
NULL,
|
|
change_payload,
|
|
NOW()
|
|
)
|
|
RETURNING id INTO change_event_id;
|
|
|
|
change_payload := change_payload || jsonb_build_object(
|
|
'event_id', change_event_id
|
|
);
|
|
|
|
UPDATE earth_data_change_events
|
|
SET payload = change_payload
|
|
WHERE id = change_event_id;
|
|
|
|
PERFORM pg_notify(
|
|
'planet_earth_data_changes',
|
|
change_payload::text
|
|
);
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE OR REPLACE FUNCTION planet_emit_collected_data_changed_statement(
|
|
change_operation TEXT,
|
|
change_source TEXT,
|
|
source_record_count INTEGER,
|
|
source_entity_keys TEXT[]
|
|
)
|
|
RETURNS VOID AS $$
|
|
BEGIN
|
|
PERFORM planet_emit_earth_data_changed_statement(
|
|
'collected_data',
|
|
change_operation,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE OR REPLACE FUNCTION planet_notify_earth_table_changed_statement()
|
|
RETURNS trigger AS $$
|
|
DECLARE
|
|
change_source TEXT;
|
|
source_record_count INTEGER;
|
|
source_entity_keys TEXT[];
|
|
BEGIN
|
|
IF TG_OP = 'INSERT' THEN
|
|
FOR change_source IN
|
|
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
|
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) changed_rows
|
|
LOOP
|
|
SELECT
|
|
COUNT(*),
|
|
ARRAY(
|
|
SELECT DISTINCT COALESCE(
|
|
NULLIF(row_data->>'entity_key', ''),
|
|
NULLIF(row_data->>'source_id', ''),
|
|
NULLIF(row_data->>'incident_key', ''),
|
|
NULLIF(row_data->>'id', ''),
|
|
NULLIF(row_data->>'mmsi', '')
|
|
)
|
|
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_keys
|
|
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
|
LIMIT 20
|
|
)
|
|
INTO source_record_count, source_entity_keys
|
|
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_count
|
|
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
|
|
|
PERFORM planet_emit_earth_data_changed_statement(
|
|
TG_TABLE_NAME,
|
|
TG_OP,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END LOOP;
|
|
ELSIF TG_OP = 'DELETE' THEN
|
|
FOR change_source IN
|
|
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
|
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) changed_rows
|
|
LOOP
|
|
SELECT
|
|
COUNT(*),
|
|
ARRAY(
|
|
SELECT DISTINCT COALESCE(
|
|
NULLIF(row_data->>'entity_key', ''),
|
|
NULLIF(row_data->>'source_id', ''),
|
|
NULLIF(row_data->>'incident_key', ''),
|
|
NULLIF(row_data->>'id', ''),
|
|
NULLIF(row_data->>'mmsi', '')
|
|
)
|
|
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_keys
|
|
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
|
LIMIT 20
|
|
)
|
|
INTO source_record_count, source_entity_keys
|
|
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_count
|
|
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
|
|
|
PERFORM planet_emit_earth_data_changed_statement(
|
|
TG_TABLE_NAME,
|
|
TG_OP,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END LOOP;
|
|
ELSIF TG_OP = 'UPDATE' THEN
|
|
FOR change_source IN
|
|
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
|
FROM (
|
|
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
|
UNION ALL
|
|
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
|
) changed_rows
|
|
LOOP
|
|
SELECT
|
|
COUNT(*),
|
|
ARRAY(
|
|
SELECT DISTINCT COALESCE(
|
|
NULLIF(row_data->>'entity_key', ''),
|
|
NULLIF(row_data->>'source_id', ''),
|
|
NULLIF(row_data->>'incident_key', ''),
|
|
NULLIF(row_data->>'id', ''),
|
|
NULLIF(row_data->>'mmsi', '')
|
|
)
|
|
FROM (
|
|
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
|
UNION ALL
|
|
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
|
) rows_for_keys
|
|
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
|
LIMIT 20
|
|
)
|
|
INTO source_record_count, source_entity_keys
|
|
FROM (
|
|
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
|
UNION ALL
|
|
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
|
) rows_for_count
|
|
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
|
|
|
PERFORM planet_emit_earth_data_changed_statement(
|
|
TG_TABLE_NAME,
|
|
TG_OP,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END LOOP;
|
|
END IF;
|
|
|
|
RETURN NULL;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE OR REPLACE FUNCTION planet_notify_collected_data_changed_statement()
|
|
RETURNS trigger AS $$
|
|
DECLARE
|
|
change_source TEXT;
|
|
source_record_count INTEGER;
|
|
source_entity_keys TEXT[];
|
|
BEGIN
|
|
IF TG_OP = 'INSERT' THEN
|
|
FOR change_source IN
|
|
SELECT DISTINCT source FROM new_rows WHERE source IS NOT NULL
|
|
LOOP
|
|
SELECT
|
|
COUNT(*),
|
|
ARRAY(
|
|
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
|
FROM new_rows
|
|
WHERE source = change_source
|
|
LIMIT 20
|
|
)
|
|
INTO source_record_count, source_entity_keys
|
|
FROM new_rows
|
|
WHERE source = change_source;
|
|
|
|
PERFORM planet_emit_collected_data_changed_statement(
|
|
TG_OP,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END LOOP;
|
|
ELSIF TG_OP = 'DELETE' THEN
|
|
FOR change_source IN
|
|
SELECT DISTINCT source FROM old_rows WHERE source IS NOT NULL
|
|
LOOP
|
|
SELECT
|
|
COUNT(*),
|
|
ARRAY(
|
|
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
|
FROM old_rows
|
|
WHERE source = change_source
|
|
LIMIT 20
|
|
)
|
|
INTO source_record_count, source_entity_keys
|
|
FROM old_rows
|
|
WHERE source = change_source;
|
|
|
|
PERFORM planet_emit_collected_data_changed_statement(
|
|
TG_OP,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END LOOP;
|
|
ELSIF TG_OP = 'UPDATE' THEN
|
|
FOR change_source IN
|
|
SELECT DISTINCT source FROM (
|
|
SELECT source FROM new_rows
|
|
UNION
|
|
SELECT source FROM old_rows
|
|
) changed_sources
|
|
WHERE source IS NOT NULL
|
|
LOOP
|
|
SELECT
|
|
COUNT(*),
|
|
ARRAY(
|
|
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
|
FROM (
|
|
SELECT id, source_id, entity_key, source FROM new_rows
|
|
UNION ALL
|
|
SELECT id, source_id, entity_key, source FROM old_rows
|
|
) changed_rows
|
|
WHERE source = change_source
|
|
LIMIT 20
|
|
)
|
|
INTO source_record_count, source_entity_keys
|
|
FROM (
|
|
SELECT id, source_id, entity_key, source FROM new_rows
|
|
UNION ALL
|
|
SELECT id, source_id, entity_key, source FROM old_rows
|
|
) changed_rows
|
|
WHERE source = change_source;
|
|
|
|
PERFORM planet_emit_collected_data_changed_statement(
|
|
TG_OP,
|
|
change_source,
|
|
source_record_count,
|
|
source_entity_keys
|
|
);
|
|
END LOOP;
|
|
END IF;
|
|
|
|
RETURN NULL;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
"""
|
|
)
|
|
)
|
|
for statement in (
|
|
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed ON collected_data",
|
|
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_insert ON collected_data",
|
|
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_update ON collected_data",
|
|
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_delete ON collected_data",
|
|
"DROP FUNCTION IF EXISTS planet_notify_collected_data_changed()",
|
|
"""
|
|
CREATE TRIGGER tr_planet_collected_data_changed_insert
|
|
AFTER INSERT ON collected_data
|
|
REFERENCING NEW TABLE AS new_rows
|
|
FOR EACH STATEMENT
|
|
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
|
""",
|
|
"""
|
|
CREATE TRIGGER tr_planet_collected_data_changed_update
|
|
AFTER UPDATE ON collected_data
|
|
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
|
FOR EACH STATEMENT
|
|
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
|
""",
|
|
"""
|
|
CREATE TRIGGER tr_planet_collected_data_changed_delete
|
|
AFTER DELETE ON collected_data
|
|
REFERENCING OLD TABLE AS old_rows
|
|
FOR EACH STATEMENT
|
|
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
|
""",
|
|
):
|
|
await conn.execute(text(statement))
|
|
for table_name in (
|
|
"bgp_observations",
|
|
"bgp_anomalies",
|
|
"bgp_incidents",
|
|
"bgp_collector_locations",
|
|
"vessel_static",
|
|
"vessel_position",
|
|
"ais_raw_observations",
|
|
"ais_source_health",
|
|
"compute_center_locations",
|
|
"earth_interactables",
|
|
"earth_news_items",
|
|
):
|
|
for statement in (
|
|
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_insert ON {table_name}",
|
|
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_update ON {table_name}",
|
|
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_delete ON {table_name}",
|
|
f"""
|
|
CREATE TRIGGER tr_planet_{table_name}_changed_insert
|
|
AFTER INSERT ON {table_name}
|
|
REFERENCING NEW TABLE AS new_rows
|
|
FOR EACH STATEMENT
|
|
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
|
""",
|
|
f"""
|
|
CREATE TRIGGER tr_planet_{table_name}_changed_update
|
|
AFTER UPDATE ON {table_name}
|
|
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
|
FOR EACH STATEMENT
|
|
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
|
""",
|
|
f"""
|
|
CREATE TRIGGER tr_planet_{table_name}_changed_delete
|
|
AFTER DELETE ON {table_name}
|
|
REFERENCING OLD TABLE AS old_rows
|
|
FOR EACH STATEMENT
|
|
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
|
""",
|
|
):
|
|
await conn.execute(text(statement))
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
ALTER TABLE collection_tasks
|
|
ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued',
|
|
ADD COLUMN IF NOT EXISTS phase_progress DOUBLE PRECISION,
|
|
ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
|
|
ADD COLUMN IF NOT EXISTS phase_current BIGINT,
|
|
ADD COLUMN IF NOT EXISTS phase_total BIGINT,
|
|
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30),
|
|
ADD COLUMN IF NOT EXISTS source VARCHAR(100),
|
|
ADD COLUMN IF NOT EXISTS task_type VARCHAR(30) NOT NULL DEFAULT 'collect',
|
|
ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
ADD COLUMN IF NOT EXISTS rollback_policy VARCHAR(40) NOT NULL DEFAULT 'keep_committed_batches',
|
|
ADD COLUMN IF NOT EXISTS dedupe_key VARCHAR(180),
|
|
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(120),
|
|
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
|
|
ADD COLUMN IF NOT EXISTS requested_cancel_at TIMESTAMPTZ,
|
|
ADD COLUMN IF NOT EXISTS cancel_reason TEXT
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
ALTER TABLE earth_news_items
|
|
ADD COLUMN IF NOT EXISTS content_language VARCHAR(32) NOT NULL DEFAULT 'en',
|
|
ADD COLUMN IF NOT EXISTS localizations JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
ADD COLUMN IF NOT EXISTS enrichment_status VARCHAR(80) NOT NULL DEFAULT 'pending',
|
|
ADD COLUMN IF NOT EXISTS enrichment_error TEXT,
|
|
ADD COLUMN IF NOT EXISTS enriched_at TIMESTAMPTZ
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
ALTER TABLE earth_interactables
|
|
ADD COLUMN IF NOT EXISTS altitude DOUBLE PRECISION,
|
|
ADD COLUMN IF NOT EXISTS revision INTEGER NOT NULL DEFAULT 1,
|
|
ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
|
ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_collected_data_source_source_id
|
|
ON collected_data (source, source_id)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_earth_news_enrichment_status
|
|
ON earth_news_items (enrichment_status)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_earth_news_enriched_at
|
|
ON earth_news_items (enriched_at)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_earth_interactables_layer_deleted
|
|
ON earth_interactables (layer, is_deleted)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_earth_interactables_updated_at
|
|
ON earth_interactables (updated_at)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_collection_tasks_source_status
|
|
ON collection_tasks (source, status)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_collection_tasks_queue
|
|
ON collection_tasks (status, created_at, id)
|
|
WHERE status = 'queued'
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_collection_tasks_dedupe
|
|
ON collection_tasks (dedupe_key)
|
|
WHERE dedupe_key IS NOT NULL
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_collected_data_source_current_id
|
|
ON collected_data (source, is_current, id)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_collected_data_source_task_id
|
|
ON collected_data (source, task_id, id)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_entity
|
|
ON ais_raw_observations (target_schema, observed_at, entity_key)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_desc
|
|
ON ais_raw_observations (target_schema, observed_at DESC)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_ais_raw_payload_lon_lat
|
|
ON ais_raw_observations (
|
|
((normalized_payload->>'lon')::double precision),
|
|
((normalized_payload->>'lat')::double precision)
|
|
)
|
|
WHERE target_schema = 'vessel_ais'
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
UPDATE collected_data
|
|
SET entity_key = source || ':' || COALESCE(source_id, id::text)
|
|
WHERE entity_key IS NULL
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
UPDATE collected_data
|
|
SET is_current = TRUE
|
|
WHERE is_current IS NULL
|
|
"""
|
|
)
|
|
)
|
|
|
|
async with async_session_factory() as session:
|
|
from app.services.bgp_collector_locations import (
|
|
seed_default_bgp_collector_locations,
|
|
)
|
|
from app.services.compute_center_locations import (
|
|
seed_compute_center_locations_from_source_coords,
|
|
)
|
|
|
|
await seed_default_bgp_collector_locations(session)
|
|
await seed_compute_center_locations_from_source_coords(session)
|
|
await seed_default_datasources(session)
|
|
await purge_legacy_earth_boundary_datasources(session)
|
|
await ensure_default_admin_user(session)
|