Implement rain rate processing and website view

.gitignore

@@ -160,3 +160,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+database/archive_data/

database/migrations/010_add_rain_data_table.sql

@@ -0,0 +1,46 @@
+-- Migration 010: Add cml_rain_data table for processed rain rate estimates
+-- This table stores the processed rain rate estimates and intermediate products from CML data
+
+-- Create the main table
+CREATE TABLE cml_rain_data (
+    time TIMESTAMPTZ NOT NULL,
+    cml_id TEXT NOT NULL,
+    sublink_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    tl REAL,              -- Total loss (TSL - RSL)
+    wet BOOLEAN,          -- Wet/dry classification
+    baseline REAL,        -- Baseline attenuation
+    waa REAL,             -- Wet antenna attenuation
+    a_rain REAL,          -- Rain-induced path attenuation
+    r REAL,               -- Rain rate estimate (mm/h)
+    PRIMARY KEY (time, cml_id, sublink_id, user_id)
+);
+
+-- Convert to hypertable (TimescaleDB)
+SELECT create_hypertable('cml_rain_data', 'time');
+
+-- Enable compression (same strategy as cml_data)
+ALTER TABLE cml_rain_data SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'user_id, cml_id'
+);
+
+-- Add compression policy (compress chunks older than 7 days)
+SELECT add_compression_policy('cml_rain_data', INTERVAL '7 days');
+
+-- Add Row-Level Security (RLS) for multi-user isolation
+ALTER TABLE cml_rain_data ENABLE ROW LEVEL SECURITY;
+
+-- Create policy ensuring users only see their own data
+CREATE POLICY cml_rain_data_user_policy ON cml_rain_data
+    USING (user_id = current_user);
+
+-- Grant permissions to webserver_role for admin access
+GRANT SELECT, INSERT ON cml_rain_data TO webserver_role;
+
+-- Create security-barrier view for safe access
+CREATE VIEW cml_rain_data_secure WITH (security_barrier) AS
+    SELECT * FROM cml_rain_data
+    WHERE user_id = current_user;
+
+GRANT SELECT ON cml_rain_data_secure TO webserver_role;

database/migrations/011_add_rain_stats_view.sql

@@ -0,0 +1,49 @@
+-- Migration 011: Add cml_rain_stats view for rain rate statistics
+-- This view provides aggregated statistics for rain rate data, similar to cml_stats
+
+-- Create aggregated statistics view for rain rates
+CREATE OR REPLACE VIEW cml_rain_stats AS
+SELECT
+    r.cml_id::text,
+    r.user_id,
+    COUNT(*)                                                              AS total_records,
+    COUNT(CASE WHEN r.r IS NOT NULL AND r.r > 0 THEN 1 END)               AS valid_records,
+    ROUND(
+        100.0 * COUNT(CASE WHEN r.r IS NOT NULL AND r.r > 0 THEN 1 END) / COUNT(*),
+        2
+    )                                                                     AS completeness_percent,
+    ROUND(AVG(r.r)::numeric, 2)                                           AS mean_rain_rate,
+    ROUND(STDDEV(r.r)::numeric, 2)                                        AS stddev_rain_rate,
+    MAX(r.r)                                                              AS max_rain_rate,
+    -- Last rain rate (most recent non-zero value)
+    (
+        SELECT r2.r FROM cml_rain_data r2
+        WHERE r2.cml_id = r.cml_id
+          AND r2.user_id = r.user_id
+          AND r2.r IS NOT NULL AND r2.r > 0
+        ORDER BY r2.time DESC LIMIT 1
+    )                                                                     AS last_rain_rate,
+    -- 6-hour window statistics
+    ROUND(
+        100.0 * COUNT(CASE WHEN r.r IS NOT NULL AND r.r > 0 AND r.time >= NOW() - INTERVAL '6 hours' THEN 1 END)
+              / NULLIF(COUNT(*) FILTER (WHERE r.time >= NOW() - INTERVAL '6 hours'), 0),
+        2
+    )                                                                     AS completeness_percent_6h,
+    COUNT(*) FILTER (WHERE r.time >= NOW() - INTERVAL '6 hours')          AS total_records_6h,
+    COUNT(CASE WHEN r.r IS NOT NULL AND r.r > 0 AND r.time >= NOW() - INTERVAL '6 hours' THEN 1 END) AS valid_records_6h,
+    ROUND(AVG(r.r) FILTER (WHERE r.time >= NOW() - INTERVAL '6 hours')::numeric, 2) AS mean_rain_rate_6h,
+    ROUND(STDDEV(r.r) FILTER (WHERE r.time >= NOW() - INTERVAL '6 hours')::numeric, 2) AS stddev_rain_rate_6h,
+    -- 1-hour window statistics
+    ROUND(
+        100.0 * COUNT(CASE WHEN r.r IS NOT NULL AND r.r > 0 AND r.time >= NOW() - INTERVAL '1 hour' THEN 1 END)
+              / NULLIF(COUNT(*) FILTER (WHERE r.time >= NOW() - INTERVAL '1 hour'), 0),
+        2
+    )                                                                     AS completeness_percent_1h,
+    ROUND(AVG(r.r) FILTER (WHERE r.time >= NOW() - INTERVAL '1 hour')::numeric, 2) AS mean_rain_rate_1h,
+    ROUND(STDDEV(r.r) FILTER (WHERE r.time >= NOW() - INTERVAL '1 hour')::numeric, 2) AS stddev_rain_rate_1h,
+    NOW()                                                                 AS last_update
+FROM cml_rain_data r
+GROUP BY r.cml_id, r.user_id;
+
+-- Grant permissions to webserver_role for admin access
+GRANT SELECT ON cml_rain_stats TO webserver_role;

database/migrations/012_add_rain_data_1h.sql

@@ -0,0 +1,56 @@
+-- Migration 012: Add continuous aggregate for rain data (1-hour buckets)
+-- Similar to cml_data_1h, this provides efficient querying for Grafana dashboards
+
+-- Create 1-hour continuous aggregate for rain data
+CREATE MATERIALIZED VIEW cml_rain_data_1h
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', time) AS bucket,
+    user_id,
+    cml_id,
+    sublink_id,
+    MIN(r) AS r_min,
+    MAX(r) AS r_max,
+    AVG(r) AS r_avg,
+    MIN(tl) AS tl_min,
+    MAX(tl) AS tl_max,
+    AVG(tl) AS tl_avg,
+    MIN(a_rain) AS a_rain_min,
+    MAX(a_rain) AS a_rain_max,
+    AVG(a_rain) AS a_rain_avg
+FROM cml_rain_data
+GROUP BY bucket, user_id, cml_id, sublink_id
+WITH NO DATA;
+
+-- Automatically refresh every hour, covering up to 2 days of history
+SELECT add_continuous_aggregate_policy('cml_rain_data_1h',
+    start_offset      => INTERVAL '2 days',
+    end_offset        => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '1 hour'
+);
+
+-- Enable compression for chunks older than 7 days
+ALTER TABLE cml_rain_data_1h SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'user_id, cml_id',
+    timescaledb.compress_orderby = 'bucket'
+);
+
+SELECT add_compression_policy('cml_rain_data_1h', INTERVAL '7 days');
+
+-- Enable Row-Level Security
+ALTER TABLE cml_rain_data_1h ENABLE ROW LEVEL SECURITY;
+
+-- RLS policy for per-user isolation
+CREATE POLICY cml_rain_data_1h_user_policy ON cml_rain_data_1h
+    USING (user_id = current_user);
+
+-- Grant permissions
+GRANT SELECT ON cml_rain_data_1h TO webserver_role;
+
+-- Create security-barrier view for safe access
+CREATE VIEW cml_rain_data_1h_secure WITH (security_barrier) AS
+    SELECT * FROM cml_rain_data_1h
+    WHERE user_id = current_user;
+
+GRANT SELECT ON cml_rain_data_1h_secure TO webserver_role;

docker-compose.yml

@@ -138,17 +138,38 @@ services:
 
   processor:
     build: ./processor
+    restart: unless-stopped
+    depends_on:
+      database:
+        condition: service_healthy
+    volumes:
+      - ./processor/config:/app/config:ro
+      - processor_state:/app/data/state
+    environment:
+      - DATABASE_URL=postgresql://myuser:mypassword@database:5432/mydatabase
+
+  # Jupyter notebook for rain processing development and validation
+  processor_notebook:
+    build:
+      context: ./processor
+      dockerfile: Dockerfile.jupyter
     ports:
-      - "5002:5002"
+      - "9888:8888"
     depends_on:
-      - database
+      database:
+        condition: service_healthy
+    volumes:
+      - ./processor:/app:ro
+      - processor_notebook_data:/app/data
     environment:
       - DATABASE_URL=postgresql://myuser:mypassword@database:5432/mydatabase
+    profiles:
+      - notebook
 
   webserver:
     build: ./webserver
     ports:
-      - "5000:5000"
+      - "5001:5000"
     depends_on:
       - database
       - sftp_receiver
@@ -274,4 +295,6 @@ volumes:
   grafana_data:
   mno_openmrg_data_to_upload:
   mno_orange_cameroun_data_to_upload:
+  processor_state:
+  processor_notebook_data:
   # minio_data:  # Uncomment if using MinIO