flowchart TB

subgraph EX[EXTRACT layer]
  E1[News collection from external APIs]
  E2[OpenAI MCP summarization and metadata extraction]
  E3[Metadata validation and cleanup]
  E4[Publish metadata event to Kafka topic]
  E1 --> E2 --> E3 --> E4
end

subgraph K[Kafka cluster]
  K1[Metadata topic]
end

E4 --> K1

subgraph TR[TRANSFORM layer]
  T1[Kafka consumer reads events]
  T2[Normalize metadata]
  T3[Apply business rules]
  T4[Publish processed event to Redis channels]
  T1 --> T2 --> T3 --> T4
end

K1 --> T1

subgraph LD[LOAD layer]
  L1[Store structured metadata in MySQL]
  L2[Store event data in Redis with TTL]
  L3[Cache real-time views in Redis]
end

T3 --> L1
T4 --> L2
T4 --> L3

subgraph BE[FastAPI backend]
  LS1[Redis listener subscribes to channels]
  LS2[SSE manager pushes news to clients]
  LS3[WebSocket manager pushes scope data]
  LS4[Update Redis based on events]
  SC1[Scheduler runs periodic jobs]
  SC2[Scheduler updates Redis]
  A1[Issue and validate tokens by user tier]
  A2[Apply rate limits and security]
  A3[Read from DB and return API responses]

  LS1 --> LS2
  LS1 --> LS3
  LS1 --> LS4
  SC1 --> SC2
  A1 --> A2 --> A3
end

L2 --> LS1
L3 --> LS1
L1 --> A3

subgraph FE[Frontend clients]
  FE1[Receive real-time news via SSE]
  FE2[Receive live scores via WebSocket]
  FE3[Call REST APIs for data]
end

LS2 --> FE1
LS3 --> FE2
A3 --> FE3

Summary