Close WebSocket connections that stop responding (#7807)

Both notification hubs send a Ping every 15 seconds but ignore the
Pong, so the loop only ends on a Close frame, EOF or a socket error.
When a client disappears without the TCP connection being closed, for
example behind a reverse proxy or CDN that keeps the upstream
connection open, none of these ever happens. The connection, its file
descriptor and its map entry are then kept forever, until the process
runs out of file descriptors and stops accepting connections and
opening the database.

Keep track of when something was last received from the client, Pongs
included, and close the connection when nothing was received for 30
seconds. This is the default ClientTimeoutInterval of ASP.NET Core
SignalR, which the official server uses.

Fixes #7805

Co-authored-by: Cassian433 <262026629+Cassian433@users.noreply.github.com>
Co-authored-by: Daniel García <dani-garcia@users.noreply.github.com>

src/api/notifications.rs

@@ -1,7 +1,7 @@
 use std::{
     net::IpAddr,
     sync::{Arc, LazyLock},
-    time::Duration,
+    time::{Duration, Instant},
 };
 
 use chrono::{NaiveDateTime, Utc};
@@ -41,6 +41,14 @@ pub static WS_ANONYMOUS_SUBSCRIPTIONS: LazyLock<Arc<AnonymousWebSocketSubscripti
 /// One connection is needed per pending login request, several at once are only expected behind NAT.
 const MAX_ANONYMOUS_CONNECTIONS_PER_IP: u32 = 25;
 
+/// How often a Ping is sent to the client.
+const WS_PING_INTERVAL: Duration = Duration::from_secs(15);
+
+/// Close the connection if nothing, not even a Pong, was received from the client for this long.
+/// Otherwise half-open connections (client gone without a FIN, e.g. behind a proxy or NAT) are kept forever.
+/// Same as the default `ClientTimeoutInterval` of ASP.NET Core SignalR, which the official server uses.
+const WS_CLIENT_TIMEOUT: Duration = Duration::from_secs(30);
+
 static NOTIFICATIONS_DISABLED: LazyLock<bool> = LazyLock::new(|| !CONFIG.enable_websocket() && !CONFIG.push_enabled());
 
 pub fn routes() -> Vec<Route> {
@@ -156,12 +164,16 @@ fn websockets_hub<'r>(
         rocket_ws::Stream! { ws => {
             let mut ws = ws;
             let _guard = guard;
-            let mut interval = tokio::time::interval(Duration::from_secs(15));
+            let mut interval = tokio::time::interval(WS_PING_INTERVAL);
+            let mut last_received = Instant::now();
             loop {
                 tokio::select! {
                     res = ws.next() =>  {
                         match res {
                             Some(Ok(message)) => {
+                                // Any message, including a Pong, means the client is still there
+                                last_received = Instant::now();
+
                                 match message {
                                     // Respond to any pings
                                     Message::Ping(ping) => yield Message::Pong(ping),
@@ -195,7 +207,13 @@ fn websockets_hub<'r>(
                         }
                     }
 
-                    _ = interval.tick() => yield Message::Ping(create_ping())
+                    _ = interval.tick() => {
+                        // The client stopped responding without closing the connection, drop it
+                        if last_received.elapsed() > WS_CLIENT_TIMEOUT {
+                            break;
+                        }
+                        yield Message::Ping(create_ping());
+                    }
                 }
             }
         }}
@@ -229,12 +247,16 @@ fn anonymous_websockets_hub<'r>(ws: WebSocket, token: String, ip: ClientIp) -> R
         rocket_ws::Stream! { ws => {
             let mut ws = ws;
             let _guard = guard;
-            let mut interval = tokio::time::interval(Duration::from_secs(15));
+            let mut interval = tokio::time::interval(WS_PING_INTERVAL);
+            let mut last_received = Instant::now();
             loop {
                 tokio::select! {
                     res = ws.next() =>  {
                         match res {
                             Some(Ok(message)) => {
+                                // Any message, including a Pong, means the client is still there
+                                last_received = Instant::now();
+
                                 match message {
                                     // Respond to any pings
                                     Message::Ping(ping) => yield Message::Pong(ping),
@@ -268,7 +290,13 @@ fn anonymous_websockets_hub<'r>(ws: WebSocket, token: String, ip: ClientIp) -> R
                         }
                     }
 
-                    _ = interval.tick() => yield Message::Ping(create_ping())
+                    _ = interval.tick() => {
+                        // The client stopped responding without closing the connection, drop it
+                        if last_received.elapsed() > WS_CLIENT_TIMEOUT {
+                            break;
+                        }
+                        yield Message::Ping(create_ping());
+                    }
                 }
             }
         }}