refactor: [torrust#1228] get metrics from HTTP and UDP Tracker Core Stats

Stats have been splited into HTTP and UDP stats.

Parallel change, step 3:

1. [x] Start using HTTP Tracker Core Stats
2. [x] Start using UDP  Tracker Core Stats
3. [x] Get metrics from HTTP and UDP Tracker Core Stats
4. [ ] Remove deprecated unified HTTP and UDP stats.

Author: josecelano

src/container.rs

@@ -100,8 +100,8 @@ pub struct HttpApiContainer {
     pub keys_handler: Arc<KeysHandler>,
     pub whitelist_manager: Arc<WhitelistManager>,
     pub ban_service: Arc<RwLock<BanService>>,
-    pub stats_event_sender: Arc<Option<Box<dyn Sender>>>,
-    pub stats_repository: Arc<Repository>,
+    pub http_stats_repository: Arc<http_tracker_core::statistics::repository::Repository>,
+    pub udp_stats_repository: Arc<udp_tracker_core::statistics::repository::Repository>,
 }
 
 impl HttpApiContainer {
@@ -114,8 +114,8 @@ impl HttpApiContainer {
             keys_handler: app_container.keys_handler.clone(),
             whitelist_manager: app_container.whitelist_manager.clone(),
             ban_service: app_container.ban_service.clone(),
-            stats_event_sender: app_container.stats_event_sender.clone(),
-            stats_repository: app_container.stats_repository.clone(),
+            http_stats_repository: app_container.http_stats_repository.clone(),
+            udp_stats_repository: app_container.udp_stats_repository.clone(),
         }
     }
 }

src/packages/mod.rs

@@ -3,4 +3,5 @@
 //! It will be moved to the directory `packages`.
 pub mod http_tracker_core;
 pub mod statistics;
+pub mod tracker_api_core;
 pub mod udp_tracker_core;

src/packages/tracker_api_core/mod.rs

@@ -0,0 +1 @@
+pub mod statistics;

src/packages/tracker_api_core/statistics/metrics.rs

@@ -0,0 +1,87 @@
+/// Metrics collected by the tracker.
+///
+/// - Number of connections handled
+/// - Number of `announce` requests handled
+/// - Number of `scrape` request handled
+///
+/// These metrics are collected for each connection type: UDP and HTTP
+/// and also for each IP version used by the peers: IPv4 and IPv6.
+#[derive(Debug, PartialEq, Default)]
+pub struct Metrics {
+    /// Total number of TCP (HTTP tracker) connections from IPv4 peers.
+    /// Since the HTTP tracker spec does not require a handshake, this metric
+    /// increases for every HTTP request.
+    pub tcp4_connections_handled: u64,
+
+    /// Total number of TCP (HTTP tracker) `announce` requests from IPv4 peers.
+    pub tcp4_announces_handled: u64,
+
+    /// Total number of TCP (HTTP tracker) `scrape` requests from IPv4 peers.
+    pub tcp4_scrapes_handled: u64,
+
+    /// Total number of TCP (HTTP tracker) connections from IPv6 peers.
+    pub tcp6_connections_handled: u64,
+
+    /// Total number of TCP (HTTP tracker) `announce` requests from IPv6 peers.
+    pub tcp6_announces_handled: u64,
+
+    /// Total number of TCP (HTTP tracker) `scrape` requests from IPv6 peers.
+    pub tcp6_scrapes_handled: u64,
+
+    // UDP
+    /// Total number of UDP (UDP tracker) requests aborted.
+    pub udp_requests_aborted: u64,
+
+    /// Total number of UDP (UDP tracker) requests banned.
+    pub udp_requests_banned: u64,
+
+    /// Total number of banned IPs.
+    pub udp_banned_ips_total: u64,
+
+    /// Average rounded time spent processing UDP connect requests.
+    pub udp_avg_connect_processing_time_ns: u64,
+
+    /// Average rounded time spent processing UDP announce requests.
+    pub udp_avg_announce_processing_time_ns: u64,
+
+    /// Average rounded time spent processing UDP scrape requests.
+    pub udp_avg_scrape_processing_time_ns: u64,
+
+    // UDPv4
+    /// Total number of UDP (UDP tracker) requests from IPv4 peers.
+    pub udp4_requests: u64,
+
+    /// Total number of UDP (UDP tracker) connections from IPv4 peers.
+    pub udp4_connections_handled: u64,
+
+    /// Total number of UDP (UDP tracker) `announce` requests from IPv4 peers.
+    pub udp4_announces_handled: u64,
+
+    /// Total number of UDP (UDP tracker) `scrape` requests from IPv4 peers.
+    pub udp4_scrapes_handled: u64,
+
+    /// Total number of UDP (UDP tracker) responses from IPv4 peers.
+    pub udp4_responses: u64,
+
+    /// Total number of UDP (UDP tracker) `error` requests from IPv4 peers.
+    pub udp4_errors_handled: u64,
+
+    // UDPv6
+    /// Total number of UDP (UDP tracker) requests from IPv6 peers.
+    pub udp6_requests: u64,
+
+    /// Total number of UDP (UDP tracker) `connection` requests from IPv6 peers.
+    pub udp6_connections_handled: u64,
+
+    /// Total number of UDP (UDP tracker) `announce` requests from IPv6 peers.
+    pub udp6_announces_handled: u64,
+
+    /// Total number of UDP (UDP tracker) `scrape` requests from IPv6 peers.
+    pub udp6_scrapes_handled: u64,
+
+    /// Total number of UDP (UDP tracker) responses from IPv6 peers.
+    pub udp6_responses: u64,
+
+    /// Total number of UDP (UDP tracker) `error` requests from IPv6 peers.
+    pub udp6_errors_handled: u64,
+}

src/packages/tracker_api_core/statistics/mod.rs

@@ -0,0 +1,2 @@
+pub mod metrics;
+pub mod services;

src/packages/tracker_api_core/statistics/services.rs

@@ -0,0 +1,127 @@
+use std::sync::Arc;
+
+use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
+use packages::tracker_api_core::statistics::metrics::Metrics;
+use tokio::sync::RwLock;
+use torrust_tracker_primitives::torrent_metrics::TorrentsMetrics;
+
+use crate::packages::{self, http_tracker_core, udp_tracker_core};
+use crate::servers::udp::server::banning::BanService;
+
+/// All the metrics collected by the tracker.
+#[derive(Debug, PartialEq)]
+pub struct TrackerMetrics {
+    /// Domain level metrics.
+    ///
+    /// General metrics for all torrents (number of seeders, leechers, etcetera)
+    pub torrents_metrics: TorrentsMetrics,
+
+    /// Application level metrics. Usage statistics/metrics.
+    ///
+    /// Metrics about how the tracker is been used (number of udp announce requests, number of http scrape requests, etcetera)
+    pub protocol_metrics: Metrics,
+}
+
+/// It returns all the [`TrackerMetrics`]
+pub async fn get_metrics(
+    in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
+    ban_service: Arc<RwLock<BanService>>,
+    http_stats_repository: Arc<http_tracker_core::statistics::repository::Repository>,
+    udp_stats_repository: Arc<udp_tracker_core::statistics::repository::Repository>,
+) -> TrackerMetrics {
+    let torrents_metrics = in_memory_torrent_repository.get_torrents_metrics();
+    let udp_banned_ips_total = ban_service.read().await.get_banned_ips_total();
+    let http_stats = http_stats_repository.get_stats().await;
+    let udp_stats = udp_stats_repository.get_stats().await;
+
+    TrackerMetrics {
+        torrents_metrics,
+        protocol_metrics: Metrics {
+            // TCPv4
+            tcp4_connections_handled: http_stats.tcp4_connections_handled,
+            tcp4_announces_handled: http_stats.tcp4_announces_handled,
+            tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled,
+            // TCPv6
+            tcp6_connections_handled: http_stats.tcp6_connections_handled,
+            tcp6_announces_handled: http_stats.tcp6_announces_handled,
+            tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled,
+            // UDP
+            udp_requests_aborted: udp_stats.udp_requests_aborted,
+            udp_requests_banned: udp_stats.udp_requests_banned,
+            udp_banned_ips_total: udp_banned_ips_total as u64,
+            udp_avg_connect_processing_time_ns: udp_stats.udp_avg_connect_processing_time_ns,
+            udp_avg_announce_processing_time_ns: udp_stats.udp_avg_announce_processing_time_ns,
+            udp_avg_scrape_processing_time_ns: udp_stats.udp_avg_scrape_processing_time_ns,
+            // UDPv4
+            udp4_requests: udp_stats.udp4_requests,
+            udp4_connections_handled: udp_stats.udp4_connections_handled,
+            udp4_announces_handled: udp_stats.udp4_announces_handled,
+            udp4_scrapes_handled: udp_stats.udp4_scrapes_handled,
+            udp4_responses: udp_stats.udp4_responses,
+            udp4_errors_handled: udp_stats.udp4_errors_handled,
+            // UDPv6
+            udp6_requests: udp_stats.udp6_requests,
+            udp6_connections_handled: udp_stats.udp6_connections_handled,
+            udp6_announces_handled: udp_stats.udp6_announces_handled,
+            udp6_scrapes_handled: udp_stats.udp6_scrapes_handled,
+            udp6_responses: udp_stats.udp6_responses,
+            udp6_errors_handled: udp_stats.udp6_errors_handled,
+        },
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
+    use bittorrent_tracker_core::{self};
+    use tokio::sync::RwLock;
+    use torrust_tracker_configuration::Configuration;
+    use torrust_tracker_primitives::torrent_metrics::TorrentsMetrics;
+    use torrust_tracker_test_helpers::configuration;
+
+    use crate::packages::tracker_api_core::statistics::metrics::Metrics;
+    use crate::packages::tracker_api_core::statistics::services::{get_metrics, TrackerMetrics};
+    use crate::packages::{http_tracker_core, udp_tracker_core};
+    use crate::servers::udp::server::banning::BanService;
+    use crate::servers::udp::server::launcher::MAX_CONNECTION_ID_ERRORS_PER_IP;
+
+    pub fn tracker_configuration() -> Configuration {
+        configuration::ephemeral()
+    }
+
+    #[tokio::test]
+    async fn the_statistics_service_should_return_the_tracker_metrics() {
+        let config = tracker_configuration();
+
+        let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default());
+        let ban_service = Arc::new(RwLock::new(BanService::new(MAX_CONNECTION_ID_ERRORS_PER_IP)));
+
+        // HTTP stats
+        let (_http_stats_event_sender, http_stats_repository) =
+            http_tracker_core::statistics::setup::factory(config.core.tracker_usage_statistics);
+        let http_stats_repository = Arc::new(http_stats_repository);
+
+        // UDP stats
+        let (_udp_stats_event_sender, udp_stats_repository) =
+            udp_tracker_core::statistics::setup::factory(config.core.tracker_usage_statistics);
+        let udp_stats_repository = Arc::new(udp_stats_repository);
+
+        let tracker_metrics = get_metrics(
+            in_memory_torrent_repository.clone(),
+            ban_service.clone(),
+            http_stats_repository.clone(),
+            udp_stats_repository.clone(),
+        )
+        .await;
+
+        assert_eq!(
+            tracker_metrics,
+            TrackerMetrics {
+                torrents_metrics: TorrentsMetrics::default(),
+                protocol_metrics: Metrics::default(),
+            }
+        );
+    }
+}

src/servers/apis/v1/context/stats/handlers.rs

@@ -6,13 +6,12 @@ use axum::extract::State;
 use axum::response::Response;
 use axum_extra::extract::Query;
 use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
-use packages::statistics::repository::Repository;
 use serde::Deserialize;
 use tokio::sync::RwLock;
 
 use super::responses::{metrics_response, stats_response};
-use crate::packages;
-use crate::packages::statistics::services::get_metrics;
+use crate::packages::tracker_api_core::statistics::services::get_metrics;
+use crate::packages::{http_tracker_core, udp_tracker_core};
 use crate::servers::udp::server::banning::BanService;
 
 #[derive(Deserialize, Debug, Default)]
@@ -41,10 +40,15 @@ pub struct QueryParams {
 /// for more information about this endpoint.
 #[allow(clippy::type_complexity)]
 pub async fn get_stats_handler(
-    State(state): State<(Arc<InMemoryTorrentRepository>, Arc<RwLock<BanService>>, Arc<Repository>)>,
+    State(state): State<(
+        Arc<InMemoryTorrentRepository>,
+        Arc<RwLock<BanService>>,
+        Arc<http_tracker_core::statistics::repository::Repository>,
+        Arc<udp_tracker_core::statistics::repository::Repository>,
+    )>,
     params: Query<QueryParams>,
 ) -> Response {
-    let metrics = get_metrics(state.0.clone(), state.1.clone(), state.2.clone()).await;
+    let metrics = get_metrics(state.0.clone(), state.1.clone(), state.2.clone(), state.3.clone()).await;
 
     match params.0.format {
         Some(format) => match format {

src/servers/apis/v1/context/stats/resources.rs

@@ -2,7 +2,7 @@
 //! API context.
 use serde::{Deserialize, Serialize};
 
-use crate::packages::statistics::services::TrackerMetrics;
+use crate::packages::tracker_api_core::statistics::services::TrackerMetrics;
 
 /// It contains all the statistics generated by the tracker.
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
@@ -118,11 +118,11 @@ impl From<TrackerMetrics> for Stats {
 
 #[cfg(test)]
 mod tests {
-    use packages::statistics::metrics::Metrics;
+    use packages::tracker_api_core::statistics::metrics::Metrics;
     use torrust_tracker_primitives::torrent_metrics::TorrentsMetrics;
 
     use super::Stats;
-    use crate::packages::statistics::services::TrackerMetrics;
+    use crate::packages::tracker_api_core::statistics::services::TrackerMetrics;
     use crate::packages::{self};
 
     #[test]

src/servers/apis/v1/context/stats/responses.rs

@@ -3,7 +3,7 @@
 use axum::response::{IntoResponse, Json, Response};
 
 use super::resources::Stats;
-use crate::packages::statistics::services::TrackerMetrics;
+use crate::packages::tracker_api_core::statistics::services::TrackerMetrics;
 
 /// `200` response that contains the [`Stats`] resource as json.
 #[must_use]

src/servers/apis/v1/context/stats/routes.rs

@@ -18,7 +18,8 @@ pub fn add(prefix: &str, router: Router, http_api_container: &Arc<HttpApiContain
         get(get_stats_handler).with_state((
             http_api_container.in_memory_torrent_repository.clone(),
             http_api_container.ban_service.clone(),
-            http_api_container.stats_repository.clone(),
+            http_api_container.http_stats_repository.clone(),
+            http_api_container.udp_stats_repository.clone(),
         )),
     )
 }

tests/servers/api/environment.rs

@@ -61,8 +61,8 @@ impl Environment<Stopped> {
             keys_handler: app_container.keys_handler.clone(),
             whitelist_manager: app_container.whitelist_manager.clone(),
             ban_service: app_container.ban_service.clone(),
-            stats_event_sender: app_container.stats_event_sender.clone(),
-            stats_repository: app_container.stats_repository.clone(),
+            http_stats_repository: app_container.http_stats_repository.clone(),
+            udp_stats_repository: app_container.udp_stats_repository.clone(),
         });
 
         Self {