Add KL divergence loss function (TheAlgorithms#656)

avats-dev · web-flow · commit 9e41aad7dadf · 2024-01-09T21:57:17.000+02:00
diff --git a/DIRECTORY.md b/DIRECTORY.md
@@ -145,6 +145,7 @@
     * Loss Function
       * [Mean Absolute Error Loss](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/loss_function/mean_absolute_error_loss.rs)
       * [Mean Squared Error Loss](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/loss_function/mean_squared_error_loss.rs)
+      * [KL Divergence Loss](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/loss_function/kl_divergence_loss.rs)
     * Optimization
       * [Adam](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/optimization/adam.rs)
       * [Gradient Descent](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/optimization/gradient_descent.rs)
diff --git a/src/machine_learning/loss_function/kl_divergence_loss.rs b/src/machine_learning/loss_function/kl_divergence_loss.rs
@@ -0,0 +1,37 @@
+//! # KL divergence Loss Function
+//!
+//! For a pair of actual and predicted probability distributions represented as vectors `actual` and `predicted`, the KL divergence loss is calculated as:
+//!
+//! `L = -Σ(actual[i] * ln(predicted[i]/actual[i]))` for all `i` in the range of the vectors
+//!
+//! Where `ln` is the natural logarithm function, and `Σ` denotes the summation over all elements of the vectors.
+//!
+//! ## KL divergence Loss Function Implementation
+//!
+//! This implementation takes two references to vectors of f64 values, `actual` and `predicted`, and returns the KL divergence loss between them.
+//!
+pub fn kld_loss(actual: &[f64], predicted: &[f64]) -> f64 {
+    // epsilon to handle if any of the elements are zero
+    let eps = 0.00001f64;
+    let loss: f64 = actual
+        .iter()
+        .zip(predicted.iter())
+        .map(|(&a, &p)| ((a + eps) * ((a + eps) / (p + eps)).ln()))
+        .sum();
+    loss
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_kld_loss() {
+        let test_vector_actual = vec![1.346112, 1.337432, 1.246655];
+        let test_vector = vec![1.033836, 1.082015, 1.117323];
+        assert_eq!(
+            kld_loss(&test_vector_actual, &test_vector),
+            0.7752789394328498
+        );
+    }
+}
diff --git a/src/machine_learning/loss_function/mod.rs b/src/machine_learning/loss_function/mod.rs
@@ -1,5 +1,7 @@
+mod kl_divergence_loss;
 mod mean_absolute_error_loss;
 mod mean_squared_error_loss;
 
+pub use self::kl_divergence_loss::kld_loss;
 pub use self::mean_absolute_error_loss::mae_loss;
 pub use self::mean_squared_error_loss::mse_loss;
diff --git a/src/machine_learning/mod.rs b/src/machine_learning/mod.rs
@@ -8,6 +8,7 @@ mod simpson;
 pub use self::cholesky::cholesky;
 pub use self::k_means::k_means;
 pub use self::linear_regression::linear_regression;
+pub use self::loss_function::kld_loss;
 pub use self::loss_function::mae_loss;
 pub use self::loss_function::mse_loss;
 pub use self::optimization::gradient_descent;
