From 72af1652548a10f75d9c630b2146933640e1d2d2 Mon Sep 17 00:00:00 2001 From: Holger Nahrstaedt Date: Fri, 4 Sep 2020 09:37:01 +0200 Subject: [PATCH 1/4] Fixes GaussianProcessRegressor for sklearn >= 0.23 Fixes changed introduced by https://github.com/scikit-learn/scikit-learn/pull/15782/files --- doc/whats_new/v0.8.rst | 7 +++++++ skopt/learning/gaussian_process/gpr.py | 18 ++++++++++++++---- skopt/tests/test_acquisition.py | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/whats_new/v0.8.rst b/doc/whats_new/v0.8.rst index ea8bd641f..b2ec63e3c 100644 --- a/doc/whats_new/v0.8.rst +++ b/doc/whats_new/v0.8.rst @@ -4,6 +4,13 @@ .. _changes_0_8: +Version 0.8.1 +============= +**September 2020** + +- |Fix| GaussianProcessRegressor on sklearn 0.23 normalizes the + variance to 1, which needs to reverted on predict. + Version 0.8.0 ============= **September 2020** diff --git a/skopt/learning/gaussian_process/gpr.py b/skopt/learning/gaussian_process/gpr.py index 331d19ad7..9af582f9a 100644 --- a/skopt/learning/gaussian_process/gpr.py +++ b/skopt/learning/gaussian_process/gpr.py @@ -224,10 +224,15 @@ def fit(self, X, y): self.K_inv_ = L_inv.dot(L_inv.T) # Fix deprecation warning #462 - if int(sklearn.__version__[2:4]) >= 19: + if int(sklearn.__version__[2:4]) >= 23: + self.y_train_std_ = self._y_train_std self.y_train_mean_ = self._y_train_mean + elif int(sklearn.__version__[2:4]) >= 19: + self.y_train_mean_ = self._y_train_mean + self.y_train_std_ = 1 else: self.y_train_mean_ = self.y_train_mean + self.y_train_std_ = 1 return self @@ -309,11 +314,14 @@ def predict(self, X, return_std=False, return_cov=False, else: # Predict based on GP posterior K_trans = self.kernel_(X, self.X_train_) y_mean = K_trans.dot(self.alpha_) # Line 4 (y_mean = f_star) - y_mean = self.y_train_mean_ + y_mean # undo normal. + # undo normalisation + y_mean = self.y_train_std_ * y_mean + self.y_train_mean_ if return_cov: v = cho_solve((self.L_, True), K_trans.T) # Line 5 y_cov = self.kernel_(X) - K_trans.dot(v) # Line 6 + # undo normalisation + y_cov = y_cov * self.y_train_std_**2 return y_mean, y_cov elif return_std: @@ -330,17 +338,19 @@ def predict(self, X, return_std=False, return_cov=False, warnings.warn("Predicted variances smaller than 0. " "Setting those variances to 0.") y_var[y_var_negative] = 0.0 + # undo normalisation + y_var = y_var * self.y_train_std_**2 y_std = np.sqrt(y_var) if return_mean_grad: grad = self.kernel_.gradient_x(X[0], self.X_train_) - grad_mean = np.dot(grad.T, self.alpha_) + grad_mean = np.dot(grad.T, self.alpha_) * self.y_train_std_ if return_std_grad: grad_std = np.zeros(X.shape[1]) if not np.allclose(y_std, grad_std): grad_std = -np.dot(K_trans, - np.dot(K_inv, grad))[0] / y_std + np.dot(K_inv, grad))[0] / y_std * self.y_train_std_**2 return y_mean, y_std, grad_mean, grad_std if return_std: diff --git a/skopt/tests/test_acquisition.py b/skopt/tests/test_acquisition.py index 13117dcbe..a50b9116b 100644 --- a/skopt/tests/test_acquisition.py +++ b/skopt/tests/test_acquisition.py @@ -119,6 +119,19 @@ def test_acquisition_gradient(): check_gradient_correctness(X_new, gpr, acq_func, np.max(y)) +@pytest.mark.fast_test +def test_acquisition_gradient_cookbook(): + rng = np.random.RandomState(0) + X = rng.randn(20, 5) + y = rng.randn(20) + X_new = rng.randn(5) + gpr = cook_estimator("GP", Space(((-5.0, 5.0),)), random_state=0) + gpr.fit(X, y) + + for acq_func in ["LCB", "PI", "EI"]: + check_gradient_correctness(X_new, gpr, acq_func, np.max(y)) + + @pytest.mark.fast_test @pytest.mark.parametrize("acq_func", ["EIps", "PIps"]) def test_acquisition_per_second(acq_func): @@ -160,4 +173,5 @@ def test_acquisition_per_second_gradient(acq_func): gpr = cook_estimator("GP", Space(((-5.0, 5.0),)), random_state=0) mor = MultiOutputRegressor(gpr) mor.fit(X, y) + check_gradient_correctness(X_new, mor, acq_func, 1.5) From faa47ad36c5d2ce6af112e7af2666c5cf48322c6 Mon Sep 17 00:00:00 2001 From: Holger Nahrstaedt Date: Fri, 4 Sep 2020 09:40:17 +0200 Subject: [PATCH 2/4] Fix linting --- skopt/learning/gaussian_process/gpr.py | 9 ++++++--- skopt/tests/test_acquisition.py | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/skopt/learning/gaussian_process/gpr.py b/skopt/learning/gaussian_process/gpr.py index 9af582f9a..6b13bb686 100644 --- a/skopt/learning/gaussian_process/gpr.py +++ b/skopt/learning/gaussian_process/gpr.py @@ -344,13 +344,16 @@ def predict(self, X, return_std=False, return_cov=False, if return_mean_grad: grad = self.kernel_.gradient_x(X[0], self.X_train_) - grad_mean = np.dot(grad.T, self.alpha_) * self.y_train_std_ - + grad_mean = np.dot(grad.T, self.alpha_) + # undo normalisation + grad_mean = grad_mean * self.y_train_std_ if return_std_grad: grad_std = np.zeros(X.shape[1]) if not np.allclose(y_std, grad_std): grad_std = -np.dot(K_trans, - np.dot(K_inv, grad))[0] / y_std * self.y_train_std_**2 + np.dot(K_inv, grad))[0] / y_std + # undo normalisation + grad_std = grad_std * self.y_train_std_**2 return y_mean, y_std, grad_mean, grad_std if return_std: diff --git a/skopt/tests/test_acquisition.py b/skopt/tests/test_acquisition.py index a50b9116b..a75d20550 100644 --- a/skopt/tests/test_acquisition.py +++ b/skopt/tests/test_acquisition.py @@ -173,5 +173,4 @@ def test_acquisition_per_second_gradient(acq_func): gpr = cook_estimator("GP", Space(((-5.0, 5.0),)), random_state=0) mor = MultiOutputRegressor(gpr) mor.fit(X, y) - check_gradient_correctness(X_new, mor, acq_func, 1.5) From b019bd36b9432a830c39bd0f5a6c21922b2242cf Mon Sep 17 00:00:00 2001 From: Holger Nahrstaedt Date: Fri, 4 Sep 2020 09:46:22 +0200 Subject: [PATCH 3/4] Fix wrong label --- doc/whats_new/v0.9.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats_new/v0.9.rst b/doc/whats_new/v0.9.rst index e4c613f90..42bec8647 100644 --- a/doc/whats_new/v0.9.rst +++ b/doc/whats_new/v0.9.rst @@ -2,7 +2,7 @@ .. currentmodule:: skopt -.. _changes_0_8: +.. _changes_0_9: Version 0.9.0 ============= From ff574d1d5be1076549b5111e373abd70cbe3ad05 Mon Sep 17 00:00:00 2001 From: Holger Nahrstaedt Date: Fri, 4 Sep 2020 09:59:02 +0200 Subject: [PATCH 4/4] Add missing ref to 0.9 --- doc/templates/index.html | 2 ++ doc/whats_new.rst | 1 + doc/whats_new/v0.8.rst | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/templates/index.html b/doc/templates/index.html index b3f0736fa..7aff01909 100644 --- a/doc/templates/index.html +++ b/doc/templates/index.html @@ -109,6 +109,8 @@

News

  • On-going development: What's new (Changelog)
    • +
  • Sep 2020. scikit-optimize 0.8.1 (Changelog). +
  • Sep 2020. scikit-optimize 0.8 (Changelog).
  • Feb 2020. scikit-optimize 0.7.2 (Changelog).
  • Feb 2020. scikit-optimize 0.7.1 (Changelog).
  • Jan 2020. scikit-optimize 0.7 (Changelog). diff --git a/doc/whats_new.rst b/doc/whats_new.rst index b1e166f8e..15dd110f7 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -8,6 +8,7 @@ Release notes for all scikit-optimize releases are linked in this this page. .. toctree:: :maxdepth: 1 + Version 0.9 Version 0.8 Version 0.7 Version 0.6 diff --git a/doc/whats_new/v0.8.rst b/doc/whats_new/v0.8.rst index b2ec63e3c..822ff1cbb 100644 --- a/doc/whats_new/v0.8.rst +++ b/doc/whats_new/v0.8.rst @@ -2,7 +2,7 @@ .. currentmodule:: skopt -.. _changes_0_8: +.. _changes_0_8_1: Version 0.8.1 ============= @@ -11,6 +11,8 @@ Version 0.8.1 - |Fix| GaussianProcessRegressor on sklearn 0.23 normalizes the variance to 1, which needs to reverted on predict. +.. _changes_0_8: + Version 0.8.0 ============= **September 2020**