[SPARK-55237][SQL] Suppress annoying messages when looking up nonexistent DBs

pan3793 · LuciferYang · commit 75cd9be439b4 · 2026-01-30T11:32:36.000+08:00
### What changes were proposed in this pull request? When running `spark-sql` with embedded HMS and default log4j config, I see a lot of noisy logs. ``` spark-sql (default)> create database test_db; 26/01/27 15:56:33 WARN ObjectStore: Failed to get database test_db, returning NoSuchObjectException 26/01/27 15:56:33 WARN ObjectStore: Failed to get database test_db, returning NoSuchObjectException 26/01/27 15:56:33 WARN ObjectStore: Failed to get database test_db, returning NoSuchObjectException Time taken: 0.87 seconds spark-sql (default)> ``` This PR modifies the default log4j config files to suppress those logs. Note, `ObjectStore` will not be called by the Spark driver when using remote HMS (typical deployment of production cases) ### Why are the changes needed? Improve UX on `spark-sql` with embedded HMS and default log4j config. ### Does this PR introduce _any_ user-facing change? Only affect logs. User would see less noise when running `spark-sql` with embedded HMS and default log4j config. ### How was this patch tested? before (see above section) vs after ``` spark-sql (default)> create database test_db2; Time taken: 0.686 seconds ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #54002 from pan3793/SPARK-55237. Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: yangjie01 <yangjie01@baidu.com>
diff --git a/common/utils-java/src/main/resources/org/apache/spark/log4j2-defaults.properties b/common/utils-java/src/main/resources/org/apache/spark/log4j2-defaults.properties
@@ -41,12 +41,12 @@ logger.repl2.level = info
 logger.repl.name = org.apache.spark.repl.Main
 logger.repl.level = warn
 
-# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs
-# in SparkSQL with Hive support
+# SPARK-9183, SPARK-55237: Settings to avoid annoying messages when looking up
+# nonexistent DBs/UDFs in spark-sql with Hive support
 logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
 logger.metastore.level = fatal
-logger.hive_functionregistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
-logger.hive_functionregistry.level = error
+logger.metastore2.name = org.apache.hadoop.hive.metastore.ObjectStore
+logger.metastore2.level = error
 
 # Parquet related logging
 logger.parquet.name = org.apache.parquet.CorruptStatistics
diff --git a/common/utils-java/src/main/resources/org/apache/spark/log4j2-json-layout.properties b/common/utils-java/src/main/resources/org/apache/spark/log4j2-json-layout.properties
@@ -41,12 +41,12 @@ logger.repl2.level = info
 logger.repl.name = org.apache.spark.repl.Main
 logger.repl.level = warn
 
-# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs
-# in SparkSQL with Hive support
+# SPARK-9183, SPARK-55237: Settings to avoid annoying messages when looking up
+# nonexistent DBs/UDFs in spark-sql with Hive support
 logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
 logger.metastore.level = fatal
-logger.hive_functionregistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
-logger.hive_functionregistry.level = error
+logger.metastore2.name = org.apache.hadoop.hive.metastore.ObjectStore
+logger.metastore2.level = error
 
 # Parquet related logging
 logger.parquet.name = org.apache.parquet.CorruptStatistics
diff --git a/conf/log4j2-json-layout.properties.template b/conf/log4j2-json-layout.properties.template
@@ -49,8 +49,8 @@ logger.parquet1.level = error
 logger.parquet2.name = parquet
 logger.parquet2.level = error
 
-# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
-logger.RetryingHMSHandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
-logger.RetryingHMSHandler.level = fatal
-logger.FunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
-logger.FunctionRegistry.level = error
+# SPARK-9183, SPARK-55237: Settings to avoid annoying messages when looking up nonexistent DBs/UDFs in spark-sql with Hive support
+logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
+logger.metastore.level = fatal
+logger.metastore2.name = org.apache.hadoop.hive.metastore.ObjectStore
+logger.metastore2.level = error
diff --git a/conf/log4j2.properties.template b/conf/log4j2.properties.template
@@ -55,8 +55,8 @@ logger.parquet1.level = error
 logger.parquet2.name = parquet
 logger.parquet2.level = error
 
-# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
-logger.RetryingHMSHandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
-logger.RetryingHMSHandler.level = fatal
-logger.FunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
-logger.FunctionRegistry.level = error
+# SPARK-9183, SPARK-55237: Settings to avoid annoying messages when looking up nonexistent DBs/UDFs in spark-sql with Hive support
+logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
+logger.metastore.level = fatal
+logger.metastore2.name = org.apache.hadoop.hive.metastore.ObjectStore
+logger.metastore2.level = error
