From d4644b0381979c7be8b2c069d95a76b35894cabd Mon Sep 17 00:00:00 2001 From: stiga-huang Date: Tue, 16 Dec 2025 18:50:47 +0800 Subject: [PATCH] IMPALA-13607, IMPALA-14490: Deflake test_cache_valid_on_nontransactional_table_ddls When catalogd runs with --start_hms_server=true, it services all the HMS endpoints so that any HMS compatible client would be able to use catalogd as a metadata cache. For all the DDL/DML requests, catalogd just delegates them to HMS APIs without reloading related metadata in the cache. For read requests like get_table_req, catalogd serves them from its cache which could be stale. There is a flag, invalidate_hms_cache_on_ddls, to decide whether to explicitly invalidate the table when catalogd delegates a DDL/DML on the table to HMS. test_cache_valid_on_nontransactional_table_ddls is a test verifying that when invalidate_hms_cache_on_ddls=false, the cache is not updated so should have stale metadata. However, there are HMS events generated from invoking the HMS APIs. Even when invalidate_hms_cache_on_ddls=false, catalogd can still update its cache when processing the corresponding HMS events. The test fails when its check is done after catalogd applies the event (so the cache is up-to-date). If the check is done before that, the test passes. This patch deflakes the test by explicitly disabling event processing. Also updates the description of invalidate_hms_cache_on_ddls to mention the impact of event processing. Tests: - Ran the test locally 100 times. Change-Id: Ib1ffc11a793899a0dbdb009bf2ac311117f2318e Reviewed-on: http://gerrit.cloudera.org:8080/23792 Reviewed-by: Impala Public Jenkins Tested-by: Impala Public Jenkins --- be/src/catalog/catalog-server.cc | 4 +++- tests/custom_cluster/test_metastore_service.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc index aab80c870..9e9ece243 100644 --- a/be/src/catalog/catalog-server.cc +++ b/be/src/catalog/catalog-server.cc @@ -130,7 +130,9 @@ DEFINE_bool(fallback_to_hms_on_errors, true, "This configuration is only used if DEFINE_bool(invalidate_hms_cache_on_ddls, true, "This configuration is used " "only if start_hms_server is true. This is used to invalidate catalogd cache " "for non transactional tables if alter/create/delete table hms apis are " - "invoked over catalogd's metastore endpoint"); + "invoked over catalogd's metastore endpoint. Note that when this flag is false, the " + "catalogd cache could still be updated by the event processor if " + "hms_event_polling_interval_s is greater than 0"); DEFINE_bool(hms_event_incremental_refresh_transactional_table, true, "When set to true " "events processor will refresh transactional tables incrementally for partition " diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py index dcbfeb2bd..1128bc7b3 100644 --- a/tests/custom_cluster/test_metastore_service.py +++ b/tests/custom_cluster/test_metastore_service.py @@ -456,9 +456,14 @@ class TestMetastoreService(CustomClusterTestSuite): "--hms_port=5899 " "--fallback_to_hms_on_errors=true " "--invalidate_hms_cache_on_ddls=false " + "--hms_event_polling_interval_s=0 " "--enable_sync_to_latest_event_on_ddls=false" ) def test_cache_valid_on_nontransactional_table_ddls(self): + """ + Test makes sure that the cache is not invalidated by table DDLs. Disable event + processing to ensure that the cache is not updated by the event processor. + """ db_name = ImpalaTestSuite.get_random_name( "test_cache_valid_on_nontransactional_table_ddls_db") tbl_name = ImpalaTestSuite.get_random_name(