jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 18:12:08 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-14062: Adds missing timeline items in constructing PartitionDeltaUpdater

Browse Source 
PartitionDeltaUpdater has two sub-classes, PartNameBasedDeltaUpdater and
PartBasedDeltaUpdater. They are used in reloading metadata of a table.
Their constructors invoke HMS RPCs which could be slow and should be
tracked in the catalog timeline.

This patch adds missing timeline items for those HMS RPCs.

Tests:
 - Added e2e tests

Change-Id: Id231c2b15869aac2dae3258817954abf119da802
Reviewed-on: http://gerrit.cloudera.org:8080/22917
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
stiga-huang
2025-05-19 20:38:58 +08:00
committed by Impala Public Jenkins
parent c7f9469919
commit dd2d44492d
2 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
View File

@@ -1621,10 +1621,12 @@ public class HdfsTable extends Table implements FeFsTable {
partitionList = MetaStoreUtil
.fetchPartitionsByName(client, Lists.newArrayList(partitionsToUpdate),
msTable_);
catalogTimeline.markEvent("Fetched updated partitions");
} else {
partitionList =
MetaStoreUtil.fetchAllPartitions(
client_, msTable_, NUM_PARTITION_FETCH_RETRIES);
catalogTimeline.markEvent("Fetched all partitions");
}
LOG.debug("Time taken to fetch all partitions of table {}: {} msec", getFullName(),
sw.stop().elapsed(TimeUnit.MILLISECONDS));
@@ -1726,6 +1728,7 @@ public class HdfsTable extends Table implements FeFsTable {
// (~.3 secs for 30K partitions).
partitionNamesFromHms_ = new HashSet<>(client_
.listPartitionNames(db_.getName(), name_, (short) -1));
catalogTimeline.markEvent("Fetched partition names");
}
@Override

15
tests/query_test/test_observability.py
View File

@@ -617,12 +617,15 @@ class TestObservability(ImpalaTestSuite):
])
# REFRESH
stmt = "refresh %s.t1" % unique_database
self.__verify_event_labels_in_profile(stmt, [
"Got Metastore client",
"Fetched table from Metastore",
"Loaded file metadata for 2 partitions",
"Finished resetMetadata request"
])
for i in (0, 1):
self.client.execute("set refresh_updated_hms_partitions=" + str(i))
self.__verify_event_labels_in_profile(stmt, [
"Got Metastore client",
"Fetched table from Metastore",
"Fetched partition names" if i == 0 else "Fetched all partitions",
"Loaded file metadata for 2 partitions",
"Finished resetMetadata request"
])
# Drop table
stmt = "drop table %s.t1" % unique_database
self.__verify_event_labels_in_profile(stmt, [