jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 18:12:08 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-12209: Always include format-version in DESCRIBE FORMATTED and SHOW CREATE TABLE for Iceberg tables

Browse Source 
HiveCatalog does not include format-version for Iceberg tables in the
table's parameters, therefore the output of SHOW CREATE TABLE may not
replicate the original table.
This patch makes sure to add it to both the SHOW CREATE TABLE and
DESCRIBE FORMATTED/EXTENDED output.

Additionally, adds ICEBERG_DEFAULT_FORMAT_VERSION variable to E2E
tests, deducting from IMPALA_ICEBERG_VERSION environment variable.

If Iceberg version is at least 1.4, default format-version is 2, before
1.4 it's 1. This way tests can work with multiple Iceberg versions.

Testing:
 * updated show-create-table.test and show-create-table-with-stats.test
   for Iceberg tables
 * added format-version checks to multiple DESCRIBE FORMATTED tests

Change-Id: I991edf408b24fa73e8a8abe64ac24929aeb8e2f8
Reviewed-on: http://gerrit.cloudera.org:8080/23514
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Daniel Vanko
2025-10-07 16:07:59 +02:00
committed by Impala Public Jenkins
parent f6ceca2b4d
commit 3d22c7fe05
21 changed files with 243 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
View File

@@ -189,10 +189,10 @@ public class ToSqlUtils {
commonProps.remove(KuduTable.KEY_TABLE_ID);
} else if (table instanceof FeIcebergTable) {
FeIcebergTable feIcebergTable = (FeIcebergTable) table;
if (feIcebergTable.getFormatVersion() == IcebergTable.ICEBERG_FORMAT_V1) {
commonProps.put(TableProperties.FORMAT_VERSION,
String.valueOf(IcebergTable.ICEBERG_FORMAT_V1));
}
// Add "format-version" property if it's not already present.
commonProps.putIfAbsent(IcebergTable.FORMAT_VERSION,
Integer.toString(feIcebergTable.getFormatVersion()));
// Hide Iceberg internal metadata properties
removeHiddenIcebergTableProperties(commonProps);
} else if (table instanceof FePaimonTable) {

3
fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
View File

@@ -164,6 +164,9 @@ public class IcebergTable extends Table implements FeIcebergTable {
// Internal Iceberg table property that specifies the UUID of the table.
public static final String UUID = "uuid";
// Internal Iceberg table property that specifies the table format version.
public static final String FORMAT_VERSION = "format-version";
// Parquet compression codec and compression level table properties.
public static final String PARQUET_COMPRESSION_CODEC =
"write.parquet.compression-codec";

8
fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
View File

@@ -30,6 +30,7 @@ import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.IcebergColumn;
import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
@@ -236,8 +237,13 @@ public class DescribeResultFactory {
sb.append(MetastoreShim.getAllColumnsInformation(msTable.getSd().getCols(),
msTable.getPartitionKeys(), true, false, true));
if (table instanceof FeIcebergTable) {
FeIcebergTable feIcebergTable = (FeIcebergTable) table;
sb.append(MetastoreShim.getPartitionTransformInformation(
FeIcebergTable.Utils.getPartitionTransformKeys((FeIcebergTable) table)));
FeIcebergTable.Utils.getPartitionTransformKeys(feIcebergTable)));
// msTable is a deep copy hence we can add the "format-version" parameter to it
msTable.getParameters().putIfAbsent(IcebergTable.FORMAT_VERSION,
Integer.toString(feIcebergTable.getFormatVersion()));
}
// Add the extended table metadata information.
sb.append(MetastoreShim.getTableInformation(msTable));