diff --git a/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java b/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java index 06e340586..834a2110e 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java @@ -65,7 +65,7 @@ public class HdfsPartition implements Comparable { * Metadata for a single file in this partition. * TODO: Do we even need this class? Just get rid of it and use the Thrift version? */ - static public class FileDescriptor { + static public class FileDescriptor implements Comparable { private final THdfsFileDesc fileDescriptor_; public String getFileName() { return fileDescriptor_.getFile_name(); } @@ -116,6 +116,14 @@ public class HdfsPartition implements Comparable { .add("FileName", getFileName()) .add("Length", getFileLength()).toString(); } + + /** + * Orders file descriptors lexicographically by file name. + */ + @Override + public int compareTo(FileDescriptor otherFd) { + return getFileName().compareTo(otherFd.getFileName()); + } } /** diff --git a/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java b/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java index b8b533a3a..06195697a 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java @@ -1560,23 +1560,26 @@ public class HdfsTable extends Table { TResultSet result = new TResultSet(); TResultSetMetadata resultSchema = new TResultSetMetadata(); result.setSchema(resultSchema); - resultSchema.addToColumns(new TColumn("path", Type.STRING.toThrift())); - resultSchema.addToColumns(new TColumn("size", Type.STRING.toThrift())); - resultSchema.addToColumns(new TColumn("partition", Type.STRING.toThrift())); + resultSchema.addToColumns(new TColumn("Path", Type.STRING.toThrift())); + resultSchema.addToColumns(new TColumn("Size", Type.STRING.toThrift())); + resultSchema.addToColumns(new TColumn("Partition", Type.STRING.toThrift())); result.setRows(Lists.newArrayList()); - List partitions = null; + List orderedPartitions = null; if (partitionSpec == null) { - partitions = partitions_; + orderedPartitions = Lists.newArrayList(partitions_); + Collections.sort(orderedPartitions); } else { // Get the HdfsPartition object for the given partition spec. HdfsPartition partition = getPartitionFromThriftPartitionSpec(partitionSpec); Preconditions.checkState(partition != null); - partitions = Lists.newArrayList(partition); + orderedPartitions = Lists.newArrayList(partition); } - for (HdfsPartition p: partitions) { - for (FileDescriptor fd: p.getFileDescriptors()) { + for (HdfsPartition p: orderedPartitions) { + List orderedFds = Lists.newArrayList(p.getFileDescriptors()); + Collections.sort(orderedFds); + for (FileDescriptor fd: orderedFds) { TResultRowBuilder rowBuilder = new TResultRowBuilder(); rowBuilder.add(p.getLocation() + "/" + fd.getFileName()); rowBuilder.add(PrintUtils.printBytes(fd.getFileLength())); diff --git a/testdata/workloads/functional-query/queries/QueryTest/show.test b/testdata/workloads/functional-query/queries/QueryTest/show.test index 48d9b1bd9..61aa64d08 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/show.test +++ b/testdata/workloads/functional-query/queries/QueryTest/show.test @@ -213,32 +213,49 @@ show databases like 'def*' STRING,STRING ==== ---- QUERY -show files in insert_string_partitioned +show files in alltypesagg +---- LABELS +Path,Size,Partition ---- RESULTS -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__/000000_0',regex:.*,'year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=1/100101.txt',regex:.*,'year=2010/month=1/day=1' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=2/100102.txt',regex:.*,'year=2010/month=1/day=2' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=3/100103.txt',regex:.*,'year=2010/month=1/day=3' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=4/100104.txt',regex:.*,'year=2010/month=1/day=4' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=5/100105.txt',regex:.*,'year=2010/month=1/day=5' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=6/100106.txt',regex:.*,'year=2010/month=1/day=6' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=7/100107.txt',regex:.*,'year=2010/month=1/day=7' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=8/100108.txt',regex:.*,'year=2010/month=1/day=8' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=9/100109.txt',regex:.*,'year=2010/month=1/day=9' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=10/100110.txt',regex:.*,'year=2010/month=1/day=10' ---- TYPES STRING,STRING,STRING ==== ---- QUERY -show files in functional.insert_string_partitioned +show files in functional.alltypesagg +---- LABELS +Path,Size,Partition ---- RESULTS -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__/000000_0',regex:.*,'year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=1/100101.txt',regex:.*,'year=2010/month=1/day=1' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=2/100102.txt',regex:.*,'year=2010/month=1/day=2' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=3/100103.txt',regex:.*,'year=2010/month=1/day=3' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=4/100104.txt',regex:.*,'year=2010/month=1/day=4' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=5/100105.txt',regex:.*,'year=2010/month=1/day=5' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=6/100106.txt',regex:.*,'year=2010/month=1/day=6' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=7/100107.txt',regex:.*,'year=2010/month=1/day=7' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=8/100108.txt',regex:.*,'year=2010/month=1/day=8' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=9/100109.txt',regex:.*,'year=2010/month=1/day=9' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=10/100110.txt',regex:.*,'year=2010/month=1/day=10' ---- TYPES STRING,STRING,STRING ==== ---- QUERY -show files in insert_string_partitioned partition(s2='__HIVE_DEFAULT_PARTITION__') +show files in alltypesagg partition(year=2010,month=1,day=NULL) +---- LABELS +Path,Size,Partition ---- RESULTS -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' -row_regex: '$NAMENODE/test-warehouse/functional.db/insert_string_partitioned/s2=__HIVE_DEFAULT_PARTITION__/[a-z0-9]+\-[a-z0-9]+_\d+_data\.\d+\..*','7B','s2=__HIVE_DEFAULT_PARTITION__' +'hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__/000000_0',regex:.*,'year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__' ---- TYPES STRING,STRING,STRING ====