diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift index 89152fdf0..9b2586afe 100644 --- a/common/thrift/Frontend.thrift +++ b/common/thrift/Frontend.thrift @@ -263,8 +263,14 @@ struct TCreateTableLikeParams { // Do not throw an error if a table of the same name already exists. 4: required bool if_not_exists + // Optional file format for this table + 5: optional TFileFormat file_format + + // Optional comment for the table + 6: optional string comment + // Optional storage location for the table - 5: optional string location + 7: optional string location } // Parameters of CREATE TABLE commands diff --git a/fe/src/main/cup/sql-parser.y b/fe/src/main/cup/sql-parser.y index 462d4d7f0..ed842bea4 100644 --- a/fe/src/main/cup/sql-parser.y +++ b/fe/src/main/cup/sql-parser.y @@ -387,11 +387,19 @@ create_db_stmt ::= create_tbl_like_stmt ::= KW_CREATE external_val:external KW_TABLE if_not_exists_val:if_not_exists - table_name:table KW_LIKE table_name:other_table location_val:location + table_name:table KW_LIKE table_name:other_table comment_val:comment + KW_STORED KW_AS file_format_val:file_format location_val:location {: - RESULT = new CreateTableLikeStmt(table, other_table, external, location, - if_not_exists); - :} + RESULT = new CreateTableLikeStmt(table, other_table, external, comment, + file_format, location, if_not_exists); + :} + | KW_CREATE external_val:external KW_TABLE if_not_exists_val:if_not_exists + table_name:table KW_LIKE table_name:other_table comment_val:comment + location_val:location + {: + RESULT = new CreateTableLikeStmt(table, other_table, external, comment, + null, location, if_not_exists); + :} ; create_tbl_stmt ::= diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CreateTableLikeStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/CreateTableLikeStmt.java index 19fd6517e..1621cd98a 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/CreateTableLikeStmt.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/CreateTableLikeStmt.java @@ -37,6 +37,8 @@ public class CreateTableLikeStmt extends ParseNodeBase { private final TableName tableName; private final TableName srcTableName; private final boolean isExternal; + private final String comment; + private final FileFormat fileFormat; private final String location; private final boolean ifNotExists; @@ -49,16 +51,21 @@ public class CreateTableLikeStmt extends ParseNodeBase { * @param tableName - Name of the new table * @param srcTableName - Name of the source table (table to copy) * @param isExternal - If true, the table's data will be preserved if dropped. + * @param comment - Comment to attach to the table + * @param fileFormat - File format of the table * @param location - The HDFS location of where the table data will stored. * @param ifNotExists - If true, no errors are thrown if the table already exists */ public CreateTableLikeStmt(TableName tableName, TableName srcTableName, - boolean isExternal, String location, boolean ifNotExists) { + boolean isExternal, String comment, FileFormat fileFormat, String location, + boolean ifNotExists) { Preconditions.checkNotNull(tableName); Preconditions.checkNotNull(srcTableName); this.tableName = tableName; this.srcTableName = srcTableName; this.isExternal = isExternal; + this.comment = comment; + this.fileFormat = fileFormat; this.location = location; this.ifNotExists = ifNotExists; } @@ -97,6 +104,14 @@ public class CreateTableLikeStmt extends ParseNodeBase { return ifNotExists; } + public String getComment() { + return comment; + } + + public FileFormat getFileFormat() { + return fileFormat; + } + public String getLocation() { return location; } @@ -111,7 +126,7 @@ public class CreateTableLikeStmt extends ParseNodeBase { sb.append("EXTERNAL "); } sb.append("TABLE "); - if (ifNotExists) { + if (ifNotExists) { sb.append("IF NOT EXISTS "); } if (tableName.getDb() != null) { @@ -122,9 +137,14 @@ public class CreateTableLikeStmt extends ParseNodeBase { sb.append(srcTableName.getDb() + "."); } sb.append(srcTableName.getTbl()); - + if (comment != null) { + sb.append(" COMMENT '" + comment + "'"); + } + if (fileFormat != null) { + sb.append(" STORED AS " + fileFormat); + } if (location != null) { - sb.append(" LOCATION = '" + location + "'"); + sb.append(" LOCATION '" + location + "'"); } return sb.toString(); } @@ -134,6 +154,10 @@ public class CreateTableLikeStmt extends ParseNodeBase { params.setTable_name(new TTableName(getDb(), getTbl())); params.setSrc_table_name(new TTableName(getSrcDb(), getSrcTbl())); params.setIs_external(isExternal()); + params.setComment(comment); + if (fileFormat != null) { + params.setFile_format(fileFormat.toThrift()); + } params.setLocation(location); params.setIf_not_exists(getIfNotExists()); return params; @@ -155,12 +179,12 @@ public class CreateTableLikeStmt extends ParseNodeBase { if (!analyzer.getCatalog().containsTable(srcDbName, getSrcTbl())) { throw new AnalysisException(String.format("Source table does not exist: %s.%s", - srcDbName, getSrcTbl())); + srcDbName, getSrcTbl())); } if (analyzer.getCatalog().containsTable(dbName, getTbl()) && !ifNotExists) { throw new AnalysisException(String.format("Table already exists: %s.%s", - dbName, getTbl())); + dbName, getTbl())); } } } diff --git a/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java b/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java index 1651d4539..a2b3d2512 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java @@ -136,7 +136,7 @@ public class Catalog { /** * Invalidate the metadata for the given db name and marks the db metadata load * state as uninitialized. Invalidating the metadata will cause the next access to - * the db to reload (synchronize) its metadata from the metastore. + * the db to reload (synchronize) its metadata from the metastore. * If ifExists is true, this will only invalidate if the db name already exists in * the dbNameMap. If ifExists is false, the db metadata will be invalidated and the * metadata state will be set as UNINITIALIZED (potentially adding a new item to the @@ -157,7 +157,7 @@ public class Catalog { } /** - * Removes the database from the metadata cache + * Removes the database from the metadata cache */ public void remove(String dbName) { dbName = dbName.toLowerCase(); @@ -313,7 +313,7 @@ public class Catalog { } throw new IllegalStateException(e); } - + LOG.error(e); LOG.error("Error initializing Catalog. Catalog may be empty."); } @@ -480,7 +480,7 @@ public class Catalog { * Removes a column from the given table. After performing the operation the * table metadata is marked as invalid and will be reloaded on the next access. */ - public void alterTableDropCol(TableName tableName, String colName) + public void alterTableDropCol(TableName tableName, String colName) throws MetaException, InvalidObjectException, org.apache.thrift.TException, DatabaseNotFoundException, TableNotFoundException, ColumnNotFoundException, TableLoadingException { @@ -508,7 +508,7 @@ public class Catalog { * Renames an existing table. After renaming the table, the metadata is marked as * invalid and will be reloaded on the next access. */ - public void alterTableRename(TableName tableName, TableName newTableName) + public void alterTableRename(TableName tableName, TableName newTableName) throws MetaException, InvalidObjectException, org.apache.thrift.TException, DatabaseNotFoundException, TableNotFoundException, TableLoadingException { synchronized (metastoreDdlLock) { @@ -530,7 +530,7 @@ public class Catalog { * changing the file format the table metadata is marked as invalid and will be reloaded * on the next access. */ - public void alterTableSetFileFormat(TableName tableName, + public void alterTableSetFileFormat(TableName tableName, List partitionSpec, FileFormat fileFormat) throws MetaException, InvalidObjectException, org.apache.thrift.TException, DatabaseNotFoundException, PartitionNotFoundException, TableNotFoundException, TableLoadingException { @@ -538,14 +538,14 @@ public class Catalog { if (partitionSpec == null) { synchronized (metastoreDdlLock) { org.apache.hadoop.hive.metastore.api.Table msTbl = getMetaStoreTable(tableName); - setStorageDescriptorFileFormat(msTbl.getSd(), fileFormat); + setStorageDescriptorFileFormat(msTbl.getSd(), fileFormat); applyAlterTable(msTbl); } } else { synchronized (metastoreDdlLock) { HdfsPartition partition = getHdfsPartition( tableName.getDb(), tableName.getTbl(), partitionSpec); - org.apache.hadoop.hive.metastore.api.Partition msPartition = + org.apache.hadoop.hive.metastore.api.Partition msPartition = partition.getMetaStorePartition(); Preconditions.checkNotNull(msPartition); setStorageDescriptorFileFormat(msPartition.getSd(), fileFormat); @@ -556,7 +556,7 @@ public class Catalog { /** * Helper method for setting the file format on a given storage descriptor. - */ + */ private void setStorageDescriptorFileFormat(StorageDescriptor sd, FileFormat fileFormat) { StorageDescriptor tempSd = @@ -571,7 +571,7 @@ public class Catalog { * Changes the HDFS storage location for the given table. This is a metadata only * operation, existing table data will not be as part of changing the location. */ - public void alterTableSetLocation(TableName tableName, + public void alterTableSetLocation(TableName tableName, List partitionSpec, String location) throws MetaException, InvalidObjectException, org.apache.thrift.TException, DatabaseNotFoundException, PartitionNotFoundException, TableNotFoundException, TableLoadingException { @@ -586,7 +586,7 @@ public class Catalog { synchronized (metastoreDdlLock) { HdfsPartition partition = getHdfsPartition(tableName.getDb(), tableName.getTbl(), partitionSpec); - org.apache.hadoop.hive.metastore.api.Partition msPartition = + org.apache.hadoop.hive.metastore.api.Partition msPartition = partition.getMetaStorePartition(); Preconditions.checkNotNull(msPartition); msPartition.getSd().setLocation(location); @@ -600,12 +600,12 @@ public class Catalog { * metastoreDdlLock before calling this method. * Note: The metastore interface is not very safe because it only accepts a * an entire metastore.api.Table object rather than a delta of what to change. This - * means an external modification to the table could be overwritten by an ALTER TABLE + * means an external modification to the table could be overwritten by an ALTER TABLE * command if the metadata is not completely in-sync. This affects both Hive and * Impala, but is more important in Impala because the metadata is cached for a * longer period of time. */ - private void applyAlterTable(org.apache.hadoop.hive.metastore.api.Table msTbl) + private void applyAlterTable(org.apache.hadoop.hive.metastore.api.Table msTbl) throws MetaException, InvalidObjectException, org.apache.thrift.TException { MetaStoreClient msClient = getMetaStoreClient(); try { @@ -617,7 +617,7 @@ public class Catalog { } } - private void applyAlterPartition(TableName tableName, + private void applyAlterPartition(TableName tableName, org.apache.hadoop.hive.metastore.api.Partition msPartition) throws MetaException, InvalidObjectException, org.apache.thrift.TException { MetaStoreClient msClient = getMetaStoreClient(); @@ -725,7 +725,7 @@ public class Catalog { * @param tableName - Fully qualified name of the new table. * @param column - List of column definitions for the new table. * @param partitionColumn - List of partition column definitions for the new table. - * @param isExternal + * @param isExternal * If true, table is created as external which means the data will not be deleted * if dropped. External tables can also be created on top of existing data. * @param comment - Optional comment to attach to the table (null for no comment). @@ -774,29 +774,33 @@ public class Catalog { } LOG.info(String.format("Creating table %s", tableName)); - createTable(tbl, ifNotExists); + createTable(tbl, ifNotExists); } /** * Creates a new table in the metastore based on the definition of an existing table. * No data is copied as part of this process, it is a metadata only operation. If the * creation succeeds, an entry is added to the metadata cache to lazily load the new - * table's metadata on the next access. + * table's metadata on the next access. * * @param tableName - Fully qualified name of the new table. * @param srcTableName - Fully qualified name of the old table. - * @param isExternal + * @param isExternal * If true, table is created as external which means the data will not be deleted * if dropped. External tables can also be created on top of existing data. - * @param comment - Optional comment to attach to the table (null for no comment). + * @param comment - Optional comment to attach to the table or an empty string for no + comment. Null to copy comment from the source table. + * @param fileFormat - The file format for the new table or null to copy file format + * from source table. * @param location - Hdfs path to use as the location for table data or null to use * default location. * @param ifNotExists - If true, no errors are thrown if the table already exists */ public void createTableLike(TableName tableName, TableName srcTableName, - boolean isExternal, String location, boolean ifNotExists) throws MetaException, - NoSuchObjectException, AlreadyExistsException, InvalidObjectException, - org.apache.thrift.TException, ImpalaException, TableLoadingException { + boolean isExternal, String comment, FileFormat fileFormat, String location, + boolean ifNotExists) throws MetaException, NoSuchObjectException, + AlreadyExistsException, InvalidObjectException, org.apache.thrift.TException, + ImpalaException, TableLoadingException { checkTableNameFullyQualified(tableName); checkTableNameFullyQualified(srcTableName); if (ifNotExists && containsTable(tableName.getDb(), tableName.getTbl())) { @@ -812,22 +816,28 @@ public class Catalog { if (tbl.getParameters() == null) { tbl.setParameters(new HashMap()); } + if (comment != null) { + tbl.getParameters().put("comment", comment); + } // The EXTERNAL table property should not be copied from the old table. if (isExternal) { tbl.setTableType(TableType.EXTERNAL_TABLE.toString()); tbl.putToParameters("EXTERNAL", "TRUE"); - } else { + } else { tbl.setTableType(TableType.MANAGED_TABLE.toString()); if (tbl.getParameters().containsKey("EXTERNAL")) { - tbl.getParameters().remove("EXTERNAL"); + tbl.getParameters().remove("EXTERNAL"); } } // The LOCATION property should not be copied from the old table. If the location // is null (the caller didn't specify a custom location) this will clear the value - // and the table will use the default table location from the parent database. + // and the table will use the default table location from the parent database. tbl.getSd().setLocation(location); + if (fileFormat != null) { + setStorageDescriptorFileFormat(tbl.getSd(), fileFormat); + } LOG.info(String.format("Creating table %s LIKE %s", tableName, srcTableName)); - createTable(tbl, ifNotExists); + createTable(tbl, ifNotExists); } private void createTable(org.apache.hadoop.hive.metastore.api.Table newTable, @@ -938,7 +948,7 @@ public class Catalog { * If matchPattern is null, all strings are considered to match. If it is the * empty string, no strings match. */ - private List filterStringsByPattern(Iterable candidates, + private List filterStringsByPattern(Iterable candidates, String matchPattern) { List filtered = Lists.newArrayList(); if (matchPattern == null) { @@ -948,13 +958,13 @@ public class Catalog { // Hive ignores pretty much all metacharacters, so we have to escape them. final String metaCharacters = "+?.^()]\\/{}"; final Pattern regex = Pattern.compile("([" + Pattern.quote(metaCharacters) + "])"); - + for (String pattern: Arrays.asList(matchPattern.split("\\|"))) { Matcher matcher = regex.matcher(pattern); pattern = matcher.replaceAll("\\\\$1").replace("*", ".*"); patterns.add(pattern); } - + for (String candidate: candidates) { for (String pattern: patterns) { // Empty string matches nothing in Hive's implementation @@ -1043,7 +1053,7 @@ public class Catalog { /** * Returns a deep copy of the metastore.api.Table object for the given TableName. - */ + */ private org.apache.hadoop.hive.metastore.api.Table getMetaStoreTable( TableName tableName) throws DatabaseNotFoundException, TableNotFoundException, TableLoadingException { diff --git a/fe/src/main/java/com/cloudera/impala/service/Frontend.java b/fe/src/main/java/com/cloudera/impala/service/Frontend.java index eb8e2aea0..f1c9fc4fc 100644 --- a/fe/src/main/java/com/cloudera/impala/service/Frontend.java +++ b/fe/src/main/java/com/cloudera/impala/service/Frontend.java @@ -269,11 +269,11 @@ public class Frontend { * Creates a new table in the metastore. */ public void createTableLike(TableName tableName, TableName oldTableName, - boolean isExternal,String location, boolean ifNotExists) - throws MetaException, NoSuchObjectException, org.apache.thrift.TException, - AlreadyExistsException, InvalidObjectException, ImpalaException, - TableLoadingException { - catalog.createTableLike(tableName, oldTableName, isExternal, + boolean isExternal, String comment, FileFormat fileFormat, String location, + boolean ifNotExists) throws MetaException, NoSuchObjectException, + org.apache.thrift.TException, AlreadyExistsException, InvalidObjectException, + ImpalaException, TableLoadingException { + catalog.createTableLike(tableName, oldTableName, isExternal, comment, fileFormat, location, ifNotExists); } diff --git a/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java b/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java index 37819fea1..885d6c5e4 100644 --- a/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java +++ b/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java @@ -227,9 +227,18 @@ public class JniFrontend { TableLoadingException { TCreateTableLikeParams params = new TCreateTableLikeParams(); deserializeThrift(params, thriftCreateTableLikeParams); + FileFormat fileFormat = null; + if (params.isSetFile_format()) { + fileFormat = FileFormat.fromThrift(params.getFile_format()); + } + String comment = null; + if (params.isSetComment()) { + comment = params.getComment(); + } frontend.createTableLike(TableName.fromThrift(params.getTable_name()), TableName.fromThrift(params.getSrc_table_name()), - params.isIs_external(), params.getLocation(), params.isIf_not_exists()); + params.isIs_external(), comment, fileFormat, params.getLocation(), + params.isIf_not_exists()); } public void dropDatabase(byte[] thriftDropDbParams) diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java index 9cc03d262..46006249c 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java @@ -689,7 +689,7 @@ public class ParserTest { private void testInsert(boolean overwrite, boolean use_kw_table) { String qualifier = overwrite ? "overwrite" : "into"; qualifier = use_kw_table ? qualifier + " table" : qualifier; - + // Entire unpartitioned table. ParsesOk("insert " + qualifier + " t select a from src where b > 5"); // Static partition with one partitioning key. @@ -926,7 +926,7 @@ public class ParserTest { ParserError(String.format("ALTER TABLE CHANGE %s c1 c2 int", kw)); } } - + @Test public void TestAlterTableSet() { // Supported file formats @@ -983,6 +983,13 @@ public class ParserTest { ParsesOk("CREATE TABLE IF NOT EXISTS Bar2 LIKE Bar1"); ParsesOk("CREATE EXTERNAL TABLE IF NOT EXISTS Bar2 LIKE Bar1"); ParsesOk("CREATE EXTERNAL TABLE IF NOT EXISTS Bar2 LIKE Bar1 LOCATION '/a/b'"); + ParsesOk("CREATE TABLE Foo2 LIKE Foo COMMENT 'sdafsdf'"); + ParsesOk("CREATE TABLE Foo2 LIKE Foo COMMENT ''"); + ParsesOk("CREATE TABLE Foo2 LIKE Foo STORED AS PARQUETFILE"); + ParsesOk("CREATE TABLE Foo2 LIKE Foo COMMENT 'tbl' " + + "STORED AS PARQUETFILE LOCATION '/a/b'"); + ParsesOk("CREATE TABLE Foo2 LIKE Foo STORED AS TEXTFILE LOCATION '/a/b'"); + // Table and column names starting with digits. ParsesOk("CREATE TABLE 01_Foo (01_i int, 02_j string)"); @@ -1017,6 +1024,11 @@ public class ParserTest { } ParserError("CREATE TABLE Foo (i int, s string) STORED AS SEQFILE"); ParserError("CREATE TABLE Foo (i int, s string) STORED TEXTFILE"); + ParserError("CREATE TABLE Foo LIKE Bar STORED AS TEXT"); + ParserError("CREATE TABLE Foo LIKE Bar COMMENT"); + ParserError("CREATE TABLE Foo LIKE Bar STORED TEXTFILE"); + ParserError("CREATE TABLE Foo LIKE Bar STORED AS"); + ParserError("CREATE TABLE Foo LIKE Bar LOCATION"); // Row format syntax ParsesOk("CREATE TABLE T (i int) ROW FORMAT DELIMITED"); @@ -1064,7 +1076,6 @@ public class ParserTest { // Invalid syntax ParserError("CREATE TABLE IF EXISTS Foo.Bar (i int)"); ParserError("CREATE TABLE Bar LIKE Bar2 (i int)"); - ParserError("CREATE TABLE Bar LIKE Bar2 STORED AS TEXTFILE"); ParserError("CREATE IF NOT EXISTS TABLE Foo.Bar (i int)"); ParserError("CREATE TABLE Foo (d double) STORED TEXTFILE"); ParserError("CREATE TABLE Foo (d double) AS TEXTFILE"); diff --git a/testdata/workloads/functional-query/queries/QueryTest/create.test b/testdata/workloads/functional-query/queries/QueryTest/create.test index 2685e0032..d6fda569f 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/create.test +++ b/testdata/workloads/functional-query/queries/QueryTest/create.test @@ -167,7 +167,8 @@ string ==== ---- QUERY # CREATE TABLE LIKE on partitioned table -create table alltypes_test like functional.alltypes +create table alltypes_test like functional_seq_snap.alltypes +stored as parquetfile ---- RESULTS ==== ---- QUERY @@ -178,10 +179,11 @@ select count(*) from alltypes_test ---- TYPES BIGINT ==== ----- QUERY +---- QUERY +# Should be able to insert into this table insert overwrite table alltypes_test partition (year=2009, month=4) -select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, +select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col from functional.alltypes where year=2009 and month=4 ---- RESULTS @@ -196,6 +198,36 @@ select count(*) from alltypes_test BIGINT ==== ---- QUERY +# Show this was actually a different file format (parquet) by trying to +# change the format to textfile and reading the results. +alter table alltypes_test partition (year=2009, month=4) +set fileformat textfile +---- RESULTS +==== +---- QUERY +select count(*) from alltypes_test +---- RESULTS +127 +---- TYPES +BIGINT +==== +---- QUERY +# This should copy the file format from the source table (rc) +create external table jointbl_like like functional_rc_gzip.jointbl +location '/test-warehouse/jointbl_rc_gzip' +---- RESULTS +==== +---- QUERY +# should get some results back +select * from jointbl_like order by test_id limit 3 +---- RESULTS +1001,'Name1',94611,5000 +1002,'Name2',94611,5000 +1003,'Name3',94611,5000 +---- TYPES +bigint,string,int,int +==== +---- QUERY # CREATE TABLE LIKE on unpartitioned table. create table testtbl_like like testtbl ---- RESULTS @@ -249,6 +281,10 @@ drop table alltypes_test ---- RESULTS ==== ---- QUERY +drop table jointbl_like +---- RESULTS +==== +---- QUERY drop table testtbl_like ---- RESULTS ====