mirror of
https://github.com/apache/impala.git
synced 2026-01-22 18:02:34 -05:00
4df03a31ec
Similar to the previous patch, this patch adds UTF-8 support in instr() and locate() builtin functions so they can have consistent behaviors with Hive's. These two string functions both have an optional argument as position: INSTR(STRING str, STRING substr[, BIGINT position[, BIGINT occurrence]]) LOCATE(STRING substr, STRING str[, INT pos]) Their return values are positions of the matched substring. In UTF-8 mode (turned on by set UTF8_MODE=true), these positions are counted by UTF-8 characters instead of bytes. Error handling: Malformed UTF-8 characters are counted as one byte per character. This is consistent with Hive since Hive replaces those bytes to U+FFFD (REPLACEMENT CHARACTER). E.g. GenericUDFInstr calls Text#toString(), which performs the replacement. We can provide more behaviors on error handling like ignoring them or reporting errors. IMPALA-10761 will focus on this. Tests: - Add BE unit tests and e2e tests - Add random tests to make sure malformed UTF-8 characters won't crash us. Change-Id: Ic13c3d04649c1aea56c1aaa464799b5e4674f662 Reviewed-on: http://gerrit.cloudera.org:8080/17580 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
171 lines
4.1 KiB
Plaintext
171 lines
4.1 KiB
Plaintext
====
|
|
---- QUERY
|
|
set utf8_mode=true;
|
|
select length('你好'), length('你好hello'), length('你好 hello 你好')
|
|
---- RESULTS
|
|
2,7,11
|
|
---- TYPES
|
|
INT,INT,INT
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=false;
|
|
select length('你好'), length('你好hello'), length('你好 hello 你好')
|
|
---- RESULTS
|
|
6,11,19
|
|
---- TYPES
|
|
INT,INT,INT
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=true;
|
|
select substring('你好hello', 1, 3)
|
|
---- RESULTS: RAW_STRING
|
|
'你好h'
|
|
---- TYPES
|
|
STRING
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=false;
|
|
select substring('你好hello', 1, 3)
|
|
---- RESULTS: RAW_STRING
|
|
'你'
|
|
---- TYPES
|
|
STRING
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=true;
|
|
select reverse('你好hello你好');
|
|
---- RESULTS: RAW_STRING
|
|
'好你olleh好你'
|
|
---- TYPES
|
|
STRING
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=off;
|
|
select id, length(name), substring(name, 1, 3), length(substring(name, 1, 3)) from utf8_str_tiny
|
|
---- RESULTS: RAW_STRING
|
|
1,6,'张',3
|
|
2,6,'李',3
|
|
3,6,'王',3
|
|
4,9,'李',3
|
|
5,5,'Ali',3
|
|
6,6,'陈',3
|
|
7,7,'Бo',3
|
|
8,5,'Jö',3
|
|
9,9,'ひ',3
|
|
10,6,'서',3
|
|
---- TYPES
|
|
INT,INT,STRING,INT
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=true;
|
|
select id, length(name), substring(name, 1, 2), reverse(name) from utf8_str_tiny
|
|
---- RESULTS: RAW_STRING
|
|
1,2,'张三','三张'
|
|
2,2,'李四','四李'
|
|
3,2,'王五','五王'
|
|
4,3,'李小','龙小李'
|
|
5,5,'Al','ecilA'
|
|
6,4,'陈B','boB陈'
|
|
7,5,'Бo','cиpoБ'
|
|
8,4,'Jö','gröJ'
|
|
9,3,'ひな','たなひ'
|
|
10,2,'서연','연서'
|
|
---- TYPES
|
|
INT,INT,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
# Test utf8 functions in where clause.
|
|
set utf8_mode=true;
|
|
select id, name from functional.utf8_str_tiny
|
|
where length(name) = 2 and substring(name, 1, 1) = '李';
|
|
---- RESULTS: RAW_STRING
|
|
2,'李四'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
# Test utf8 functions in group by clause. group_concat() may produce undetermined results
|
|
# due to the order. Here we wrap it with length().
|
|
set utf8_mode=true;
|
|
select substring(name, 1, 1), length(group_concat(name)) from functional.utf8_str_tiny
|
|
group by substring(name, 1, 1);
|
|
---- RESULTS: RAW_STRING
|
|
'A',5
|
|
'ひ',3
|
|
'陈',4
|
|
'王',2
|
|
'张',2
|
|
'서',2
|
|
'J',4
|
|
'Б',5
|
|
'李',7
|
|
---- TYPES
|
|
STRING,INT
|
|
====
|
|
---- QUERY
|
|
# Test utf8 functions in group by and having clauses. group_concat() may produce
|
|
# undetermined results due to the order. Here we wrap it with length().
|
|
set utf8_mode=true;
|
|
select substring(name, 1, 1), length(group_concat(name)) from functional.utf8_str_tiny
|
|
group by substring(name, 1, 1)
|
|
having length(group_concat(name)) = 7;
|
|
---- RESULTS: RAW_STRING
|
|
'李',7
|
|
---- TYPES
|
|
STRING,INT
|
|
====
|
|
---- QUERY
|
|
# Each Chinese character is encoded into 3 bytes in UTF-8.
|
|
set utf8_mode=false;
|
|
select instr('最快的SQL引擎跑SQL', 'SQL'),
|
|
instr('最快的SQL引擎跑SQL', '引擎'),
|
|
instr('最快的SQL引擎跑SQL', 'SQL引擎'),
|
|
instr('最快的SQL引擎跑SQL', '跑SQL'),
|
|
instr('最快的SQL引擎跑SQL', 'SQL', 1, 2),
|
|
instr('最快的SQL引擎跑SQL', 'SQL', -1, 2);
|
|
---- RESULTS
|
|
10,13,10,19,22,10
|
|
---- TYPES
|
|
INT,INT,INT,INT,INT,INT
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=true;
|
|
select instr('最快的SQL引擎跑SQL', 'SQL'),
|
|
instr('最快的SQL引擎跑SQL', '引擎'),
|
|
instr('最快的SQL引擎跑SQL', 'SQL引擎'),
|
|
instr('最快的SQL引擎跑SQL', '跑SQL'),
|
|
instr('最快的SQL引擎跑SQL', 'SQL', 1, 2),
|
|
instr('最快的SQL引擎跑SQL', 'SQL', -1, 2);
|
|
---- RESULTS
|
|
4,7,4,9,10,4
|
|
---- TYPES
|
|
INT,INT,INT,INT,INT,INT
|
|
====
|
|
---- QUERY
|
|
# Each Chinese character is encoded into 3 bytes in UTF-8.
|
|
set utf8_mode=false;
|
|
select locate('SQL', '最快的SQL引擎跑SQL'),
|
|
locate('引擎', '最快的SQL引擎跑SQL'),
|
|
locate('SQL引擎', '最快的SQL引擎跑SQL'),
|
|
locate('跑SQL', '最快的SQL引擎跑SQL'),
|
|
locate('SQL', '最快的SQL引擎跑SQL', 4),
|
|
locate('SQL', '最快的SQL引擎跑SQL', 11);
|
|
---- RESULTS
|
|
10,13,10,19,10,22
|
|
---- TYPES
|
|
INT,INT,INT,INT,INT,INT
|
|
====
|
|
---- QUERY
|
|
set utf8_mode=true;
|
|
select locate('SQL', '最快的SQL引擎跑SQL'),
|
|
locate('引擎', '最快的SQL引擎跑SQL'),
|
|
locate('SQL引擎', '最快的SQL引擎跑SQL'),
|
|
locate('跑SQL', '最快的SQL引擎跑SQL'),
|
|
locate('SQL', '最快的SQL引擎跑SQL', 4),
|
|
locate('SQL', '最快的SQL引擎跑SQL', 10);
|
|
---- RESULTS
|
|
4,7,4,9,4,10
|
|
---- TYPES
|
|
INT,INT,INT,INT,INT,INT
|
|
====
|