jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity
Files
9d112dae23870b6729473047da94f1bc0ea89ceb
impala/testdata/workloads/functional-query/queries/QueryTest/lineage.test
Zoltan Borok-Nagy d19f895c62 IMPALA-12237: Add information about the table type in the lineage log 
Apache Atlas needs table type information to correctly build the lineage
graph. This patch set adds a new field to the metadata of the lineage
graph vertices: 'tableType'.

Table type can be:
 * hive
 * iceberg
 * kudu
 * hbase
 * view
 * virtual
 * external-datasource

Tests:
 * updated current tests with the new field
 * added new tests focusing on Iceberg

Change-Id: I13aeb256ff6b1d0e3c2eb43f7f75513ffc2cd20e
Reviewed-on: http://gerrit.cloudera.org:8080/20120
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
2023-06-30 23:27:31 +00:00

6793 lines
147 KiB
Plaintext
Raw Blame History

====
---- QUERY
drop database if exists lineage_test_db cascade
====
---- QUERY
create database lineage_test_db
====
---- QUERY
create table lineage_test_db.alltypes like functional_hbase.alltypes
====
---- QUERY
create table lineage_test_db.alltypesnopart like functional.alltypesnopart
====
---- QUERY
create table lineage_test_db.alltypessmall like functional.alltypessmall
====
---- QUERY
create table lineage_test_db.alltypesinsert like functional.alltypesinsert
====
---- QUERY
create table lineage_test_db.iceberg_partitioned like functional_parquet.iceberg_partitioned
====
---- LINEAGE
{
"hash": "d4c603c171ccad77991279cc4d643625",
"timestamp": 1687533491,
"vertices": [],
"edges": [],
"queryId": "7547f4882e03cdde:941c4bad00000000",
"user": "anurag",
"queryText": "create table lineage_test_db.foo (id int)",
"endTime": 1687533491
}
---- QUERY
# Test lineage is created for DDLs.
create table lineage_test_db.foo (id int)
====
---- LINEAGE
{
"hash": "ce2d65d4f720533b497fad12de070058",
"timestamp": 1687533496,
"vertices": [],
"edges": [],
"queryId": "554879b21cb9ae72:5e564c9200000000",
"user": "anurag",
"queryText": "create external table lineage_test_db.ext_tbl_loc (id int) location '/test-warehouse/lineage_test_db.db/'",
"endTime": 1687533496,
"tableLocation": "hdfs://localhost:20500/test-warehouse/lineage_test_db.db"
}
---- QUERY
# Test location is populated for create external table with location specified.
create external table lineage_test_db.ext_tbl_loc (id int) location
'/test-warehouse/lineage_test_db.db/'
====
---- LINEAGE
{
"hash": "8a07e4a2c8adb0ec4b11276bc0914848",
"timestamp": 1687533501,
"vertices": [],
"edges": [],
"queryId": "1948dbf64fdfe0c2:2cbbe10700000000",
"user": "anurag",
"queryText": "create external table lineage_test_db.ext_tbl (id int)",
"endTime": 1687533501,
"tableLocation": "hdfs://localhost:20500/test-warehouse/lineage_test_db.db/ext_tbl"
}
---- QUERY
# Test location is populated for create external table when location is not specified.
create external table lineage_test_db.ext_tbl (id int)
====
---- LINEAGE
{
"hash": "43a8b8b9355ae5b918c880af6f94bb3e",
"timestamp": 1687533506,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.bigint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.bool_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.date_string_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.double_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.float_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.int_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 14,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.month",
"id": 15,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 16,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.smallint_col",
"id": 17,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 18,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.string_col",
"id": 19,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 20,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.timestamp_col",
"id": 21,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 22,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.tinyint_col",
"id": 23,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 24,
"metadata": {
"tableCreateTime": 1687533507,
"tableName": "lineage_test_db.ext_ctas",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.alltypes.year",
"id": 25,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
9
],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
11
],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [
13
],
"targets": [
12
],
"edgeType": "PROJECTION"
},
{
"sources": [
15
],
"targets": [
14
],
"edgeType": "PROJECTION"
},
{
"sources": [
17
],
"targets": [
16
],
"edgeType": "PROJECTION"
},
{
"sources": [
19
],
"targets": [
18
],
"edgeType": "PROJECTION"
},
{
"sources": [
21
],
"targets": [
20
],
"edgeType": "PROJECTION"
},
{
"sources": [
23
],
"targets": [
22
],
"edgeType": "PROJECTION"
},
{
"sources": [
25
],
"targets": [
24
],
"edgeType": "PROJECTION"
}
],
"queryId": "c0450321c0d12cbe:30419c8200000000",
"user": "anurag",
"queryText": "create external table lineage_test_db.ext_ctas as select * from lineage_test_db.alltypes",
"endTime": 1687533508,
"tableLocation": "hdfs://localhost:20500/test-warehouse/lineage_test_db.db/ext_ctas"
}
---- QUERY
# Test lineage is populate for create external table as select.
create external table lineage_test_db.ext_ctas as select * from lineage_test_db.alltypes
====
---- QUERY
create view lineage_test_db.alltypes_view as select * from lineage_test_db.alltypes
====
---- LINEAGE
{
"hash": "3968bd65781e9e856eaca799f4501513",
"timestamp": 1687533513,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "x",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1,
2,
3
],
"targets": [
0
],
"edgeType": "PROJECTION"
}
],
"queryId": "8e4d008c58bff1cc:1eb556a200000000",
"user": "anurag",
"queryText": "select * from ( select tinyint_col + int_col x from functional.alltypes union all select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2",
"endTime": 1687533513
}
---- QUERY
# Simple query to introduce projection dependencies. Shows resolution
# through inline views, unions, and materialization points.
select * from (
select tinyint_col + int_col x from functional.alltypes
union all
select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2
====
---- LINEAGE
{
"hash": "0158d41c97236fb9d6b56f5f6fe3d2e6",
"timestamp": 1687533518,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "sum(a.tinyint_col) OVER(...)",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 2,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.smallint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "count(b.string_col)",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.string_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 6
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.timestamp_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.year",
"id": 9,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.bigint_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.float_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.id",
"id": 12,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
2,
3
],
"targets": [
0
],
"edgeType": "PREDICATE"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
2,
3,
5,
7,
8,
9,
10,
11,
12
],
"targets": [
0,
4,
6
],
"edgeType": "PREDICATE"
}
],
"queryId": "374d7d4cfcda6f00:11963b1000000000",
"user": "anurag",
"queryText": "select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), count(b.string_col), b.timestamp_col from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) where a.year = 2010 and b.float_col > 0 group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col having count(a.int_col) > 10 order by b.bigint_col limit 10",
"endTime": 1687533519
}
---- QUERY
# Simple query to introduce predicate dependencies. Shows conjuncts in WHERE, ON, and
# HAVING clause, as well dependencies introduced by GROUP BY and ORDER BY clauses and
# analytic functions.
select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),
count(b.string_col), b.timestamp_col
from functional.alltypes a join functional.alltypessmall b on (a.id = b.id)
where a.year = 2010 and b.float_col > 0
group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col
having count(a.int_col) > 10
order by b.bigint_col limit 10
====
---- LINEAGE
{
"hash": "ad70ea47a7772f9d59c60ebd4e9c1ac8",
"timestamp": 1687533524,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 0,
"metadata": {
"tableCreateTime": 1687533524,
"tableName": "lineage_test_db.lineage_test_tbl1",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687533524,
"tableName": "lineage_test_db.lineage_test_tbl1",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
}
],
"queryId": "5341037bc766b0c5:5b9b5c6a00000000",
"user": "anurag",
"queryText": "create table lineage_test_db.lineage_test_tbl1 as select int_col, tinyint_col from functional.alltypes",
"endTime": 1687533524
}
---- QUERY
# CTAS queries
create table lineage_test_db.lineage_test_tbl1 as select int_col, tinyint_col from functional.alltypes
====
---- LINEAGE
{
"hash": "3d7db666c62b0c19ad8e0dd87fa628b0",
"timestamp": 1687533529,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 0,
"metadata": {
"tableCreateTime": 1687533529,
"tableName": "lineage_test_db.lineage_test_tbl2",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687533529,
"tableName": "lineage_test_db.lineage_test_tbl2",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.string_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 4,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.year",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.id",
"id": 6,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.month",
"id": 7,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
4,
5,
6,
7
],
"targets": [
0,
2
],
"edgeType": "PREDICATE"
}
],
"queryId": "2545984dd298803d:532b0b6100000000",
"user": "anurag",
"queryText": "create table lineage_test_db.lineage_test_tbl2 as select distinct a.int_col, a.string_col from functional.alltypes a inner join functional.alltypessmall b on (a.id = b.id) where a.year = 2009 and b.month = 2",
"endTime": 1687533530
}
---- QUERY
create table lineage_test_db.lineage_test_tbl2 as
select distinct a.int_col, a.string_col from functional.alltypes a
inner join functional.alltypessmall b on (a.id = b.id)
where a.year = 2009 and b.month = 2
====
---- LINEAGE
{
"hash": "04f29ab8b0aec034def6173b0c36c588",
"timestamp": 1687533535,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 0,
"metadata": {
"tableCreateTime": 1687533535,
"tableName": "lineage_test_db.lineage_test_tbl3",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
}
],
"queryId": "e741f71c0e666916:abefa06a00000000",
"user": "anurag",
"queryText": "create table lineage_test_db.lineage_test_tbl3 as select * from (select * from (select int_col from functional.alltypestiny limit 1) v1 ) v2",
"endTime": 1687533535
}
---- QUERY
create table lineage_test_db.lineage_test_tbl3 as
select * from
(select * from
(select int_col from functional.alltypestiny limit 1) v1 ) v2
====
---- LINEAGE
{
"hash": "07232504cc67447abf639eed1c6c2411",
"timestamp": 1687533540,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.bigint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.bool_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.date_string_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.double_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.float_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.int_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 14,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.month",
"id": 15,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 16,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.smallint_col",
"id": 17,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 18,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.string_col",
"id": 19,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 20,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.timestamp_col",
"id": 21,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 22,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.tinyint_col",
"id": 23,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 24,
"metadata": {
"tableCreateTime": 1687533540,
"tableName": "lineage_test_db.lineage_test_tbl4",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypes.year",
"id": 25,
"metadata": {
"tableCreateTime": 1687531612,
"tableName": "functional_hbase.alltypes",
"tableType": "hbase"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
9
],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
11
],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [
13
],
"targets": [
12
],
"edgeType": "PROJECTION"
},
{
"sources": [
15
],
"targets": [
14
],
"edgeType": "PROJECTION"
},
{
"sources": [
17
],
"targets": [
16
],
"edgeType": "PROJECTION"
},
{
"sources": [
19
],
"targets": [
18
],
"edgeType": "PROJECTION"
},
{
"sources": [
21
],
"targets": [
20
],
"edgeType": "PROJECTION"
},
{
"sources": [
23
],
"targets": [
22
],
"edgeType": "PROJECTION"
},
{
"sources": [
25
],
"targets": [
24
],
"edgeType": "PROJECTION"
}
],
"queryId": "a940f90c620a7259:2137273d00000000",
"user": "anurag",
"queryText": "create table lineage_test_db.lineage_test_tbl4 as select * from functional_hbase.alltypes limit 5",
"endTime": 1687533540
}
---- QUERY
# CTAS from HBase table
create table lineage_test_db.lineage_test_tbl4 as select * from functional_hbase.alltypes limit 5
====
---- LINEAGE
{
"hash": "861e5f5d91ce0e9e23db6e7f7f451fac",
"timestamp": 1687533545,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 7,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 12,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypes",
"tableType": "hbase"
}
}
],
"edges": [
{
"sources": [],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
1
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
3
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
5
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
12
],
"edgeType": "PROJECTION"
}
],
"queryId": "664e64cfb1d3fd09:ef8dd84600000000",
"user": "anurag",
"queryText": "insert into lineage_test_db.alltypes values (1, 1, true, \"1999-12-01\", 2.0, 1.0, 1, 12, 2, \"abs\", cast(now() as timestamp), 1, 1999)",
"endTime": 1687533546
}
---- QUERY
# Insert into an HBase table
insert into
lineage_test_db.alltypes
values (1, 1, true, "1999-12-01", 2.0, 1.0, 1, 12, 2, "abs",
cast(now() as timestamp), 1, 1999)
====
---- LINEAGE
{
"hash": "8b1ddbc13665ae532868598c025d1668",
"timestamp": 1687533551,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bool_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypesnopart",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.timestamp_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
5
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [
13
],
"targets": [
12
],
"edgeType": "PROJECTION"
}
],
"queryId": "784e59087868d497:418a666800000000",
"user": "anurag",
"queryText": "insert into table lineage_test_db.alltypesnopart (id, bool_col, timestamp_col) select id, bool_col, timestamp_col from functional.alltypes",
"endTime": 1687533551
}
---- QUERY
insert into table lineage_test_db.alltypesnopart (id, bool_col, timestamp_col)
select id, bool_col, timestamp_col
from functional.alltypes
====
---- LINEAGE
{
"hash": "e7c1630d4593c64d0372e32c28484667",
"timestamp": 1687533556,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 0,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 1,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 2,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.smallint_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 14,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.month",
"id": 15,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.year",
"id": 16,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
1
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
3
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
5
],
"edgeType": "PROJECTION"
},
{
"sources": [
8
],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
12
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
13
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
14
],
"edgeType": "PROJECTION"
},
{
"sources": [
15,
16
],
"targets": [
0,
1,
2,
3,
4,
5,
7,
9,
10,
11,
12,
13,
14
],
"edgeType": "PREDICATE"
}
],
"queryId": "50459cec26bebcb8:6d71e38a00000000",
"user": "anurag",
"queryText": "insert into table lineage_test_db.alltypessmall (smallint_col, int_col) partition (year=2009, month=04) select smallint_col, int_col from functional.alltypes where year=2009 and month=05",
"endTime": 1687533556
}
---- QUERY
insert into table lineage_test_db.alltypessmall (smallint_col, int_col)
partition (year=2009, month=04)
select smallint_col, int_col
from functional.alltypes
where year=2009 and month=05
====
---- LINEAGE
{
"hash": "727d89dd58676c0d36995a90e625607e",
"timestamp": 1687533561,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 0,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_seq_snap.alltypes.year",
"id": 1,
"metadata": {
"tableCreateTime": 1687531272,
"tableName": "functional_seq_snap.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 2,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_seq_snap.alltypes.month",
"id": 3,
"metadata": {
"tableCreateTime": 1687531272,
"tableName": "functional_seq_snap.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 4,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_seq_snap.alltypes.id",
"id": 5,
"metadata": {
"tableCreateTime": 1687531272,
"tableName": "functional_seq_snap.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_seq_snap.alltypes.int_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687531272,
"tableName": "functional_seq_snap.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 14,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 15,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_seq_snap.alltypes.string_col",
"id": 16,
"metadata": {
"tableCreateTime": 1687531272,
"tableName": "functional_seq_snap.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 17,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
10
],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
12
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
13
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
14
],
"edgeType": "PROJECTION"
},
{
"sources": [
16
],
"targets": [
15
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
17
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3
],
"targets": [
0,
2,
4,
6,
7,
8,
9,
11,
12,
13,
14,
15,
17
],
"edgeType": "PREDICATE"
}
],
"queryId": "ca4e2ac78776624d:6e44f11c00000000",
"user": "anurag",
"queryText": "insert into table lineage_test_db.alltypessmall (id, string_col, int_col) partition (year, month) select id, string_col, int_col, year, month from functional_seq_snap.alltypes where year=2009 and month>10",
"endTime": 1687533562
}
---- QUERY
insert into table lineage_test_db.alltypessmall (id, string_col, int_col)
partition (year, month)
select id, string_col, int_col, year, month
from functional_seq_snap.alltypes
where year=2009 and month>10
====
---- LINEAGE
{
"hash": "516fa0d094aca2d64042484a84e19208",
"timestamp": 1687533567,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 0,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 1,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.month",
"id": 2,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 3,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 4,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bool_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.smallint_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 14,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 15,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.float_col",
"id": 16,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 17,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.double_col",
"id": 18,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 19,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.date_string_col",
"id": 20,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 21,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.string_col",
"id": 22,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 23,
"metadata": {
"tableCreateTime": 1687533490,
"tableName": "lineage_test_db.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.timestamp_col",
"id": 24,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.year",
"id": 25,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
2
],
"targets": [
1
],
"edgeType": "PROJECTION"
},
{
"sources": [
4
],
"targets": [
3
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
5
],
"edgeType": "PROJECTION"
},
{
"sources": [
8
],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [
10
],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [
12
],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [
14
],
"targets": [
13
],
"edgeType": "PROJECTION"
},
{
"sources": [
16
],
"targets": [
15
],
"edgeType": "PROJECTION"
},
{
"sources": [
18
],
"targets": [
17
],
"edgeType": "PROJECTION"
},
{
"sources": [
20
],
"targets": [
19
],
"edgeType": "PROJECTION"
},
{
"sources": [
22
],
"targets": [
21
],
"edgeType": "PROJECTION"
},
{
"sources": [
24
],
"targets": [
23
],
"edgeType": "PROJECTION"
},
{
"sources": [
2,
4,
25
],
"targets": [
0,
1,
3,
5,
7,
9,
11,
13,
15,
17,
19,
21,
23
],
"edgeType": "PREDICATE"
}
],
"queryId": "e64b728b24bbd5e9:d50607e600000000",
"user": "anurag",
"queryText": "insert into table lineage_test_db.alltypessmall partition (year=2009, month) select min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col), min(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col), min(timestamp_col), month from functional.alltypes where year=2009 and month>10 group by month having min(id) > 10",
"endTime": 1687533567
}
---- QUERY
insert into table lineage_test_db.alltypessmall
partition (year=2009, month)
select min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col),
min(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col),
min(timestamp_col), month
from functional.alltypes
where year=2009 and month>10
group by month
having min(id) > 10
====
---- LINEAGE
{
"hash": "c82935c5165916f6c56db043120b1b45",
"timestamp": 1687533572,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "max(tinyint_col) OVER(...)",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
2
],
"targets": [
0
],
"edgeType": "PREDICATE"
}
],
"queryId": "b4403b6c2b81f788:b8af2f7700000000",
"user": "anurag",
"queryText": "select max(tinyint_col) over(partition by int_col) from functional.alltypes group by int_col, tinyint_col",
"endTime": 1687533573
}
---- QUERY
# Select statements with analytic functions
select
max(tinyint_col) over(partition by int_col)
from functional.alltypes
group by int_col, tinyint_col
====
---- LINEAGE
{
"hash": "6c646de10e62d1155b5216ae8589ff6d",
"timestamp": 1687533578,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "rank() OVER(...)",
"id": 2
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
1
],
"targets": [
2
],
"edgeType": "PREDICATE"
}
],
"queryId": "fb4794254fb75b20:d1daf37500000000",
"user": "anurag",
"queryText": "select int_col, rank() over(order by int_col) from functional.alltypesagg",
"endTime": 1687533578
}
---- QUERY
select int_col, rank() over(order by int_col) from functional.alltypesagg
====
---- LINEAGE
{
"hash": "3c1677d7fcf4b2db27e84f30f5b3e96f",
"timestamp": 1687533583,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "count(a.double_col) OVER(...)",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.double_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.id",
"id": 7,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3
],
"targets": [
4
],
"edgeType": "PREDICATE"
},
{
"sources": [
6,
7
],
"targets": [
0,
2,
4
],
"edgeType": "PREDICATE"
}
],
"queryId": "8d4ddc6897bb8c9f:2f295b1700000000",
"user": "anurag",
"queryText": "select a.tinyint_col, a.int_col, count(a.double_col) over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following) from functional.alltypes a inner join functional.alltypessmall b on a.id = b.id order by a.tinyint_col, a.int_col",
"endTime": 1687533583
}
---- QUERY
select a.tinyint_col, a.int_col, count(a.double_col)
over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following)
from functional.alltypes a inner join functional.alltypessmall b on a.id = b.id
order by a.tinyint_col, a.int_col
====
---- LINEAGE
{
"hash": "547ead4bd734c7e6720ba5df3f7da88c",
"timestamp": 1687533588,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.double_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "b",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bool_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a + b",
"id": 7
},
{
"vertexType": "COLUMN",
"vertexId": "double_col + a",
"id": 8
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
3,
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
4
],
"edgeType": "PREDICATE"
},
{
"sources": [
3,
5
],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
7
],
"edgeType": "PREDICATE"
},
{
"sources": [
1,
3
],
"targets": [
8
],
"edgeType": "PROJECTION"
}
],
"queryId": "c14107654ce903b7:9bd3363200000000",
"user": "anurag",
"queryText": "with v2 as (select double_col, count(int_col) over() a, sum(int_col + bigint_col) over(partition by bool_col) b from (select * from functional.alltypes) v1) select double_col, a, b, a + b, double_col + a from v2 order by 2, 3, 4",
"endTime": 1687533589
}
---- QUERY
with v2 as
(select
double_col,
count(int_col) over() a,
sum(int_col + bigint_col) over(partition by bool_col) b
from
(select * from functional.alltypes) v1)
select double_col, a, b, a + b, double_col + a from v2
order by 2, 3, 4
====
---- LINEAGE
{
"hash": "952f8a7042ff661ed34fbc8167349899",
"timestamp": 1687533594,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.double_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "b",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bool_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a + b",
"id": 7
},
{
"vertexType": "COLUMN",
"vertexId": "double_col + a",
"id": 8
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
3,
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
4
],
"edgeType": "PREDICATE"
},
{
"sources": [
3,
5
],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
7
],
"edgeType": "PREDICATE"
},
{
"sources": [
1,
3
],
"targets": [
8
],
"edgeType": "PROJECTION"
}
],
"queryId": "1744e089e7cc73a9:ed50bff000000000",
"user": "anurag",
"queryText": "select double_col, a, b, a + b, double_col + a from (select double_col, count(int_col) over() a, sum(int_col + bigint_col) over(partition by bool_col) b from (select * from functional.alltypes) v1) v2 order by 2, 3, 4",
"endTime": 1687533594
}
---- QUERY
select double_col, a, b, a + b, double_col + a from
(select
double_col,
count(int_col) over() a,
sum(int_col + bigint_col) over(partition by bool_col) b
from
(select * from functional.alltypes) v1) v2
order by 2, 3, 4
====
---- LINEAGE
{
"hash": "7b677a184cce15efd15f3033b9b2feeb",
"timestamp": 1687533599,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.month",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.year",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.int_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.month",
"id": 6,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
6
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
6
],
"targets": [
0,
2,
4
],
"edgeType": "PREDICATE"
}
],
"queryId": "4740735360ac58fe:543457eb00000000",
"user": "anurag",
"queryText": "select a.month, a.year, b.int_col, b.month from (select year, month from functional.alltypes union all select year, month from functional.alltypes) a inner join functional.alltypessmall b on (a.month = b.month) where b.month = 1",
"endTime": 1687533599
}
---- QUERY
# Union statements
select a.month, a.year, b.int_col, b.month
from
(select year, month from functional.alltypes
union all
select year, month from functional.alltypes) a
inner join
functional.alltypessmall b
on (a.month = b.month)
where b.month = 1
====
---- LINEAGE
{
"hash": "2f2aab8dd7a5ca2112311dc4bc676e5d",
"timestamp": 1687533604,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.month",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "t2.int_col + 1",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.month",
"id": 6,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3,
5,
6
],
"targets": [
0,
2,
4
],
"edgeType": "PREDICATE"
}
],
"queryId": "704c9e102ed9744a:97d7530700000000",
"user": "anurag",
"queryText": "select t1.int_col, t2.month, t2.int_col + 1 from ( select int_col, count(*) from functional.alltypessmall where month = 1 group by int_col having count(*) > 1 order by count(*) desc limit 5 ) t1 join functional.alltypes t2 on (t1.int_col = t2.int_col) where month = 1",
"endTime": 1687533604
}
---- QUERY
# Select statements with inline views
select t1.int_col, t2.month, t2.int_col + 1
from (
select int_col, count(*)
from functional.alltypessmall
where month = 1
group by int_col
having count(*) > 1
order by count(*) desc limit 5
) t1
join functional.alltypes t2 on (t1.int_col = t2.int_col)
where month = 1
====
---- LINEAGE
{
"hash": "d86f371388b5a5024bca2b7fc487d8f7",
"timestamp": 1687533609,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.smallint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.id",
"id": 3,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.tinyint_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 6
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.int_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 8
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.float_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 10
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.string_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.day",
"id": 12,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.month",
"id": 13,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
9
],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
11
],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3,
5,
7,
9,
11,
12,
13
],
"targets": [
0,
2,
4,
6,
8,
10
],
"edgeType": "PREDICATE"
}
],
"queryId": "d5413148c4203ab5:24a6dc2f00000000",
"user": "anurag",
"queryText": "select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col from functional.alltypessmall c join ( select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, a.int_col int_col, a.month month, b.float_col float_col, b.id id from ( select * from functional.alltypesagg a where month=1 ) a join functional.alltypessmall b on (a.smallint_col = b.id) ) x on (x.tinyint_col = c.id) where x.day=1 and x.int_col > 899 and x.float_col > 4.5 and c.string_col < '7' and x.int_col + x.float_col + cast(c.string_col as float) < 1000",
"endTime": 1687533610
}
---- QUERY
select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col
from functional.alltypessmall c
join (
select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day,
a.int_col int_col, a.month month, b.float_col float_col, b.id id
from ( select * from functional.alltypesagg a where month=1 ) a
join functional.alltypessmall b on (a.smallint_col = b.id)
) x on (x.tinyint_col = c.id)
where x.day=1
and x.int_col > 899
and x.float_col > 4.5
and c.string_col < '7'
and x.int_col + x.float_col + cast(c.string_col as float) < 1000
====
---- LINEAGE
{
"hash": "38cda699ddffba2fa5bab14c8a9c4fb4",
"timestamp": 1687533615,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "c1",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypessmall.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531613,
"tableName": "functional_hbase.alltypessmall",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "c2",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypessmall.float_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531613,
"tableName": "functional_hbase.alltypessmall",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "c3",
"id": 4
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3
],
"targets": [
0,
2,
4
],
"edgeType": "PREDICATE"
}
],
"queryId": "d94c43f20a44f454:87b6d0d300000000",
"user": "anurag",
"queryText": "select c1, c2, c3 from (select c1, c2, c3 from (select int_col c1, sum(float_col) c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1) x order by 2,3 desc limit 5 ) y",
"endTime": 1687533616
}
---- QUERY
select c1, c2, c3
from
(select c1, c2, c3
from
(select int_col c1, sum(float_col) c2, min(float_col) c3
from functional_hbase.alltypessmall
group by 1) x
order by 2,3 desc
limit 5
) y
====
---- LINEAGE
{
"hash": "4e0d48d11ef88837d0267a212fb67716",
"timestamp": 1687533621,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "c1",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypessmall.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531613,
"tableName": "functional_hbase.alltypessmall",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "x2",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypessmall.tinyint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531613,
"tableName": "functional_hbase.alltypessmall",
"tableType": "hbase"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3
],
"targets": [
0,
2
],
"edgeType": "PREDICATE"
}
],
"queryId": "674320549904c2a7:e85c13d400000000",
"user": "anurag",
"queryText": "select c1, x2 from ( select c1, min(c2) x2 from ( select c1, c2, c3 from ( select int_col c1, tinyint_col c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1, 2 order by 1,2 limit 1 ) x ) x2 group by c1 ) y order by 2,1 desc limit 0",
"endTime": 1687533621
}
---- QUERY
select c1, x2
from (
select c1, min(c2) x2
from (
select c1, c2, c3
from (
select int_col c1, tinyint_col c2, min(float_col) c3
from functional_hbase.alltypessmall
group by 1, 2
order by 1,2
limit 1
) x
) x2
group by c1
) y
order by 2,1 desc
limit 0
====
---- LINEAGE
{
"hash": "073a1917cdbbb2b53ac89c62d0852270",
"timestamp": 1687533626,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
}
],
"queryId": "1f45c1b8f43f6b96:a19ff31600000000",
"user": "anurag",
"queryText": "select id from functional.view_view",
"endTime": 1687533626
}
---- QUERY
# Views
select id from functional.view_view
====
---- LINEAGE
{
"hash": "b0ebbaee6fd5fcc500e03aa8dbb08707",
"timestamp": 1687533631,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PREDICATE"
}
],
"queryId": "b7476b57060dceb7:b59a484600000000",
"user": "anurag",
"queryText": "select t.id from (select id from functional.alltypes_view) t where t.id < 10",
"endTime": 1687533631
}
---- QUERY
select t.id from (select id from functional.alltypes_view) t
where t.id < 10
====
---- LINEAGE
{
"hash": "70664c8c26f6a2da7c7f53c4b97b8892",
"timestamp": 1687533636,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.string_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.float_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bool_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.id",
"id": 7,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
6,
7
],
"targets": [
0,
2,
4
],
"edgeType": "PREDICATE"
}
],
"queryId": "09489cc02e60feec:c8ddfb0b00000000",
"user": "anurag",
"queryText": "select string_col, float_col, bool_col from functional.alltypes where id in (select id from functional.alltypesagg)",
"endTime": 1687533637
}
---- QUERY
# Subqueries
select string_col, float_col, bool_col
from functional.alltypes
where id in
(select id from functional.alltypesagg)
====
---- LINEAGE
{
"hash": "97267ebbbeae23fa3bb8ddeb8375e7b9",
"timestamp": 1687533642,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "1",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.tinyint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.tinyint_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
2
],
"targets": [
0
],
"edgeType": "PREDICATE"
}
],
"queryId": "684366081fd04903:ce9b652700000000",
"user": "anurag",
"queryText": "select 1 from functional.alltypesagg a where exists (select id, count(int_col) over (partition by bool_col) from functional.alltypestiny b where a.tinyint_col = b.tinyint_col group by id, int_col, bool_col) and tinyint_col < 10",
"endTime": 1687533642
}
---- QUERY
select 1
from functional.alltypesagg a
where exists
(select id, count(int_col) over (partition by bool_col)
from functional.alltypestiny b
where a.tinyint_col = b.tinyint_col
group by id, int_col, bool_col)
and tinyint_col < 10
====
---- LINEAGE
{
"hash": "b59cc1541d8583e3f1359e9f48c0963d",
"timestamp": 1687533647,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "int_col + 1",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col - 1",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.bool_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.int_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
4,
5,
6
],
"targets": [
0,
2
],
"edgeType": "PREDICATE"
}
],
"queryId": "bc4aa55c595aa11f:b69f692600000000",
"user": "anurag",
"queryText": "select int_col + 1, tinyint_col - 1 from functional.alltypes a where a.int_col < (select max(int_col) from functional.alltypesagg g where g.bool_col = true) and a.bigint_col > 10",
"endTime": 1687533647
}
---- QUERY
select int_col + 1, tinyint_col - 1
from functional.alltypes a
where a.int_col <
(select max(int_col) from functional.alltypesagg g where g.bool_col = true)
and a.bigint_col > 10
====
---- LINEAGE
{
"hash": "ff421b2c2b249764028350613be74fb8",
"timestamp": 1687533652,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "x",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "y",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
}
],
"queryId": "a54406d43e4870fb:cf7baf2100000000",
"user": "anurag",
"queryText": "with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t",
"endTime": 1687533652
}
---- QUERY
# With clause
with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t
====
---- LINEAGE
{
"hash": "b2519388c17a48c91e5bb32948bce1c0",
"timestamp": 1687533657,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 0,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.year",
"id": 1,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 2,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.month",
"id": 3,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 4,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.id",
"id": 5,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bool_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "tinyint_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "smallint_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "int_col",
"id": 9,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.int_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "bigint_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "float_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "double_col",
"id": 13,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "date_string_col",
"id": 14,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 15,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.string_col",
"id": 16,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 17,
"metadata": {
"tableCreateTime": 1687533491,
"tableName": "lineage_test_db.alltypesinsert",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
10
],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
12
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
13
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
14
],
"edgeType": "PROJECTION"
},
{
"sources": [
16
],
"targets": [
15
],
"edgeType": "PROJECTION"
},
{
"sources": [],
"targets": [
17
],
"edgeType": "PROJECTION"
}
],
"queryId": "7d4e988e98626451:c0be63a200000000",
"user": "anurag",
"queryText": "with t1 as (select * from functional.alltypestiny) insert into lineage_test_db.alltypesinsert (id, int_col, string_col) partition(year, month) select id, int_col, string_col, year, month from t1",
"endTime": 1687533658
}
---- QUERY
with t1 as (select * from functional.alltypestiny)
insert into lineage_test_db.alltypesinsert (id, int_col, string_col) partition(year, month)
select id, int_col, string_col, year, month from t1
====
---- LINEAGE
{
"hash": "b4d1a3120fe9c4e63e9a50e2bcf0ff22",
"timestamp": 1687533663,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "lead(a, 1, NULL) OVER(...)",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bigint_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.bool_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.string_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
2,
3,
4,
5,
6
],
"targets": [
0
],
"edgeType": "PREDICATE"
}
],
"queryId": "cf494d06aa9e201a:7047aec600000000",
"user": "anurag",
"queryText": "select lead(a) over (partition by b order by c) from (select lead(id) over (partition by int_col order by bigint_col) as a, max(id) over (partition by tinyint_col order by int_col) as b, min(int_col) over (partition by string_col order by bool_col) as c from functional.alltypes) v",
"endTime": 1687533663
}
---- QUERY
# Nested analytic functions
select lead(a) over (partition by b order by c)
from
(select lead(id) over (partition by int_col order by bigint_col) as a,
max(id) over (partition by tinyint_col order by int_col) as b,
min(int_col) over (partition by string_col order by bool_col) as c
from functional.alltypes) v
====
---- LINEAGE
{
"hash": "bc19b44ea5cab4531beb8a1dc0779df6",
"timestamp": 1687533668,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533668,
"tableName": "lineage_test_db.lineage_test_view1",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
}
],
"queryId": "574dce023537d84c:5de5f32b00000000",
"user": "anurag",
"queryText": "create view lineage_test_db.lineage_test_view1 as select id from functional.alltypestiny",
"endTime": 1687533668
}
---- QUERY
# Create view statement (IMPALA-4219)
create view lineage_test_db.lineage_test_view1 as select id from functional.alltypestiny
====
---- LINEAGE
{
"hash": "4e9912a672b341c75a234945ac3a63d8",
"timestamp": 1687533673,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "a",
"id": 0,
"metadata": {
"tableCreateTime": 1687533673,
"tableName": "lineage_test_db.lineage_test_view2",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypessmall.int_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531613,
"tableName": "functional_hbase.alltypessmall",
"tableType": "hbase"
}
},
{
"vertexType": "COLUMN",
"vertexId": "b",
"id": 2,
"metadata": {
"tableCreateTime": 1687533673,
"tableName": "lineage_test_db.lineage_test_view2",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_hbase.alltypessmall.tinyint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531613,
"tableName": "functional_hbase.alltypessmall",
"tableType": "hbase"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3
],
"targets": [
0,
2
],
"edgeType": "PREDICATE"
}
],
"queryId": "b54cc29caca2cf8a:99ff97ae00000000",
"user": "anurag",
"queryText": "create view lineage_test_db.lineage_test_view2 (a, b) as select c1, x2 from ( select c1, min(c2) x2 from ( select c1, c2, c3 from ( select int_col c1, tinyint_col c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1, 2 order by 1,2 limit 1 ) x ) x2 group by c1 ) y order by 2,1 desc limit 0",
"endTime": 1687533673
}
---- QUERY
# Create view with nested inline views (IMPALA-4219)
create view lineage_test_db.lineage_test_view2 (a, b) as select c1, x2
from (
select c1, min(c2) x2
from (
select c1, c2, c3
from (
select int_col c1, tinyint_col c2, min(float_col) c3
from functional_hbase.alltypessmall
group by 1, 2
order by 1,2
limit 1
) x
) x2
group by c1
) y
order by 2,1 desc
limit 0
====
---- LINEAGE
{
"hash": "fb45884e3b8668e1239a677983ffc706",
"timestamp": 1687533678,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "a1",
"id": 0,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.smallint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a2",
"id": 2,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.id",
"id": 3,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a3",
"id": 4,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.tinyint_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a4",
"id": 6,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a5",
"id": 7,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.int_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a6",
"id": 9,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.float_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "a7",
"id": 11,
"metadata": {
"tableCreateTime": 1687533679,
"tableName": "lineage_test_db.lineage_test_view3",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.string_col",
"id": 12,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.day",
"id": 13,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypesagg.month",
"id": 14,
"metadata": {
"tableCreateTime": 1687531275,
"tableName": "functional.alltypesagg",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
8
],
"targets": [
7
],
"edgeType": "PROJECTION"
},
{
"sources": [
10
],
"targets": [
9
],
"edgeType": "PROJECTION"
},
{
"sources": [
12
],
"targets": [
11
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
3,
5,
8,
10,
12,
13,
14
],
"targets": [
0,
2,
4,
6,
7,
9,
11
],
"edgeType": "PREDICATE"
}
],
"queryId": "2a43cfbaea01dda0:c08ec40600000000",
"user": "anurag",
"queryText": "create view lineage_test_db.lineage_test_view3 (a1, a2, a3, a4, a5, a6, a7) as select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col from functional.alltypessmall c join ( select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, a.int_col int_col, a.month month, b.float_col float_col, b.id id from ( select * from functional.alltypesagg a where month=1 ) a join functional.alltypessmall b on (a.smallint_col = b.id) ) x on (x.tinyint_col = c.id) where x.day=1 and x.int_col > 899 and x.float_col > 4.5 and c.string_col < '7' and x.int_col + x.float_col + cast(c.string_col as float) < 1000",
"endTime": 1687533679
}
---- QUERY
# Create view with complex select statement (IMPALA-4219)
create view lineage_test_db.lineage_test_view3 (a1, a2, a3, a4, a5, a6, a7) as
select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col
from functional.alltypessmall c
join (
select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day,
a.int_col int_col, a.month month, b.float_col float_col, b.id id
from ( select * from functional.alltypesagg a where month=1 ) a
join functional.alltypessmall b on (a.smallint_col = b.id)
) x on (x.tinyint_col = c.id)
where x.day=1
and x.int_col > 899
and x.float_col > 4.5
and c.string_col < '7'
and x.int_col + x.float_col + cast(c.string_col as float) < 1000
====
---- LINEAGE
{
"hash": "51e5187935df610408fd670895640190",
"timestamp": 1687533684,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "_c0",
"id": 0,
"metadata": {
"tableCreateTime": 1687533684,
"tableName": "lineage_test_db.lineage_test_view4",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.tinyint_col",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 2,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.smallint_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "_c1",
"id": 4,
"metadata": {
"tableCreateTime": 1687533684,
"tableName": "lineage_test_db.lineage_test_view4",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.string_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 6,
"metadata": {
"tableCreateTime": 1687533684,
"tableName": "lineage_test_db.lineage_test_view4",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.timestamp_col",
"id": 7,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.int_col",
"id": 8,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.year",
"id": 9,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.bigint_col",
"id": 10,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.float_col",
"id": 11,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypessmall.id",
"id": 12,
"metadata": {
"tableCreateTime": 1687531269,
"tableName": "functional.alltypessmall",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
2,
3
],
"targets": [
0
],
"edgeType": "PREDICATE"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
2,
3,
5,
7,
8,
9,
10,
11,
12
],
"targets": [
0,
4,
6
],
"edgeType": "PREDICATE"
}
],
"queryId": "d448e01e4832881e:59ee2b4a00000000",
"user": "anurag",
"queryText": "create view lineage_test_db.lineage_test_view4 as select * from ( select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), count(b.string_col), b.timestamp_col from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) where a.year = 2010 and b.float_col > 0 group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col having count(a.int_col) > 10 order by b.bigint_col limit 10) t",
"endTime": 1687533684
}
---- QUERY
# Create view with select statement containing an analytic function (IMPALA-4219)
create view lineage_test_db.lineage_test_view4 as
select * from (
select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),
count(b.string_col), b.timestamp_col
from functional.alltypes a join functional.alltypessmall b on (a.id = b.id)
where a.year = 2010 and b.float_col > 0
group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col
having count(a.int_col) > 10
order by b.bigint_col limit 10) t
====
---- LINEAGE
{
"hash": "47f1c5e1dc11a2764f9ed27f9f2ef1ce",
"timestamp": 1687533689,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533513,
"tableName": "lineage_test_db.alltypes_view",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypestiny.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531270,
"tableName": "functional.alltypestiny",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
}
],
"queryId": "b54e1dc4a678c946:eb23e54700000000",
"user": "anurag",
"queryText": "alter view lineage_test_db.alltypes_view as select id from functional.alltypestiny",
"endTime": 1687533689
}
---- QUERY
# Alter view statement (IMPALA-4219)
alter view lineage_test_db.alltypes_view as select id from functional.alltypestiny
====
---- LINEAGE
{
"hash": "b187fe7a4a6b47249f3084d3890b2c74",
"timestamp": 1687533694,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "x",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.complex_struct_col.f1",
"id": 1,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_struct_col.f1",
"id": 2,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_struct_col.f2",
"id": 3,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.nested_struct_col.f2.f12.f21",
"id": 4,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.year",
"id": 5,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1,
2,
3
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
4,
5
],
"targets": [
0
],
"edgeType": "PREDICATE"
}
],
"queryId": "ee4f63ccf2181e85:2b38e8dd00000000",
"user": "anurag",
"queryText": "select * from ( select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes where year = 2000 order by nested_struct_col.f2.f12.f21 limit 10 union all select sum(f1) y from (select complex_struct_col.f1 f1 from functional.allcomplextypes group by 1) v1) v2",
"endTime": 1687533694
}
---- QUERY
# Test references to struct fields.
select * from (
select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes
where year = 2000
order by nested_struct_col.f2.f12.f21 limit 10
union all
select sum(f1) y from
(select complex_struct_col.f1 f1 from functional.allcomplextypes
group by 1) v1) v2
====
---- LINEAGE
{
"hash": "43303560d034d3c1b8fa72962da76f62",
"timestamp": 1687533699,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "item",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_array_col.item",
"id": 1,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "key",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.key",
"id": 3,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "f1",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.value.f1",
"id": 5,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "f2",
"id": 6
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.value.f2",
"id": 7,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
5
],
"targets": [
0,
2,
4,
6
],
"edgeType": "PREDICATE"
}
],
"queryId": "394a9bd88d04af98:9452981700000000",
"user": "anurag",
"queryText": "select * from functional.allcomplextypes.int_array_col a inner join functional.allcomplextypes.struct_map_col m on (a.item = m.f1)",
"endTime": 1687533700
}
---- QUERY
# Test absolute collection table refs. Ensure that the vertex ids of sources are
# always an absolute explicit path.
select * from functional.allcomplextypes.int_array_col a inner join
functional.allcomplextypes.struct_map_col m on (a.item = m.f1)
====
---- LINEAGE
{
"hash": "d34d7d50b599df3912e4265f15706810",
"timestamp": 1687533705,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "year",
"id": 2
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.year",
"id": 3,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "month",
"id": 4
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.month",
"id": 5,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "item",
"id": 6
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_array_col.item",
"id": 7,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "key",
"id": 8
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.key",
"id": 9,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "f1",
"id": 10
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.value.f1",
"id": 11,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "f2",
"id": 12
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.value.f2",
"id": 13,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_array_col",
"id": 14,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col",
"id": 15,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
9
],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
11
],
"targets": [
10
],
"edgeType": "PROJECTION"
},
{
"sources": [
13
],
"targets": [
12
],
"edgeType": "PROJECTION"
},
{
"sources": [
7,
11,
14,
15
],
"targets": [
0,
2,
4,
6,
8,
10,
12
],
"edgeType": "PREDICATE"
}
],
"queryId": "b043e925908e15a2:1a5b1afe00000000",
"user": "anurag",
"queryText": "select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m where a.item = m.f1",
"endTime": 1687533705
}
---- QUERY
# Test relative collection table refs. Ensure that the vertex ids of sources are
# always an absolute explicit path.
select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m
where a.item = m.f1
====
---- LINEAGE
{
"hash": "e1684998c49aeacac122c78e7b47867f",
"timestamp": 1687533710,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "ab",
"id": 0
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_array_col.item",
"id": 1,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.value.f1",
"id": 2,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "c",
"id": 3
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.map_map_col.value.value",
"id": 4,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.map_map_col.value.key",
"id": 5,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "d",
"id": 6
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.int_map_col.value",
"id": 7,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "e",
"id": 8
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.value.f21",
"id": 9,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.key",
"id": 10,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.allcomplextypes.struct_map_col.key",
"id": 11,
"metadata": {
"tableCreateTime": 1687531278,
"tableName": "functional.allcomplextypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1,
2
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
4
],
"targets": [
3
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
3
],
"edgeType": "PREDICATE"
},
{
"sources": [
1,
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
},
{
"sources": [
9
],
"targets": [
8
],
"edgeType": "PROJECTION"
},
{
"sources": [
1,
10,
11
],
"targets": [
0,
3,
6,
8
],
"edgeType": "PREDICATE"
}
],
"queryId": "04414cb7de8797d4:4caad29500000000",
"user": "anurag",
"queryText": "select a + b as ab, c, d, e from functional.allcomplextypes t, (select sum(item) a from t.int_array_col where item < 10) v1, (select count(f1) b from t.struct_map_col group by key) v2, (select avg(value) over(partition by key) c from t.map_map_col.value) v3, (select item d from t.int_array_col union all select value from t.int_map_col) v4, (select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5",
"endTime": 1687533710
}
---- QUERY
# Test relative collection table refs in inline views.
select a + b as ab, c, d, e from functional.allcomplextypes t,
(select sum(item) a from t.int_array_col
where item < 10) v1,
(select count(f1) b from t.struct_map_col
group by key) v2,
(select avg(value) over(partition by key) c from t.map_map_col.value) v3,
(select item d from t.int_array_col
union all
select value from t.int_map_col) v4,
(select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5
====
---- LINEAGE
{
"hash": "5b43652748f22f00515ddf6ed699aec0",
"timestamp": 1687533715,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687533715,
"tableName": "lineage_test_db.lineage_test_view5",
"tableType": "view"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531267,
"tableName": "functional.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PREDICATE"
}
],
"queryId": "764a4986b0d631c2:a52f087b00000000",
"user": "anurag",
"queryText": "create view lineage_test_db.lineage_test_view5 as select id from functional.alltypes_view v where not exists (select 1 from functional.alltypes a where v.id = a.id)",
"endTime": 1687533715
}
---- QUERY
# IMPALA-4206: Test creating a view whose definition has a subquery and a view reference.
create view lineage_test_db.lineage_test_view5 as
select id from functional.alltypes_view v
where not exists (select 1 from functional.alltypes a where v.id = a.id)
====
---- LINEAGE
{
"hash": "f69515f6a2cfcd95c1914fc7da09834a",
"timestamp": 1687615426,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687615426,
"tableName": "lineage_test_db.ice_ctas",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.alltypes.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531213,
"tableName": "functional_parquet.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "string_col",
"id": 2,
"metadata": {
"tableCreateTime": 1687615426,
"tableName": "lineage_test_db.ice_ctas",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.alltypes.string_col",
"id": 3,
"metadata": {
"tableCreateTime": 1687531213,
"tableName": "functional_parquet.alltypes",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "timestamp_col",
"id": 4,
"metadata": {
"tableCreateTime": 1687615426,
"tableName": "lineage_test_db.ice_ctas",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.alltypes.timestamp_col",
"id": 5,
"metadata": {
"tableCreateTime": 1687531213,
"tableName": "functional_parquet.alltypes",
"tableType": "hive"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
}
],
"queryId": "db47d0dcf9aa4267:6cfbbef700000000",
"user": "boroknagyz",
"queryText": "create table lineage_test_db.ice_ctas partitioned by spec (truncate(1, string_col)) stored by iceberg as select id, string_col, timestamp_col from functional_parquet.alltypes",
"endTime": 1687615427
}
---- QUERY
# Execute lineage tests for Iceberg tables.
create table lineage_test_db.ice_ctas partitioned by spec (truncate(1, string_col)) stored by iceberg as
select id, string_col, timestamp_col from functional_parquet.alltypes
====
---- LINEAGE
{
"hash": "c37e1110bfc63dfe2e3e4c132a9def2f",
"timestamp": 1687615432,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "user",
"id": 2,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.user",
"id": 3,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "action",
"id": 4,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.action",
"id": 5,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "event_time",
"id": 6,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.event_time",
"id": 7,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
}
],
"queryId": "1942b41abdcfba11:1d8e573d00000000",
"user": "boroknagyz",
"queryText": "insert into lineage_test_db.iceberg_partitioned select * from functional_parquet.iceberg_partitioned",
"endTime": 1687615432
}
---- QUERY
insert into lineage_test_db.iceberg_partitioned
select * from functional_parquet.iceberg_partitioned
====
---- LINEAGE
{
"hash": "c37e1110bfc63dfe2e3e4c132a9def2f",
"timestamp": 1687615437,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "user",
"id": 2,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.user",
"id": 3,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "action",
"id": 4,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.action",
"id": 5,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "event_time",
"id": 6,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
},
{
"vertexType": "COLUMN",
"vertexId": "functional_parquet.iceberg_partitioned.event_time",
"id": 7,
"metadata": {
"tableCreateTime": 1687531274,
"tableName": "functional_parquet.iceberg_partitioned",
"tableType": "iceberg"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
},
{
"sources": [
3
],
"targets": [
2
],
"edgeType": "PROJECTION"
},
{
"sources": [
5
],
"targets": [
4
],
"edgeType": "PROJECTION"
},
{
"sources": [
7
],
"targets": [
6
],
"edgeType": "PROJECTION"
}
],
"queryId": "5e4febca0b4e3922:96171f9700000000",
"user": "boroknagyz",
"queryText": "insert into lineage_test_db.iceberg_partitioned select * from functional_parquet.iceberg_partitioned",
"endTime": 1687615438
}
---- QUERY
insert into lineage_test_db.iceberg_partitioned
select * from functional_parquet.iceberg_partitioned
====
---- LINEAGE
{
"hash": "2d727b3181ee6d49a388eec56f315806",
"timestamp": 1687615443,
"vertices": [
{
"vertexType": "COLUMN",
"vertexId": "id",
"id": 0,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.foo",
"tableType": "hive"
}
},
{
"vertexType": "COLUMN",
"vertexId": "lineage_test_db.iceberg_partitioned.id",
"id": 1,
"metadata": {
"tableCreateTime": 1687615420,
"tableName": "lineage_test_db.iceberg_partitioned",
"tableType": "iceberg"
}
}
],
"edges": [
{
"sources": [
1
],
"targets": [
0
],
"edgeType": "PROJECTION"
}
],
"queryId": "46450d3ee417fa15:9c00c0ce00000000",
"user": "boroknagyz",
"queryText": "insert into lineage_test_db.foo select id from lineage_test_db.iceberg_partitioned",
"endTime": 1687615443
}
---- QUERY
insert into lineage_test_db.foo
select id from lineage_test_db.iceberg_partitioned
====
Reference in New Issue View Git Blame Copy Permalink