3ff14ecbb5
* feat(TDI-42591): bigquery dynamic column support * Add helpers for bigquery input components * feat(TDI-42591): bigquery dynamic column support * Add dynamic column support to bigquery input component * feat(TDI-42591): bigquery dynamic column support * Add dynamic column support for bigquery output component * feat(TDI-42591): bigquery dynamic column support * Fix PR comments * feat(TDI-42591): bigquery dynamic column support * Fix fixed component name issue * feat(TDI-42591): bigquery dynamic column support * Add support of geography type * Add support of numeric type * Fix bytes type support * feat(TDI-42591): bigquery dynamic column support * Fix OAuth null objects issue * Fix byte array service account issue * feat(TDI-42591): bigquery dynamic column support * fix compilation error * feat(TDI-42591): bigquery dynamic column support * Exclude record type from table creation for OAuth
319 lines
10 KiB
XML
319 lines
10 KiB
XML
<COMPONENT>
|
|
<HEADER
|
|
PLATEFORM="ALL"
|
|
SERIAL=""
|
|
VERSION="0.102"
|
|
STATUS="ALPHA"
|
|
|
|
COMPATIBILITY="ALL"
|
|
AUTHOR="Talend"
|
|
RELEASE_DATE="20050320A"
|
|
STARTABLE="false"
|
|
PARTITIONING="NONE"
|
|
>
|
|
<SIGNATURE/>
|
|
</HEADER>
|
|
|
|
<FAMILIES>
|
|
<FAMILY>Big Data/Google BigQuery</FAMILY>
|
|
</FAMILIES>
|
|
|
|
<DOCUMENTATION>
|
|
<URL/>
|
|
</DOCUMENTATION>
|
|
|
|
<CONNECTORS>
|
|
<CONNECTOR CTYPE="FLOW" MAX_INPUT="1" MAX_OUTPUT="0"/>
|
|
<CONNECTOR CTYPE="ITERATE" MAX_OUTPUT="1" MAX_INPUT="1"/>
|
|
<CONNECTOR CTYPE="SUBJOB_OK" MAX_INPUT="1" />
|
|
<CONNECTOR CTYPE="SUBJOB_ERROR" MAX_INPUT="1" />
|
|
<CONNECTOR CTYPE="COMPONENT_OK"/>
|
|
<CONNECTOR CTYPE="COMPONENT_ERROR"/>
|
|
<CONNECTOR CTYPE="RUN_IF"/>
|
|
</CONNECTORS>
|
|
|
|
<PARAMETERS>
|
|
<PARAMETER
|
|
NAME="MAPPING"
|
|
FIELD="MAPPING_TYPE"
|
|
NUM_ROW="120"
|
|
SHOW="false"
|
|
>
|
|
<DEFAULT>bigquery_id</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="PROPERTY" FIELD="PROPERTY_TYPE" SHOW="true" NUM_ROW="10" REPOSITORY_VALUE="DELIMITED" />
|
|
|
|
<PARAMETER NAME="FILENAME" FIELD="FILE" NUM_ROW="11" REQUIRED="true" REPOSITORY_VALUE="FILE_PATH">
|
|
<DEFAULT>"__COMP_DEFAULT_FILE_DIR__/biquery_bulk.txt"</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="APPEND" FIELD="CHECK" REQUIRED="true" NUM_ROW="12" REPOSITORY_VALUE="APPEND">
|
|
<DEFAULT>false</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="SCHEMA" FIELD="SCHEMA_TYPE" REQUIRED="true" NUM_ROW="13">
|
|
<DEFAULT/>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="15" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
|
|
<ITEMS DEFAULT="SERVICEACCOUNT">
|
|
<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
|
|
<ITEM NAME="OAUTH" VALUE="OAUTH" />
|
|
</ITEMS>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
|
REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
|
FIELD="FILE"
|
|
NUM_ROW="18"
|
|
REQUIRED="true"
|
|
SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
|
|
GROUP="AUTHENTICATION"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="CLIENT_ID"
|
|
FIELD="TEXT"
|
|
NUM_ROW="20"
|
|
REQUIRED="true"
|
|
SHOW_IF="AUTH_MODE == 'OAUTH'"
|
|
GROUP="AUTHENTICATION"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="CLIENT_SECRET"
|
|
FIELD="PASSWORD"
|
|
NUM_ROW="21"
|
|
REQUIRED="true"
|
|
SHOW_IF="AUTH_MODE == 'OAUTH'"
|
|
GROUP="AUTHENTICATION"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="AUTHORIZATION_CODE"
|
|
FIELD="TEXT"
|
|
NUM_ROW="30"
|
|
REQUIRED="true"
|
|
SHOW_IF="AUTH_MODE == 'OAUTH'"
|
|
GROUP="AUTHENTICATION"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="PROJECT_ID"
|
|
FIELD="TEXT"
|
|
NUM_ROW="40"
|
|
REQUIRED="true"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="DATASET"
|
|
FIELD="TEXT"
|
|
NUM_ROW="49"
|
|
REQUIRED="true"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER
|
|
NAME="TABLE"
|
|
FIELD="TEXT"
|
|
NUM_ROW="50"
|
|
REQUIRED="true"
|
|
>
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="CREATE_TABLE_IF_NOT_EXIST" FIELD="CHECK" NUM_ROW="50" REQUIRED="true">
|
|
<DEFAULT>true</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="ACTION_ON_DATA"
|
|
FIELD="CLOSED_LIST"
|
|
NUM_ROW="51"
|
|
REQUIRED="true"
|
|
>
|
|
<ITEMS DEFAULT="TRUNCATE">
|
|
<ITEM NAME="TRUNCATE" VALUE="TRUNCATE" />
|
|
<ITEM NAME="APPEND" VALUE="APPEND" />
|
|
<ITEM NAME="EMPTY" VALUE="EMPTY" />
|
|
</ITEMS>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="BULK_FILE_ALREADY_EXIST" FIELD="CHECK" NUM_ROW="59" REQUIRED="true" SHOW="false">
|
|
<DEFAULT>false</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="GS_ACCESS_KEY" FIELD="TEXT" NUM_ROW="60" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="GS_SECRET_KEY" FIELD="PASSWORD" NUM_ROW="61" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="BUCKET_NAME" FIELD="TEXT" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
|
|
<DEFAULT>""</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER
|
|
NAME="GS_FILE"
|
|
FIELD="TEXT"
|
|
NUM_ROW="70"
|
|
REQUIRED="true"
|
|
GROUP="GS_CONF"
|
|
>
|
|
<DEFAULT>"gs://"</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER
|
|
NAME="GS_FILE_HEADER"
|
|
FIELD="TEXT"
|
|
NUM_ROW="71"
|
|
REQUIRED="true"
|
|
GROUP="GS_CONF"
|
|
>
|
|
<DEFAULT>0</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="DIE_ON_ERROR" FIELD="CHECK" NUM_ROW="80" REQUIRED="true">
|
|
<DEFAULT>false</DEFAULT>
|
|
</PARAMETER>
|
|
</PARAMETERS>
|
|
|
|
<ADVANCED_PARAMETERS>
|
|
<PARAMETER NAME="TOKEN_NAME" FIELD="FILE" NUM_ROW="10"
|
|
REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
|
|
<DEFAULT>"__COMP_DEFAULT_FILE_DIR__/token.properties"</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="SET_FIELD_DELIMITER" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" SHOW="false">
|
|
<DEFAULT>true</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="FIELD_DELIMITER" FIELD="TEXT" REQUIRED="true" NUM_ROW="20"
|
|
SHOW="false">
|
|
<DEFAULT>","</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="ROWSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="ROW_SEPARATOR" SHOW="false">
|
|
<DEFAULT>"\n"</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="FIELDSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="FIELD_SEPARATOR">
|
|
<DEFAULT>","</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="33" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
|
|
<DEFAULT>false</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="ESCAPE_CHAR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="ESCAPE_CHAR" SHOW="false">
|
|
<DEFAULT>"\\"</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="TEXT_ENCLOSURE" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="TEXT_ENCLOSURE" SHOW="false">
|
|
<DEFAULT>"\""</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="CREATE" FIELD="CHECK" REQUIRED="true" NUM_ROW="40">
|
|
<DEFAULT>true</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER NAME="FLUSHONROW" FIELD="CHECK" REQUIRED="true" NUM_ROW="45">
|
|
<DEFAULT>false</DEFAULT>
|
|
</PARAMETER>
|
|
|
|
<PARAMETER
|
|
NAME="FLUSHONROW_NUM"
|
|
FIELD="TEXT"
|
|
REQUIRED="true"
|
|
NUM_ROW="45"
|
|
SHOW_IF="FLUSHONROW == 'true'"
|
|
>
|
|
<DEFAULT>1</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER
|
|
NAME="RECORDS_CONTAIN_NULL_VALUE"
|
|
FIELD="CHECK"
|
|
SHOW="false"
|
|
NUM_ROW="50"
|
|
>
|
|
<DEFAULT>true</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER
|
|
NAME="CHECK_DISK_SPACE"
|
|
FIELD="CHECK"
|
|
SHOW="true"
|
|
NUM_ROW="50"
|
|
>
|
|
<DEFAULT>false</DEFAULT>
|
|
</PARAMETER>
|
|
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="60" REQUIRED="true" REPOSITORY_VALUE="ENCODING">
|
|
<DEFAULT>"ISO-8859-15"</DEFAULT>
|
|
</PARAMETER>
|
|
</ADVANCED_PARAMETERS>
|
|
|
|
<CODEGENERATION>
|
|
<TEMPLATES INPUT="tBQOB" OUTPUT="tBQBE">
|
|
|
|
<TEMPLATE NAME="tBQOB" COMPONENT="tBigQueryOutputBulk">
|
|
<LINK_TO NAME="tBQBE" CTYPE="ROWS_END"/>
|
|
</TEMPLATE>
|
|
|
|
<TEMPLATE NAME="tBQBE" COMPONENT="tBigQueryBulkExec"/>
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.AUTH_MODE" TARGET="tBQBE.AUTH_MODE" />
|
|
<TEMPLATE_PARAM SOURCE="self.SERVICE_ACCOUNT_CREDENTIALS_FILE" TARGET="tBQBE.SERVICE_ACCOUNT_CREDENTIALS_FILE" />
|
|
<TEMPLATE_PARAM SOURCE="self.CLIENT_ID" TARGET="tBQBE.CLIENT_ID" />
|
|
<TEMPLATE_PARAM SOURCE="self.CLIENT_SECRET" TARGET="tBQBE.CLIENT_SECRET" />
|
|
<TEMPLATE_PARAM SOURCE="self.PROJECT_ID" TARGET="tBQBE.PROJECT_ID" />
|
|
<TEMPLATE_PARAM SOURCE="self.AUTHORIZATION_CODE" TARGET="tBQBE.AUTHORIZATION_CODE"/>
|
|
<TEMPLATE_PARAM SOURCE="self.DATASET" TARGET="tBQBE.DATASET" />
|
|
<TEMPLATE_PARAM SOURCE="self.TABLE" TARGET="tBQBE.TABLE" />
|
|
<TEMPLATE_PARAM SOURCE="self.CREATE_TABLE_IF_NOT_EXIST" TARGET="tBQBE.CREATE_TABLE_IF_NOT_EXIST" />
|
|
<TEMPLATE_PARAM SOURCE="self.ACTION_ON_DATA" TARGET="tBQBE.ACTION_ON_DATA" />
|
|
<TEMPLATE_PARAM SOURCE="self.GS_ACCESS_KEY" TARGET="tBQBE.GS_ACCESS_KEY" />
|
|
<TEMPLATE_PARAM SOURCE="self.GS_SECRET_KEY" TARGET="tBQBE.GS_SECRET_KEY" />
|
|
<TEMPLATE_PARAM SOURCE="self.BUCKET_NAME" TARGET="tBQBE.BUCKET_NAME" />
|
|
<TEMPLATE_PARAM SOURCE="self.GS_FILE" TARGET="tBQBE.GS_FILE" />
|
|
<TEMPLATE_PARAM SOURCE="self.GS_FILE_HEADER" TARGET="tBQBE.GS_FILE_HEADER" />
|
|
<TEMPLATE_PARAM SOURCE="self.TOKEN_NAME" TARGET="tBQBE.TOKEN_NAME" />
|
|
<TEMPLATE_PARAM SOURCE="self.SET_FIELD_DELIMITER" TARGET="tBQBE.SET_FIELD_DELIMITER" />
|
|
<TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR" TARGET="tBQBE.FIELD_DELIMITER" />
|
|
<TEMPLATE_PARAM SOURCE="self.BULK_ALREADY_EXIST" TARGET="tBQBE.BULK_ALREADY_EXIST" />
|
|
<TEMPLATE_PARAM SOURCE="self.DIE_ON_ERROR" TARGET="tBQBE.DIE_ON_ERROR" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQOB.FILENAME" />
|
|
<TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQBE.GS_LOCAL_FILE" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.ROWSEPARATOR" TARGET="tBQOB.ROWSEPARATOR" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR" TARGET="tBQOB.FIELDSEPARATOR" />
|
|
<TEMPLATE_PARAM SOURCE="self.DROP" TARGET="tBQBE.DROP" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.ESCAPE_CHAR" TARGET="tBQOB.ESCAPE_CHAR" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.TEXT_ENCLOSURE" TARGET="tBQOB.TEXT_ENCLOSURE" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.SCHEMA" TARGET="tBQOB.SCHEMA" />
|
|
<TEMPLATE_PARAM SOURCE="self.SCHEMA" TARGET="tBQBE.SCHEMA" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.ENCODING" TARGET="tBQOB.ENCODING" />
|
|
<TEMPLATE_PARAM SOURCE="self.ENCODING" TARGET="tBQBE.ENCODING" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.RECORDS_CONTAIN_NULL_VALUE" TARGET="tBQOB.RECORDS_CONTAIN_NULL_VALUE" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.APPEND" TARGET="tBQOB.APPEND" />
|
|
<TEMPLATE_PARAM SOURCE="self.CREATE" TARGET="tBQOB.CREATE" />
|
|
<TEMPLATE_PARAM SOURCE="self.FLUSHONROW" TARGET="tBQOB.FLUSHONROW" />
|
|
<TEMPLATE_PARAM SOURCE="self.FLUSHONROW_NUM" TARGET="tBQOB.FLUSHONROW_NUM" />
|
|
|
|
<TEMPLATE_PARAM SOURCE="self.DATA_ACTION" TARGET="tBQBE.DATA_ACTION" />
|
|
</TEMPLATES>
|
|
</CODEGENERATION>
|
|
<RETURNS/>
|
|
</COMPONENT>
|