patch(TPS-3225): removed too small total timeout for service account

This fix is intended to include context files in a data service
published artifact. When a dataservice job is published for the first
time and was never build from Studio before, published artifact was
missing the context files.

main/plugins/org.talend.designer.codegen/jet_stub/footer.javajet

@@ -506,7 +506,7 @@
             if (inContext == null) {
                 inContext = <%=className%>.class.getClassLoader().getResourceAsStream("config/contexts/" + contextStr + ".properties");
             }
-            if (inContext != null) {
+            if (inContext != null && context != null && context.isEmpty()) {
                 //defaultProps is in order to keep the original context value
                 defaultProps.load(inContext);
                 inContext.close();

main/plugins/org.talend.designer.codegen/jet_stub/header.javajet

@@ -312,7 +312,7 @@ public <%=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true)%> get<%=Ch
 		}
 %>
 	}
-	private ContextProperties context = new ContextProperties();
+	protected ContextProperties context = new ContextProperties(); // will be instanciated by MS.
 	public ContextProperties getContext() {
 		return this.context;
 	}

main/plugins/org.talend.designer.codegen/src/main/java/messages.properties

@@ -48,6 +48,7 @@ ComponentsFactory.componentNotFound=Component folder not found\:
 ComponentsFactory.load.components=Load Components...
 ComponentsFactory.parentNotRecompiled=Warning\: parents of *.inc.javajet are not recompiled \!
 ComponentsFactory.loadComponents.missingFolder=The folder "{0}" is missing in the plugin "{1}".
+ComponentsFactory.init.waitForFinish.timeout=Can't initialise ComponentsFactory, please try again.
 ComponentsLocalProviderService.componentLoadError=Component load error
 ComponentsLocalProviderService.talendExchangeError=Talend Exchange Error
 ComponentsPreferencePage.ButtonLabel0=Ok 

main/plugins/org.talend.designer.codegen/src/main/java/org/talend/designer/codegen/components/model/ComponentsFactory.java

@@ -30,6 +30,11 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.ResourceBundle;
 import java.util.Set;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Level;
@@ -139,66 +144,119 @@ public class ComponentsFactory implements IComponentsFactory {
     private static boolean cleanDone = false;
 
     protected static Map<String, Map<String, Set<IComponent>>> componentNameMap;
+    
+    private AtomicBoolean isInitialising;
+    
+    private volatile Lock initialiseLock;
 
     public ComponentsFactory() {
+        isInitialising = new AtomicBoolean(false);
+        initialiseLock = new ReentrantLock();
     }
 
     private void init(boolean duringLogon) {
-        removeOldComponentsUserFolder(); // not used anymore
-        long startTime = System.currentTimeMillis();
-
-        // TimeMeasure.display = true;
-        // TimeMeasure.displaySteps = true;
-        // TimeMeasure.measureActive = true;
-        // TimeMeasure.begin("initComponents");
-        componentList = Collections.synchronizedSet(new HashSet<IComponent>());
-        customComponentList = new HashSet<IComponent>();
-        skeletonList = new ArrayList<String>();
-        userComponentList = new HashSet<IComponent>();
-        String installLocation = new Path(Platform.getConfigurationLocation().getURL().getPath()).toFile().getAbsolutePath();
-        componentToProviderMap = new HashMap<IComponent, AbstractComponentsProvider>();
-        boolean isNeedClean = !cleanDone && TalendCacheUtils.isSetCleanComponentCache();
-        cleanDone = true; // only check this parameter one time, or it will reinitialize things all the time...
-        isCreated = hasComponentFile(installLocation) && !isNeedClean;
-        ComponentsCache cache = ComponentManager.getComponentCache();
-        try {
-            if (isCreated) {
-                // if cache is created and empty, means we never loaded it before.
-                // if it was already loaded, then no need to go again, since it's a static variable, it's still in
-                // memory.
-                // it avoids to reload from disk again even more for commandline at each logon, since it's no use.
-                if (cache.getComponentEntryMap().isEmpty()) {
-                    ComponentsCache loadCache = loadComponentResource(installLocation);
-                    cache.getComponentEntryMap().putAll(loadCache.getComponentEntryMap());
-                }
-            } else {
-                cache.getComponentEntryMap().clear();
-            }
-        } catch (IOException e) {
-            ExceptionHandler.process(e);
-            cache.getComponentEntryMap().clear();
-            isCreated = false;
+        if (wait4InitialiseFinish()) {
+            return;
         }
+        try {
+            try {
+                initialiseLock.lock();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            isInitialising.set(true);
+            removeOldComponentsUserFolder(); // not used anymore
+            long startTime = System.currentTimeMillis();
 
-        loadComponentsFromComponentsProviderExtension();
+            // TimeMeasure.display = true;
+            // TimeMeasure.displaySteps = true;
+            // TimeMeasure.measureActive = true;
+            // TimeMeasure.begin("initComponents");
+            componentList = Collections.synchronizedSet(new HashSet<IComponent>());
+            customComponentList = new HashSet<IComponent>();
+            skeletonList = new ArrayList<String>();
+            userComponentList = new HashSet<IComponent>();
+            String installLocation = new Path(Platform.getConfigurationLocation().getURL().getPath()).toFile().getAbsolutePath();
+            componentToProviderMap = new HashMap<IComponent, AbstractComponentsProvider>();
+            boolean isNeedClean = !cleanDone && TalendCacheUtils.isSetCleanComponentCache();
+            cleanDone = true; // only check this parameter one time, or it will reinitialize things all the time...
+            isCreated = hasComponentFile(installLocation) && !isNeedClean;
+            ComponentsCache cache = ComponentManager.getComponentCache();
+            try {
+                if (isCreated) {
+                    // if cache is created and empty, means we never loaded it before.
+                    // if it was already loaded, then no need to go again, since it's a static variable, it's still in
+                    // memory.
+                    // it avoids to reload from disk again even more for commandline at each logon, since it's no use.
+                    if (cache.getComponentEntryMap().isEmpty()) {
+                        ComponentsCache loadCache = loadComponentResource(installLocation);
+                        cache.getComponentEntryMap().putAll(loadCache.getComponentEntryMap());
+                    }
+                } else {
+                    cache.getComponentEntryMap().clear();
+                }
+            } catch (IOException e) {
+                ExceptionHandler.process(e);
+                cache.getComponentEntryMap().clear();
+                isCreated = false;
+            }
 
-        // TimeMeasure.step("initComponents", "loadComponentsFromProvider");
-        // 2.Load Component from extension point: component_definition
-        loadComponentsFromExtensions();
-        // TimeMeasure.step("initComponents", "loadComponentsFromExtension[joblets?]");
+            loadComponentsFromComponentsProviderExtension();
 
-        ComponentManager.saveResource(); // will save only if needed.
+            // TimeMeasure.step("initComponents", "loadComponentsFromProvider");
+            // 2.Load Component from extension point: component_definition
+            loadComponentsFromExtensions();
+            // TimeMeasure.step("initComponents", "loadComponentsFromExtension[joblets?]");
 
-        // init component name map, used to pick specified component immediately
-        initComponentNameMap();
+            ComponentManager.saveResource(); // will save only if needed.
 
-        // TimeMeasure.step("initComponents", "createCache");
-        log.debug(componentList.size() + " components loaded in " + (System.currentTimeMillis() - startTime) + " ms"); //$NON-NLS-1$ //$NON-NLS-2$
+            // init component name map, used to pick specified component immediately
+            initComponentNameMap();
 
-        // TimeMeasure.end("initComponents");
-        // TimeMeasure.display = false;
-        // TimeMeasure.displaySteps = false;
-        // TimeMeasure.measureActive = false;
+            // TimeMeasure.step("initComponents", "createCache");
+            log.debug(componentList.size() + " components loaded in " + (System.currentTimeMillis() - startTime) + " ms"); //$NON-NLS-1$ //$NON-NLS-2$
+
+            // TimeMeasure.end("initComponents");
+            // TimeMeasure.display = false;
+            // TimeMeasure.displaySteps = false;
+            // TimeMeasure.measureActive = false;
+        } finally {
+            isInitialising.set(false);
+            initialiseLock.unlock();
+        }
+    }
+
+    private boolean wait4InitialiseFinish() {
+        if (isInitialising.get()) {
+            try {
+                // wait for 10 min (10 * 60 seconds) by default
+                long timeout = 600L;
+                String timeoutStr = System.getProperty("studio.componentsFactory.init.timeout"); //$NON-NLS-1$
+                if (!StringUtils.isBlank(timeoutStr)) {
+                    try {
+                        timeout = Long.valueOf(timeoutStr);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+                if (initialiseLock.tryLock(timeout, TimeUnit.SECONDS)) {
+                    initialiseLock.unlock();
+                } else {
+                    // may be track in dead lock, throw exception to try to break dead lock
+                    throw new RuntimeException(Messages.getString("ComponentsFactory.init.waitForFinish.timeout")); //$NON-NLS-1$
+                }
+                // initialise successfully or not
+                return !isInitialising.get();
+            } catch (RuntimeException e) {
+                throw e;
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        // initialise failed, still need to initialise
+        return false;
     }
 
     protected void initComponentNameMap() {
@@ -729,7 +787,8 @@ public class ComponentsFactory implements IComponentsFactory {
     }
 
     @Override
-    public synchronized int size() {
+    public int size() {
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(false);
         }
@@ -737,7 +796,8 @@ public class ComponentsFactory implements IComponentsFactory {
     }
 
     @Override
-    public synchronized IComponent get(String name) {
+    public IComponent get(String name) {
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(false);
         }
@@ -757,7 +817,8 @@ public class ComponentsFactory implements IComponentsFactory {
      * @see org.talend.core.model.components.IComponentsFactory#get(java.lang.String, java.lang.String)
      */
     @Override
-    public synchronized IComponent get(String name, String paletteType) {
+    public IComponent get(String name, String paletteType) {
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(false);
         }
@@ -772,6 +833,7 @@ public class ComponentsFactory implements IComponentsFactory {
 
     @Override
     public IComponent getJobletComponent(String name, String paletteType) {
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(false);
         }
@@ -799,6 +861,7 @@ public class ComponentsFactory implements IComponentsFactory {
     @Override
     public void initializeComponents(IProgressMonitor monitor) {
         this.monitor = monitor;
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(false);
         }
@@ -809,6 +872,7 @@ public class ComponentsFactory implements IComponentsFactory {
     @Override
     public void initializeComponents(IProgressMonitor monitor, boolean duringLogon) {
         this.monitor = monitor;
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(duringLogon);
         }
@@ -822,12 +886,18 @@ public class ComponentsFactory implements IComponentsFactory {
      * @see org.talend.core.model.components.IComponentsFactory#getComponents()
      */
     @Override
-    public synchronized Set<IComponent> getComponents() {
+    public Set<IComponent> getComponents() {
+        wait4InitialiseFinish();
         if (componentList == null) {
             init(false);
         }
         return componentList;
     }
+    
+    @Override
+    public boolean isInitialising() {
+        return isInitialising.get();
+    }
 
     @Override
     public Collection<IComponent> readComponents() {
@@ -840,7 +910,8 @@ public class ComponentsFactory implements IComponentsFactory {
     }
 
     @Override
-    public synchronized Map<String, Map<String, Set<IComponent>>> getComponentNameMap() {
+    public Map<String, Map<String, Set<IComponent>>> getComponentNameMap() {
+        wait4InitialiseFinish();
         if (componentNameMap == null) {
             init(false);
         }
@@ -848,7 +919,8 @@ public class ComponentsFactory implements IComponentsFactory {
     }
 
     @Override
-    public synchronized List<IComponent> getCustomComponents() {
+    public List<IComponent> getCustomComponents() {
+        wait4InitialiseFinish();
         if (customComponentList == null) {
             init(false);
         }
@@ -862,6 +934,7 @@ public class ComponentsFactory implements IComponentsFactory {
      */
     @Override
     public List<String> getSkeletons() {
+        wait4InitialiseFinish();
         if (skeletonList == null) {
             init(false);
         }
@@ -900,6 +973,7 @@ public class ComponentsFactory implements IComponentsFactory {
                 jobletService.clearSparkStreamingJobletComponent();
             }
         }
+        isInitialising.set(false);
     }
 
     @Override

main/plugins/org.talend.designer.components.localprovider/components/tBigQueryBulkExec/tBigQueryBulkExec_begin.javajet

@@ -24,7 +24,7 @@
 	String dataset = ElementParameterParser.getValue(node, "__DATASET__");
 	String table = ElementParameterParser.getValue(node, "__TABLE__");
 	String gsFile = ElementParameterParser.getValue(node, "__GS_FILE__");
-
+	String fieldDelimiter = ElementParameterParser.getValue(node, "__FIELD_DELIMITER__");
 	String actionOnData = ElementParameterParser.getValue(node, "__ACTION_ON_DATA__");
 	boolean dieOnError = "true".equals(ElementParameterParser.getValue(node, "__DIE_ON_ERROR__"));
 
@@ -279,24 +279,44 @@
 					IMetadataColumn column = columns.get(i);
 					String columnName = column.getLabel();
 					String typeToGenerate = "string";
-					if("id_Float".equals(column.getTalendType())) {
+					if("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
 						typeToGenerate = "float";
-					} else if("id_Integer".equals(column.getTalendType())) {
+					}else if("id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType()) || "id_Short".equals(column.getTalendType())) {
 						typeToGenerate = "integer";
+					} else if("id_Character".equals(column.getTalendType())) {
+						typeToGenerate = "string";
+					} else if("id_BigDecimal".equals(column.getTalendType())) {
+						typeToGenerate = "numeric";					
 					} else if("id_Boolean".equals(column.getTalendType())) {
 						typeToGenerate = "boolean";
 					} else if("id_Date".equals(column.getTalendType())) {
-						typeToGenerate = "timestamp";
+						String pattern = column.getPattern();
+					    if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
+					        typeToGenerate = "date";
+					    }else if(pattern.length() > 12){
+					        typeToGenerate = "timestamp";
+					    }else{
+					        typeToGenerate = "string";
+					    }
 					}
 					%>
+					<%
+						String modeType = null;
+						if (!column.isNullable()) { 
+							modeType = "REQUIRED";
+						} else {
+							modeType = "NULLABLE";
+						}
+					%>
 					com.google.api.services.bigquery.model.TableFieldSchema <%=columnName%>_<%=cid%> = new com.google.api.services.bigquery.model.TableFieldSchema();
 					<%=columnName%>_<%=cid%>.setName("<%=columnName%>");
 					<%=columnName%>_<%=cid%>.setType("<%=typeToGenerate%>");
+					<%=columnName%>_<%=cid%>.setMode("<%=modeType%>");
 					fields_<%=cid%>.add(<%=columnName%>_<%=cid%>);		
 					<%
 					if(isLog4jEnabled){
 					%>
-						log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\"}");
+						log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\",\"mode\":\"<%=modeType%>\"}");
 					<%
 					}
 				}
@@ -321,7 +341,7 @@
 	
 		if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
 		%>
-			queryLoad_<%=cid%>.setFieldDelimiter(<%=ElementParameterParser.getValue(node, "__FIELD_DELIMITER__")%>);
+			queryLoad_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
 		<%
 		}
 		%>
@@ -336,7 +356,7 @@
 		queryLoad_<%=cid%>.setDestinationTable(destinationTable_<%=cid%>);
 		queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=ElementParameterParser.getValue(node, "__GS_FILE__")%>));
 		queryLoad_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
-	
+		queryLoad_<%=cid%>.setNullMarker("\\N");
 		config_<%=cid%>.setLoad(queryLoad_<%=cid%>);
 		
 		job_<%=cid%>.setConfiguration(config_<%=cid%>);
@@ -375,10 +395,10 @@
 		}
 		if (jobExec_<%=cid%>.getStatus().getState().equals("RUNNING")
 				|| jobExec_<%=cid%>.getStatus().getState().equals("PENDING")) {
-			com.google.api.services.bigquery.model.Job pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).execute();
+            		com.google.api.services.bigquery.model.Job pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).setLocation(jobExec_<%=cid%>.getJobReference().getLocation()).execute();
 			while (pollJob_<%=cid%>.getStatus().getState().equals("RUNNING") || pollJob_<%=cid%>.getStatus().getState().equals("PENDING")) {
 				Thread.sleep(1000);
-				pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).execute();
+				pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).setLocation(jobExec_<%=cid%>.getJobReference().getLocation()).execute();
 				System.out.println(String.format(
 						"Waiting on job %s ... Current status: %s", jobExec_<%=cid%>
 								.getJobReference().getJobId(), pollJob_<%=cid%>
@@ -440,6 +460,7 @@
 		/* ----START-CREATING-JOB (Cloud API)---- */
 		com.google.cloud.bigquery.TableId tableId_<%=cid%> = com.google.cloud.bigquery.TableId.of(<%=projectId%>, <%=dataset%>, <%=table%>);
 		com.google.cloud.bigquery.Table table_<%=cid%> = bigquery_<%=cid%>.getTable(tableId_<%=cid%>);
+		com.google.cloud.bigquery.LoadJobConfiguration.Builder loadJobBuilder_<%=cid%> = com.google.cloud.bigquery.LoadJobConfiguration.newBuilder(tableId_<%=cid%>, <%=gsFile%>);
 		if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> && table_<%=cid%> != null) {
 			boolean deleted = bigquery_<%=cid%>.delete(tableId_<%=cid%>);	
 			if (deleted) {
@@ -454,9 +475,7 @@
 				throw new RuntimeException("Unable to delete table " + tableId_<%=cid%>);
 			}
 		}
-		boolean tableNotExist = table_<%=cid%> == null;
-		if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> || tableNotExist) {
-			boolean typeSupported = true;
+		if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> || <%=ElementParameterParser.getBooleanValue(node, "__CREATE_TABLE_IF_NOT_EXIST__")%>) {
 			java.util.List<com.google.cloud.bigquery.Field> fields_<%=cid%> = new java.util.ArrayList<>();
 			<%
 			List<IMetadataTable> metadatas = node.getMetadataList();
@@ -468,27 +487,37 @@
 					for (int i = 0; i < nbColumns; i++ ) {
 						IMetadataColumn column = columns.get(i);
 						String columnName = column.getLabel();
-						String typeToGenerate = null;
-						if("id_String".equals(column.getTalendType())) {
+						String typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
+						if("id_String".equals(column.getTalendType()) || "id_Character".equals(column.getTalendType())) {
 							typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
 						} else if ("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
 							typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.FLOAT";
 						} else if ("id_Short".equals(column.getTalendType()) || "id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType())) {
 							typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.INTEGER";
+						} else if ("id_BigDecimal".equals(column.getTalendType())) {
+							typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";						
 						} else if ("id_Boolean".equals(column.getTalendType())) {
 							typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN";
-						} else if ("id_Date".equals(column.getTalendType())) {
-							typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP";
-						} else {
-						%>
-							typeSupported = false;
-						<%
+						} else if ("id_Date".equals(column.getTalendType())) {							
+							String pattern = column.getPattern();
+						    if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
+						        typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.DATE";
+						    }else if(pattern.length() > 12){
+						        typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP";
+						    }else{
+						        typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
+						    }
+						}
+						
+						String modeType = "NULLABLE";
+						if (!column.isNullable()) { 
+							modeType = "REQUIRED";
 						}
 						%>
-						if (!typeSupported) {
-							throw new IllegalArgumentException("unsupported type for column [<%=columnName%>]: " + "<%=column.getTalendType()%>");
-						}
-						com.google.cloud.bigquery.Field field_<%=i%> = com.google.cloud.bigquery.Field.of("<%=columnName%>", <%=typeToGenerate%>);
+						
+						com.google.cloud.bigquery.Field field_<%=i%> = com.google.cloud.bigquery.Field.newBuilder("<%=columnName%>", <%=typeToGenerate%>)
+																									  .setMode(com.google.cloud.bigquery.Field.Mode.valueOf("<%=modeType%>"))
+																									  .build();
 						fields_<%=cid%>.add(field_<%=i%>);		
 						<%
 						if(isLog4jEnabled){
@@ -500,12 +529,57 @@
 				}
 			}
 			%>
+			
 			com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
 			com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
 			table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
+			loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
 		}
-		com.google.cloud.bigquery.Job job_<%=cid%> = table_<%=cid%>.load(com.google.cloud.bigquery.FormatOptions.csv(), <%=gsFile%>);
-		job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)), com.google.cloud.RetryOption.totalTimeout(org.threeten.bp.Duration.ofSeconds(30)));
+		
+			<%
+				if("true".equals(ElementParameterParser.getValue(node, "__CREATE_TABLE_IF_NOT_EXIST__"))) {
+			%>
+				loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_IF_NEEDED);
+			<%
+			} else {
+			%>
+				loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
+			<%
+			}
+			%>
+			
+			<%
+			if("APPEND".equals(actionOnData)) {
+			%>
+				loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_APPEND);
+			<%
+			} else if("TRUNCATE".equals(actionOnData)) {
+			%>
+				loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_TRUNCATE);
+				
+			<%
+			}
+		 	else {
+			%>
+				loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_EMPTY);				
+			<%
+			}
+			%>
+		
+		loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
+                com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
+		csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
+																				
+		<%if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
+        %>
+         	loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>).build());
+        <%
+        }
+        %>		
+		
+		loadJobBuilder_<%=cid%>.setNullMarker("\\N");																
+		com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
+		job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
 		if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {
 			<%
 			if(isLog4jEnabled){
@@ -515,7 +589,8 @@
 			}
 			%>
 		} else {
-			throw new RuntimeException("Job failed: " + job_<%=cid%>.getStatus().getError());
+			List<com.google.cloud.bigquery.BigQueryError> errorList = job_<%=cid%>.getStatus().getExecutionErrors();
+			throw new RuntimeException("Job failed: " + errorList.get(errorList.size() - 1));
 		}
 		
 		/* ----END-CREATING-JOB (Cloud API)---- */

main/plugins/org.talend.designer.components.localprovider/components/tBigQueryBulkExec/tBigQueryBulkExec_java.xml

@@ -1,218 +1,217 @@
-<COMPONENT>
-  <HEADER
-    PLATEFORM="ALL"
-    SERIAL=""
-    VERSION="0.102"
-    STATUS="ALPHA"
-    COMPATIBILITY="ALL"
-    AUTHOR="Talend"
-    RELEASE_DATE="20070312A"
-    STARTABLE="true"
-    LOG4J_ENABLED="true"
-  >
-    <SIGNATURE/>
-  </HEADER>
-  <FAMILIES>
-    <FAMILY>Big Data/Google BigQuery</FAMILY>
-  </FAMILIES>
-  <DOCUMENTATION>
-    <URL/>
-  </DOCUMENTATION>
-  <CONNECTORS>
-    <CONNECTOR CTYPE="FLOW" MAX_INPUT="0" MAX_OUTPUT="0"/>
-    <CONNECTOR CTYPE="ITERATE" MAX_OUTPUT="1" MAX_INPUT="1"/>
-    <CONNECTOR CTYPE="SUBJOB_OK" MAX_INPUT="1" />
-    <CONNECTOR CTYPE="SUBJOB_ERROR" MAX_INPUT="1" />
-    <CONNECTOR CTYPE="COMPONENT_OK"/>
-    <CONNECTOR CTYPE="COMPONENT_ERROR"/>
-    <CONNECTOR CTYPE="RUN_IF"/>
-  </CONNECTORS>
-  <PARAMETERS>
-    <PARAMETER
-    	NAME="SCHEMA"
-    	FIELD="SCHEMA_TYPE"
-    	REQUIRED="true"
-    	NUM_ROW="1"
-    />
-
-    <PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="3" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
-			<ITEMS DEFAULT="SERVICEACCOUNT">
-				<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
-				<ITEM NAME="OAUTH" VALUE="OAUTH" />
-			</ITEMS>
-	</PARAMETER>
-	
-	<PARAMETER 
-			NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE" 
-			REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
-			FIELD="FILE" 
-			NUM_ROW="10" 
-			REQUIRED="true" 
-			SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
-			GROUP="AUTHENTICATION"
-			>
-			<DEFAULT>""</DEFAULT>
-	</PARAMETER>
-
-    <PARAMETER
-      NAME="CLIENT_ID"
-      FIELD="TEXT"
-      NUM_ROW="10"
-      REQUIRED="true"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-	  GROUP="AUTHENTICATION"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-	<PARAMETER
-      NAME="CLIENT_SECRET"
-      FIELD="PASSWORD"
-      NUM_ROW="20"
-      REQUIRED="true"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-	  GROUP="AUTHENTICATION"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-	<PARAMETER
-      NAME="AUTHORIZATION_CODE"
-      FIELD="TEXT"
-      NUM_ROW="30"
-      REQUIRED="true"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-	  GROUP="AUTHENTICATION"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-	<PARAMETER
-      NAME="PROJECT_ID"
-      FIELD="TEXT"
-      NUM_ROW="40"
-      REQUIRED="true"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    
-    <PARAMETER
-      NAME="DATASET"
-      FIELD="TEXT"
-      NUM_ROW="49"
-      REQUIRED="true"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="TABLE"
-      FIELD="TEXT"
-      NUM_ROW="50"
-      REQUIRED="true"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="CREATE_TABLE_IF_NOT_EXIST" FIELD="CHECK" NUM_ROW="50" REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
-	<DEFAULT>true</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="ACTION_ON_DATA"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-      FIELD="CLOSED_LIST"
-      NUM_ROW="51"
-      REQUIRED="true"
-    >
-      	<ITEMS DEFAULT="TRUNCATE">
-		<ITEM NAME="TRUNCATE" VALUE="TRUNCATE" />
-		<ITEM NAME="APPEND" VALUE="APPEND" />
-		<ITEM NAME="EMPTY" VALUE="EMPTY" />
-	</ITEMS>
-    </PARAMETER>
-    <PARAMETER NAME="BULK_FILE_ALREADY_EXIST" FIELD="CHECK" NUM_ROW="59" REQUIRED="true" GROUP="GS_CONF">
-	<DEFAULT>false</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="GS_ACCESS_KEY" FIELD="TEXT" NUM_ROW="60" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="GS_SECRET_KEY" FIELD="PASSWORD" NUM_ROW="61" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="GS_LOCAL_FILE" FIELD="FILE" NUM_ROW="62" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="BUCKET_NAME" FIELD="TEXT" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="GS_FILE"
-      FIELD="TEXT"
-      NUM_ROW="70"
-      REQUIRED="true"
-      GROUP="GS_CONF"
-    >
-      <DEFAULT>"gs://"</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="GS_FILE_HEADER"
-      FIELD="TEXT"
-      NUM_ROW="71"
-      REQUIRED="true"
-      GROUP="GS_CONF"
-    >
-      <DEFAULT>0</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="DIE_ON_ERROR" FIELD="CHECK" NUM_ROW="80" REQUIRED="true">
-	<DEFAULT>false</DEFAULT>
-    </PARAMETER>
-   </PARAMETERS>
-   <ADVANCED_PARAMETERS>
-    <PARAMETER NAME="TOKEN_NAME" FIELD="FILE" NUM_ROW="10"
-        REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
-        <DEFAULT>"__COMP_DEFAULT_FILE_DIR__/token.properties"</DEFAULT>
-    </PARAMETER>
-   	<PARAMETER NAME="SET_FIELD_DELIMITER" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" >
-		<DEFAULT>false</DEFAULT>
-	</PARAMETER>
-	<PARAMETER NAME="FIELD_DELIMITER" FIELD="TEXT" REQUIRED="true" NUM_ROW="20"
-		SHOW_IF="(SET_FIELD_DELIMITER == 'true')">
-		<DEFAULT>","</DEFAULT>
-	</PARAMETER>
-    <PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
-      <DEFAULT>false</DEFAULT>
-    </PARAMETER>
-	<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="30"
-		REQUIRED="true" REPOSITORY_VALUE="ENCODING">
-		<DEFAULT>"ISO-8859-15"</DEFAULT>
-	</PARAMETER>
-   </ADVANCED_PARAMETERS>
-   <CODEGENERATION>
-     <IMPORTS>
-		<IMPORT NAME="google-api-client-1.19.0.jar" MODULE="google-api-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-api-client-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-api-client-1.19.0.jar" REQUIRED="true" />
-		<IMPORT NAME="google-api-services-oauth2-v2-rev78-1.19.0.jar" MODULE="google-api-services-oauth2-v2-rev78-1.19.0.jar" MVN="mvn:org.talend.libraries/google-api-services-oauth2-v2-rev78-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-api-services-oauth2-v2-rev78-1.19.0.jar" REQUIRED="true" />
-		<IMPORT NAME="google-api-services-bigquery-v2-rev391-1.21.0.jar" MODULE="google-api-services-bigquery-v2-rev391-1.21.0.jar" MVN="mvn:org.talend.libraries/google-api-services-bigquery-v2-rev391-1.21.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-api-services-bigquery-v2-rev391-1.21.0.jar" REQUIRED="true" />
-		<IMPORT NAME="google-http-client-1.19.0.jar" MODULE="google-http-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-http-client-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-http-client-1.19.0.jar" REQUIRED="true" />
-		<IMPORT NAME="google-oauth-client-1.19.0.jar" MODULE="google-oauth-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-oauth-client-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-oauth-client-1.19.0.jar" REQUIRED="true" />
-		<IMPORT NAME="google-http-client-jackson2-1.19.0.jar" MODULE="google-http-client-jackson2-1.19.0.jar" MVN="mvn:org.talend.libraries/google-http-client-jackson2-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-http-client-jackson2-1.19.0.jar" REQUIRED="true" />
-		<IMPORT NAME="guava-jdk5-13.0.jar" MODULE="guava-jdk5-13.0.jar" MVN="mvn:org.talend.libraries/guava-jdk5-13.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-jdk5-13.0.jar" REQUIRED="true" />
-		<IMPORT NAME="jackson-core-2.9.5.jar" MODULE="jackson-core-2.9.5.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.5" REQUIRED="true" />
-		<!-- REQUIRED FOR GOOGLE STORAGE -->
-		<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED="true" />
-		<IMPORT NAME="commons-logging-1.1.1" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0"  UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar" REQUIRED="true" />
-		<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.5" REQUIRED="true" />
-		<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.9" REQUIRED="true" />
-		<IMPORT NAME="commons-codec-1.4" MODULE="commons-codec-1.4.jar" MVN="mvn:commons-codec/commons-codec/1.4"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.4.jar" REQUIRED="true" />
-		<IMPORT NAME="google-cloud-bigquery-1.32.0.jar" MODULE="google-cloud-bigquery-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="google-http-client-jackson-1.23.0.jar" MODULE="google-http-client-jackson-1.23.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson/1.23.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="google-auth-library-credentials-0.9.1.jar" MODULE="google-auth-library-credentials-0.9.1.jar" MVN="mvn:com.google.auth/google-auth-library-credentials/0.9.1" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="gax-httpjson-0.44.0.jar" MODULE="gax-httpjson-0.44.0.jar" MVN="mvn:com.google.api/gax-httpjson/0.44.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="jackson-core-asl-1.9.11.jar" MODULE="jackson-core-asl-1.9.11.jar" MVN="mvn:org.codehaus.jackson/jackson-core-asl/1.9.11" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="google-auth-library-oauth2-http-0.9.1.jar" MODULE="google-auth-library-oauth2-http-0.9.1.jar" MVN="mvn:com.google.auth/google-auth-library-oauth2-http/0.9.1" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="google-cloud-core-1.32.0.jar" MODULE="google-cloud-core-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="google-cloud-core-http-1.32.0.jar" MODULE="google-cloud-core-http-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core-http/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="guava-20.0.jar" MODULE="guava-20.0.jar" MVN="mvn:com.google.guava/guava/20.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="gax-1.27.0.jar" MODULE="gax-1.27.0.jar" MVN="mvn:com.google.api/gax/1.27.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="google-http-client-appengine-1.23.0.jar" MODULE="google-http-client-appengine-1.23.0.jar" MVN="mvn:com.google.http-client/google-http-client-appengine/1.23.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-		<IMPORT NAME="api-common-1.6.0.jar" MODULE="api-common-1.6.0.jar" MVN="mvn:com.google.api/api-common/1.6.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
-     </IMPORTS>
-   </CODEGENERATION>
-   <RETURNS>
-   </RETURNS>
-</COMPONENT>
+<COMPONENT>
+  <HEADER
+    PLATEFORM="ALL"
+    SERIAL=""
+    VERSION="0.102"
+    STATUS="ALPHA"
+    COMPATIBILITY="ALL"
+    AUTHOR="Talend"
+    RELEASE_DATE="20070312A"
+    STARTABLE="true"
+    LOG4J_ENABLED="true"
+  >
+    <SIGNATURE/>
+  </HEADER>
+  <FAMILIES>
+    <FAMILY>Big Data/Google BigQuery</FAMILY>
+  </FAMILIES>
+  <DOCUMENTATION>
+    <URL/>
+  </DOCUMENTATION>
+  <CONNECTORS>
+    <CONNECTOR CTYPE="FLOW" MAX_INPUT="0" MAX_OUTPUT="0"/>
+    <CONNECTOR CTYPE="ITERATE" MAX_OUTPUT="1" MAX_INPUT="1"/>
+    <CONNECTOR CTYPE="SUBJOB_OK" MAX_INPUT="1" />
+    <CONNECTOR CTYPE="SUBJOB_ERROR" MAX_INPUT="1" />
+    <CONNECTOR CTYPE="COMPONENT_OK"/>
+    <CONNECTOR CTYPE="COMPONENT_ERROR"/>
+    <CONNECTOR CTYPE="RUN_IF"/>
+  </CONNECTORS>
+  <PARAMETERS>
+    <PARAMETER
+    	NAME="SCHEMA"
+    	FIELD="SCHEMA_TYPE"
+    	REQUIRED="true"
+    	NUM_ROW="1"
+    />
+
+    <PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="3" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
+			<ITEMS DEFAULT="SERVICEACCOUNT">
+				<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
+				<ITEM NAME="OAUTH" VALUE="OAUTH" />
+			</ITEMS>
+	</PARAMETER>
+	
+	<PARAMETER 
+			NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE" 
+			REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
+			FIELD="FILE" 
+			NUM_ROW="10" 
+			REQUIRED="true" 
+			SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
+			GROUP="AUTHENTICATION"
+			>
+			<DEFAULT>""</DEFAULT>
+	</PARAMETER>
+
+    <PARAMETER
+      NAME="CLIENT_ID"
+      FIELD="TEXT"
+      NUM_ROW="10"
+      REQUIRED="true"
+      SHOW_IF="AUTH_MODE == 'OAUTH'"
+	  GROUP="AUTHENTICATION"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+	<PARAMETER
+      NAME="CLIENT_SECRET"
+      FIELD="PASSWORD"
+      NUM_ROW="20"
+      REQUIRED="true"
+      SHOW_IF="AUTH_MODE == 'OAUTH'"
+	  GROUP="AUTHENTICATION"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+	<PARAMETER
+      NAME="AUTHORIZATION_CODE"
+      FIELD="TEXT"
+      NUM_ROW="30"
+      REQUIRED="true"
+      SHOW_IF="AUTH_MODE == 'OAUTH'"
+	  GROUP="AUTHENTICATION"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+	<PARAMETER
+      NAME="PROJECT_ID"
+      FIELD="TEXT"
+      NUM_ROW="40"
+      REQUIRED="true"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    
+    <PARAMETER
+      NAME="DATASET"
+      FIELD="TEXT"
+      NUM_ROW="49"
+      REQUIRED="true"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="TABLE"
+      FIELD="TEXT"
+      NUM_ROW="50"
+      REQUIRED="true"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="CREATE_TABLE_IF_NOT_EXIST" FIELD="CHECK" NUM_ROW="50" REQUIRED="true">
+	<DEFAULT>true</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="ACTION_ON_DATA"      
+      FIELD="CLOSED_LIST"
+      NUM_ROW="51"
+      REQUIRED="true"
+    >
+      	<ITEMS DEFAULT="TRUNCATE">
+		<ITEM NAME="TRUNCATE" VALUE="TRUNCATE" />
+		<ITEM NAME="APPEND" VALUE="APPEND" />
+		<ITEM NAME="EMPTY" VALUE="EMPTY" />
+	</ITEMS>
+    </PARAMETER>
+    <PARAMETER NAME="BULK_FILE_ALREADY_EXIST" FIELD="CHECK" NUM_ROW="59" REQUIRED="true" GROUP="GS_CONF">
+	<DEFAULT>false</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="GS_ACCESS_KEY" FIELD="TEXT" NUM_ROW="60" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="GS_SECRET_KEY" FIELD="PASSWORD" NUM_ROW="61" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="GS_LOCAL_FILE" FIELD="FILE" NUM_ROW="62" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="BUCKET_NAME" FIELD="TEXT" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="GS_FILE"
+      FIELD="TEXT"
+      NUM_ROW="70"
+      REQUIRED="true"
+      GROUP="GS_CONF"
+    >
+      <DEFAULT>"gs://"</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="GS_FILE_HEADER"
+      FIELD="TEXT"
+      NUM_ROW="71"
+      REQUIRED="true"
+      GROUP="GS_CONF"
+    >
+      <DEFAULT>0</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="DIE_ON_ERROR" FIELD="CHECK" NUM_ROW="80" REQUIRED="true">
+	<DEFAULT>false</DEFAULT>
+    </PARAMETER>
+   </PARAMETERS>
+   <ADVANCED_PARAMETERS>
+    <PARAMETER NAME="TOKEN_NAME" FIELD="FILE" NUM_ROW="10"
+        REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
+        <DEFAULT>"__COMP_DEFAULT_FILE_DIR__/token.properties"</DEFAULT>
+    </PARAMETER>
+   	<PARAMETER NAME="SET_FIELD_DELIMITER" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" >
+		<DEFAULT>false</DEFAULT>
+	</PARAMETER>
+	<PARAMETER NAME="FIELD_DELIMITER" FIELD="TEXT" REQUIRED="true" NUM_ROW="20"
+		SHOW_IF="(SET_FIELD_DELIMITER == 'true')">
+		<DEFAULT>","</DEFAULT>
+	</PARAMETER>
+    <PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
+      <DEFAULT>false</DEFAULT>
+    </PARAMETER>
+	<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="30"
+		REQUIRED="true" REPOSITORY_VALUE="ENCODING">
+		<DEFAULT>"ISO-8859-15"</DEFAULT>
+	</PARAMETER>
+   </ADVANCED_PARAMETERS>
+   <CODEGENERATION>
+     <IMPORTS>
+		<IMPORT NAME="google-api-client-1.19.0.jar" MODULE="google-api-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-api-client-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-api-client-1.19.0.jar" REQUIRED="true" />
+		<IMPORT NAME="google-api-services-oauth2-v2-rev78-1.19.0.jar" MODULE="google-api-services-oauth2-v2-rev78-1.19.0.jar" MVN="mvn:org.talend.libraries/google-api-services-oauth2-v2-rev78-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-api-services-oauth2-v2-rev78-1.19.0.jar" REQUIRED="true" />
+		<IMPORT NAME="google-api-services-bigquery-v2-rev391-1.21.0.jar" MODULE="google-api-services-bigquery-v2-rev391-1.21.0.jar" MVN="mvn:org.talend.libraries/google-api-services-bigquery-v2-rev391-1.21.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-api-services-bigquery-v2-rev391-1.21.0.jar" REQUIRED="true" />
+		<IMPORT NAME="google-http-client-1.19.0.jar" MODULE="google-http-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-http-client-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-http-client-1.19.0.jar" REQUIRED="true" />
+		<IMPORT NAME="google-oauth-client-1.19.0.jar" MODULE="google-oauth-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-oauth-client-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-oauth-client-1.19.0.jar" REQUIRED="true" />
+		<IMPORT NAME="google-http-client-jackson2-1.19.0.jar" MODULE="google-http-client-jackson2-1.19.0.jar" MVN="mvn:org.talend.libraries/google-http-client-jackson2-1.19.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-http-client-jackson2-1.19.0.jar" REQUIRED="true" />
+		<IMPORT NAME="guava-jdk5-13.0.jar" MODULE="guava-jdk5-13.0.jar" MVN="mvn:org.talend.libraries/guava-jdk5-13.0/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-jdk5-13.0.jar" REQUIRED="true" />
+		<IMPORT NAME="jackson-core-2.9.5.jar" MODULE="jackson-core-2.9.5.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.5" REQUIRED="true" />
+		<!-- REQUIRED FOR GOOGLE STORAGE -->
+		<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED="true" />
+		<IMPORT NAME="commons-logging-1.1.1" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0"  UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar" REQUIRED="true" />
+		<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.5" REQUIRED="true" />
+		<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.9" REQUIRED="true" />
+		<IMPORT NAME="commons-codec-1.4" MODULE="commons-codec-1.4.jar" MVN="mvn:commons-codec/commons-codec/1.4"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.4.jar" REQUIRED="true" />
+		<IMPORT NAME="google-cloud-bigquery-1.32.0.jar" MODULE="google-cloud-bigquery-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="google-http-client-jackson-1.23.0.jar" MODULE="google-http-client-jackson-1.23.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson/1.23.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="google-auth-library-credentials-0.9.1.jar" MODULE="google-auth-library-credentials-0.9.1.jar" MVN="mvn:com.google.auth/google-auth-library-credentials/0.9.1" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="gax-httpjson-0.44.0.jar" MODULE="gax-httpjson-0.44.0.jar" MVN="mvn:com.google.api/gax-httpjson/0.44.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="jackson-core-asl-1.9.11.jar" MODULE="jackson-core-asl-1.9.11.jar" MVN="mvn:org.codehaus.jackson/jackson-core-asl/1.9.11" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="google-auth-library-oauth2-http-0.9.1.jar" MODULE="google-auth-library-oauth2-http-0.9.1.jar" MVN="mvn:com.google.auth/google-auth-library-oauth2-http/0.9.1" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="google-cloud-core-1.32.0.jar" MODULE="google-cloud-core-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="google-cloud-core-http-1.32.0.jar" MODULE="google-cloud-core-http-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core-http/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="guava-20.0.jar" MODULE="guava-20.0.jar" MVN="mvn:com.google.guava/guava/20.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="gax-1.27.0.jar" MODULE="gax-1.27.0.jar" MVN="mvn:com.google.api/gax/1.27.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="google-http-client-appengine-1.23.0.jar" MODULE="google-http-client-appengine-1.23.0.jar" MVN="mvn:com.google.http-client/google-http-client-appengine/1.23.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+		<IMPORT NAME="api-common-1.6.0.jar" MODULE="api-common-1.6.0.jar" MVN="mvn:com.google.api/api-common/1.6.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
+     </IMPORTS>
+   </CODEGENERATION>
+   <RETURNS>
+   </RETURNS>
+</COMPONENT>

main/plugins/org.talend.designer.components.localprovider/components/tBigQueryInput/tBigQueryInput_begin.javajet

@@ -1,10 +1,10 @@
 <%@ jet
 	    imports="
-		org.talend.core.model.process.INode 	
-		org.talend.core.model.process.ElementParameterParser 
+		org.talend.core.model.process.INode
+		org.talend.core.model.process.ElementParameterParser
 		org.talend.designer.codegen.config.CodeGeneratorArgument
-		org.talend.core.model.metadata.IMetadataTable 
-		org.talend.core.model.metadata.IMetadataColumn 
+		org.talend.core.model.metadata.IMetadataTable
+		org.talend.core.model.metadata.IMetadataColumn
 		org.talend.core.model.process.IConnection
 		org.talend.core.model.process.IConnectionCategory
 		org.talend.core.model.metadata.types.JavaTypesManager
@@ -28,10 +28,11 @@
 	String authorizationCode = ElementParameterParser.getValue(node,"__AUTHORIZATION_CODE__");
 	String query = ElementParameterParser.getValue(node,"__QUERY__");
 	boolean useLegacySql = ElementParameterParser.getBooleanValue(node,"__USE_LEGACY_SQL__");
-	
+	String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
+
 	query = query.replaceAll("\n"," ");
 	query = query.replaceAll("\r"," ");
-	
+
     String tokenFile = ElementParameterParser.getValue(node,"__TOKEN_NAME__");
     boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
     if (authMode.equals("OAUTH")) {
@@ -40,22 +41,22 @@
 		<%
 		String passwordFieldName = "__CLIENT_SECRET__";
 		%>
-	
+
 		<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
-		
+
 		final String CLIENT_SECRET_<%=cid%> = "{\"web\": {\"client_id\": \""+<%=clientId%>+"\",\"client_secret\": \"" +decryptedPassword_<%=cid%>+ "\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://accounts.google.com/o/oauth2/token\"}}";
 		final String PROJECT_ID_<%=cid %> = <%=projectId %>;
-	
+
 		// Static variables for API scope, callback URI, and HTTP/JSON functions
 		final List<String> SCOPES_<%=cid%> = java.util.Arrays.asList("https://www.googleapis.com/auth/bigquery");
 		final String REDIRECT_URI_<%=cid%> = "urn:ietf:wg:oauth:2.0:oob";
 		final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
 		final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
-	
+
 		com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets clientSecrets_<%=cid%> = com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.load(
 						new com.google.api.client.json.jackson2.JacksonFactory(), new java.io.InputStreamReader(new java.io.ByteArrayInputStream(
 								CLIENT_SECRET_<%=cid%>.getBytes())));
-	
+
 		com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow flow_<%=cid%> = null;
 		com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
 		long nb_line_<%=cid%> = 0;
@@ -68,7 +69,7 @@
 		<%
 		}
 		%>
-	
+
 		// Attempt to load existing refresh token
 	    String tokenFile_<%=cid %> = <%=tokenFile%>;
 		java.util.Properties properties_<%=cid%> = new java.util.Properties();
@@ -94,7 +95,7 @@
 	    	%>
 		}
 		String storedRefreshToken_<%=cid%> = (String) properties_<%=cid%>.get("refreshtoken");
-	
+
 		// Check to see if the an existing refresh token was loaded.
 		// If so, create a credential and call refreshToken() to get a new
 		// access token.
@@ -103,7 +104,7 @@
 			com.google.api.client.googleapis.auth.oauth2.GoogleCredential credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2. GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
 					.setJsonFactory(JSON_FACTORY_<%=cid%>).setClientSecrets(clientSecrets_<%=cid%>)
 					.build().setFromTokenResponse(new com.google.api.client.auth.oauth2.TokenResponse().setRefreshToken(storedRefreshToken_<%=cid%>));
-	
+
 			credential_<%=cid%>.refreshToken();
 			<%
 			if(isLog4jEnabled){
@@ -125,7 +126,7 @@
 			if(authorizationCode_<%=cid%> == null || "".equals(authorizationCode_<%=cid%>) || "\"\"".equals(authorizationCode_<%=cid%>)) {
 				String authorizeUrl_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl(
 						clientSecrets_<%=cid%>, REDIRECT_URI_<%=cid%>, SCOPES_<%=cid%>).setState("").build();
-	
+
 				System.out
 						.println("Paste this URL into a web browser to authorize BigQuery Access:\n"
 								+ authorizeUrl_<%=cid%>);
@@ -155,7 +156,7 @@
 				}
 				com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse response_<%=cid%> = flow_<%=cid%>.newTokenRequest(authorizationCode_<%=cid%>).setRedirectUri(REDIRECT_URI_<%=cid%>).execute();
 				com.google.api.client.auth.oauth2.Credential credential_<%=cid%> = flow_<%=cid%>.createAndStoreCredential(response_<%=cid%>, null);
-	
+
 				<%
 				if(isLog4jEnabled){
 				%>
@@ -171,17 +172,13 @@
 				if (outputStream_<%=cid%> != null) {
 				    outputStream_<%=cid%>.close();
 				}
-	
+
 				bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).build();
 			}
 		}
-	
-	
-		<%
-		String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
-		%>
-		<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>	
-	
+
+		<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>
+
 		// Start a Query Job
 		String querySql_<%=cid %> = <%=query %>;
 		System.out.format("Running Query : %s\n", querySql_<%=cid %>);
@@ -192,18 +189,17 @@
 		<%
 		}
 		%>
-		
+
 		BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, tokenFile_<%=cid%>);
-		
 		<%
 		if(isLog4jEnabled){
 		%>
-			log.info("<%=cid%> - Try <%="LARGE".equals(resultSizeType) ? "with" : "without"%> allow large results flag");
+			log.info("<%=cid%> - Try <%="LARGE".equals(resultSizeType) ? "with" : "without"%> the allowLargeResults flag");
 		<%
 		}
 		%>
 		com.google.api.services.bigquery.model.Job insert_<%=cid %> = bigQueryUtil_<%=cid%>.executeQuery(querySql_<%=cid%>, <%="LARGE".equals(resultSizeType) ? true : false%>, <%=useLegacySql%>);
-	
+
 		<%
 		if(isLog4jEnabled){
 		%>
@@ -221,15 +217,30 @@
 		    credentials_<%=cid%> = com.google.auth.oauth2.ServiceAccountCredentials.fromStream(credentialsStream_<%=cid%>);
 		}
 
+		String query_<%=cid%> = <%=query%>;
+		String dataset = query_<%=cid%>.substring(query_<%=cid%>.indexOf("from") + 4, query_<%=cid%>.indexOf(".")).trim();
+		String temp_table = "temp_" + dataset + java.util.UUID.randomUUID().toString().replaceAll("-", "")
+													 + "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a")
+													 + Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
+
 		com.google.cloud.bigquery.BigQuery bigquery_<%=cid%> = com.google.cloud.bigquery.BigQueryOptions.newBuilder()
 				.setCredentials(credentials_<%=cid%>)
 				.setProjectId(<%=projectId%>)
 				.build()
 				.getService();
-				
-		com.google.cloud.bigquery.QueryJobConfiguration queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>).build();
-		com.google.cloud.bigquery.JobId jobId_tBigQueryInput_1 = com.google.cloud.bigquery.JobId.of(java.util.UUID.randomUUID().toString());
-		com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>).setJobId(jobId_<%=cid%>).build());
+
+		com.google.cloud.bigquery.QueryJobConfiguration.Builder queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>);
+
+		<%
+		if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
+		%>
+			queryConfiguration_<%=cid%>.setDestinationTable(com.google.cloud.bigquery.TableId.of(dataset, temp_table)).setAllowLargeResults(true);
+		<%
+		}
+		%>
+
+		com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(java.util.UUID.randomUUID().toString());
+		com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
 
 		<%
 		if(isLog4jEnabled){
@@ -245,7 +256,6 @@
 		} else if (job_<%=cid%>.getStatus().getError() != null) {
 			throw new RuntimeException(job_<%=cid%>.getStatus().getError().toString());
 		}
-
 		<%
 		if(isLog4jEnabled){
 		%>
@@ -253,16 +263,14 @@
 		<%
 		}
 		%>
-		
 		com.google.cloud.bigquery.TableResult result_<%=cid%> = job_<%=cid%>.getQueryResults();
-
 		long nb_line_<%=cid%> = 0;
 	<%
 	} else {
 		throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
 	}
-	
-	
+
+
 	if (authMode.equals("OAUTH")) {
 	%>
 	while (true) {
@@ -274,14 +282,13 @@
 							insert_<%=cid %>.getConfiguration().getQuery()
 									.getDestinationTable().getTableId())
 					.setPageToken(pageToken_<%=cid%>).execute();
-		
-		List<com.google.api.services.bigquery.model.TableRow> rows_<%=cid %> = dataList_<%=cid %>.getRows();
 
+		List<com.google.api.services.bigquery.model.TableRow> rows_<%=cid %> = dataList_<%=cid %>.getRows();
 		if(rows_<%=cid %> == null) {
 			// Means there is no record.
 			rows_<%=cid %> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableRow>();
 		}
-		
+
 		for (com.google.api.services.bigquery.model.TableRow row_<%=cid %> : rows_<%=cid %>) {
 			java.util.List<com.google.api.services.bigquery.model.TableCell> field_<%=cid %> = row_<%=cid %>.getF();
 			Object value_<%=cid%> = null;
@@ -311,16 +318,14 @@
 							boolean advancedSeparator = (advancedSeparatorStr!=null&&!("").equals(advancedSeparatorStr))?("true").equals(advancedSeparatorStr):false;
 							String thousandsSeparator = ElementParameterParser.getValueWithJavaType(node, "__THOUSANDS_SEPARATOR__", JavaTypesManager.CHARACTER);
 							String decimalSeparator = ElementParameterParser.getValueWithJavaType(node, "__DECIMAL_SEPARATOR__", JavaTypesManager.CHARACTER);
-						
 							List<IMetadataColumn> columns = metadata.getListColumns();
 							int nbColumns = columns.size();
 							for (int i = 0; i < nbColumns; i++ ) {
 								IMetadataColumn column = columns.get(i);
 								String columnName = column.getLabel();
-							
 								String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
 								JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
-								String patternValue = column.getPattern() == null || column.getPattern().trim().length() == 0 ? null : column.getPattern(); 
+								String patternValue = column.getPattern() == null || column.getPattern().trim().length() == 0 ? null : column.getPattern();
 								if (authMode.equals("OAUTH")) {
 								%>
 								value_<%=cid%> = field_<%=cid %>.get(<%=i%>).getV();
@@ -334,27 +339,34 @@
 								}
 								%>
 								if(com.google.api.client.util.Data.isNull(value_<%=cid%>)) value_<%=cid%> = null;
+
 								if(value_<%=cid%> != null){
-									
+
 									<%
 									if (javaType == JavaTypesManager.STRING) {
 									%>
 										<%=connName%>.<%=columnName%> = value_<%=cid%>.toString();
 									<%
 									} else if (javaType == JavaTypesManager.OBJECT) {
-									%>		
+									%>
 										<%=connName%>.<%=columnName%> = value_<%=cid%>;
-									<%		
-									} else if(javaType == JavaTypesManager.DATE) {						
-									%>					
-										<%=connName%>.<%=columnName%> = ParserUtils.parseTo_Date(value_<%=cid%>.toString());				
-									<%		
-									} else if(advancedSeparator && JavaTypesManager.isNumberType(javaType)) { 
+									<%
+									} else if(javaType == JavaTypesManager.DATE) {
+									%>
+
+										if (value_<%=cid%>.toString().contains("-")) {
+											<%=connName%>.<%=columnName%> = ParserUtils.parseTo_Date(value_<%=cid%>.toString(),<%=patternValue%>);
+										} else {
+											<%=connName%>.<%=columnName%> = ParserUtils.parseTo_Date(value_<%=cid%>.toString());
+										}
+
+									<%
+									} else if(advancedSeparator && JavaTypesManager.isNumberType(javaType)) {
 									%>
 										<%=connName%>.<%=columnName%> = ParserUtils.parseTo_<%= typeToGenerate %>(ParserUtils.parseTo_Number(value_<%=cid%>.toString(), <%= thousandsSeparator %>, <%= decimalSeparator %>));
 									<%
-									} else if(javaType == JavaTypesManager.BYTE_ARRAY) { 
-									%>							
+									} else if(javaType == JavaTypesManager.BYTE_ARRAY) {
+									%>
 										<%=connName%>.<%=columnName%> = value_<%=cid%>.toString().getBytes(<%=encoding %>);
 									<%
 									} else {

main/plugins/org.talend.designer.components.localprovider/components/tBigQueryOutput/tBigQueryOutput_java.xml

@@ -1,311 +1,310 @@
-<COMPONENT>
-  <HEADER
-    PLATEFORM="ALL"
-    SERIAL=""
-    VERSION="0.102"
-    STATUS="ALPHA"
-
-    COMPATIBILITY="ALL"
-    AUTHOR="Talend"
-    RELEASE_DATE="20050320A"
-    STARTABLE="false"
-	PARTITIONING="NONE"
-  >
-    <SIGNATURE/>
-  </HEADER>
-
-  <FAMILIES>
-    <FAMILY>Big Data/Google BigQuery</FAMILY>
-  </FAMILIES>
-
-  <DOCUMENTATION>
-    <URL/>
-  </DOCUMENTATION>
-
-  <CONNECTORS>
-    <CONNECTOR CTYPE="FLOW" MAX_INPUT="1" MAX_OUTPUT="0"/>
-    <CONNECTOR CTYPE="ITERATE" MAX_OUTPUT="1" MAX_INPUT="1"/>
-    <CONNECTOR CTYPE="SUBJOB_OK" MAX_INPUT="1" />
-    <CONNECTOR CTYPE="SUBJOB_ERROR" MAX_INPUT="1" />
-    <CONNECTOR CTYPE="COMPONENT_OK"/>
-    <CONNECTOR CTYPE="COMPONENT_ERROR"/>
-    <CONNECTOR CTYPE="RUN_IF"/>
-  </CONNECTORS>
-
-  <PARAMETERS>
-    <PARAMETER NAME="PROPERTY" FIELD="PROPERTY_TYPE" SHOW="true" NUM_ROW="10" REPOSITORY_VALUE="DELIMITED" />
-
-    <PARAMETER NAME="FILENAME" FIELD="FILE" NUM_ROW="11" REQUIRED="true" REPOSITORY_VALUE="FILE_PATH">
-      <DEFAULT>"__COMP_DEFAULT_FILE_DIR__/biquery_bulk.txt"</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="APPEND" FIELD="CHECK" REQUIRED="true" NUM_ROW="12" REPOSITORY_VALUE="APPEND">
-      <DEFAULT>false</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="SCHEMA" FIELD="SCHEMA_TYPE" REQUIRED="true" NUM_ROW="13">
-      <DEFAULT/>
-    </PARAMETER>
-    
-    <PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="15" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
-			<ITEMS DEFAULT="SERVICEACCOUNT">
-				<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
-				<ITEM NAME="OAUTH" VALUE="OAUTH" />
-			</ITEMS>
-	</PARAMETER>
-
-	<PARAMETER 
-			NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE" 
-			REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
-			FIELD="FILE" 
-			NUM_ROW="18" 
-			REQUIRED="true" 
-			SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
-			GROUP="AUTHENTICATION"
-			>
-			<DEFAULT>""</DEFAULT>
-	</PARAMETER>
-
-	<PARAMETER
-      NAME="CLIENT_ID"
-      FIELD="TEXT"
-      NUM_ROW="20"
-      REQUIRED="true"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-      GROUP="AUTHENTICATION"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-
-	<PARAMETER
-      NAME="CLIENT_SECRET"
-      FIELD="PASSWORD"
-      NUM_ROW="21"
-      REQUIRED="true"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-      GROUP="AUTHENTICATION"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-
-	<PARAMETER
-      NAME="AUTHORIZATION_CODE"
-      FIELD="TEXT"
-      NUM_ROW="30"
-      REQUIRED="true"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-      GROUP="AUTHENTICATION"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-
-	<PARAMETER
-      NAME="PROJECT_ID"
-      FIELD="TEXT"
-      NUM_ROW="40"
-      REQUIRED="true"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER
-      NAME="DATASET"
-      FIELD="TEXT"
-      NUM_ROW="49"
-      REQUIRED="true"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="TABLE"
-      FIELD="TEXT"
-      NUM_ROW="50"
-      REQUIRED="true"
-    >
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="CREATE_TABLE_IF_NOT_EXIST" FIELD="CHECK" NUM_ROW="50" REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
-	<DEFAULT>true</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER
-      NAME="ACTION_ON_DATA"
-      SHOW_IF="AUTH_MODE == 'OAUTH'"
-      FIELD="CLOSED_LIST"
-      NUM_ROW="51"
-      REQUIRED="true"
-    >
-      	<ITEMS DEFAULT="TRUNCATE">
-		<ITEM NAME="TRUNCATE" VALUE="TRUNCATE" />
-		<ITEM NAME="APPEND" VALUE="APPEND" />
-		<ITEM NAME="EMPTY" VALUE="EMPTY" />
-	</ITEMS>
-    </PARAMETER>
-    <PARAMETER NAME="BULK_FILE_ALREADY_EXIST" FIELD="CHECK" NUM_ROW="59" REQUIRED="true" SHOW="false">
-	<DEFAULT>false</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="GS_ACCESS_KEY" FIELD="TEXT" NUM_ROW="60" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="GS_SECRET_KEY" FIELD="PASSWORD" NUM_ROW="61" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="BUCKET_NAME" FIELD="TEXT" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
-      <DEFAULT>""</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="GS_FILE"
-      FIELD="TEXT"
-      NUM_ROW="70"
-      REQUIRED="true"
-      GROUP="GS_CONF"
-    >
-      <DEFAULT>"gs://"</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="GS_FILE_HEADER"
-      FIELD="TEXT"
-      NUM_ROW="71"
-      REQUIRED="true"
-      GROUP="GS_CONF"
-    >
-      <DEFAULT>0</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="DIE_ON_ERROR" FIELD="CHECK" NUM_ROW="80" REQUIRED="true">
-	<DEFAULT>false</DEFAULT>
-    </PARAMETER>
-   </PARAMETERS>
-
-   <ADVANCED_PARAMETERS>
-    <PARAMETER NAME="TOKEN_NAME" FIELD="FILE" NUM_ROW="10"
-        REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
-        <DEFAULT>"__COMP_DEFAULT_FILE_DIR__/token.properties"</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="SET_FIELD_DELIMITER" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" SHOW="false">
-        <DEFAULT>true</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="FIELD_DELIMITER" FIELD="TEXT" REQUIRED="true" NUM_ROW="20"
-        SHOW="false">
-        <DEFAULT>","</DEFAULT>
-    </PARAMETER>
-	<PARAMETER NAME="ROWSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="ROW_SEPARATOR" SHOW="false">
-      <DEFAULT>"\n"</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="FIELDSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="FIELD_SEPARATOR">
-      <DEFAULT>","</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="33" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
-      <DEFAULT>false</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="ESCAPE_CHAR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="ESCAPE_CHAR" SHOW="false">
-      <DEFAULT>"\\"</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="TEXT_ENCLOSURE" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="TEXT_ENCLOSURE" SHOW="false">
-      <DEFAULT>"\""</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="CREATE" FIELD="CHECK" REQUIRED="true" NUM_ROW="40">
-      <DEFAULT>true</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER NAME="FLUSHONROW" FIELD="CHECK" REQUIRED="true" NUM_ROW="45">
-      <DEFAULT>false</DEFAULT>
-    </PARAMETER>
-
-    <PARAMETER
-	NAME="FLUSHONROW_NUM"
-	FIELD="TEXT"
-	REQUIRED="true"
-	NUM_ROW="45"
-	SHOW_IF="FLUSHONROW == 'true'"
-    >
-      <DEFAULT>1</DEFAULT>
-    </PARAMETER>
-     <PARAMETER
-      NAME="RECORDS_CONTAIN_NULL_VALUE"
-      FIELD="CHECK"
-      SHOW="false"
-      NUM_ROW="50"
-    >
-      <DEFAULT>true</DEFAULT>
-    </PARAMETER>
-    <PARAMETER
-      NAME="CHECK_DISK_SPACE"
-      FIELD="CHECK"
-      SHOW="true"
-      NUM_ROW="50"
-    >
-      <DEFAULT>false</DEFAULT>
-    </PARAMETER>
-    <PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="60" REQUIRED="true" REPOSITORY_VALUE="ENCODING">
-      <DEFAULT>"ISO-8859-15"</DEFAULT>
-    </PARAMETER>
-   </ADVANCED_PARAMETERS>
-
-  <CODEGENERATION>
-    <TEMPLATES INPUT="tBQOB" OUTPUT="tBQBE">
-
-      <TEMPLATE NAME="tBQOB" COMPONENT="tBigQueryOutputBulk">
-        <LINK_TO NAME="tBQBE" CTYPE="ROWS_END"/>
-      </TEMPLATE>
-
-      <TEMPLATE NAME="tBQBE" COMPONENT="tBigQueryBulkExec"/>
-
-      <TEMPLATE_PARAM SOURCE="self.AUTH_MODE"   TARGET="tBQBE.AUTH_MODE" />
-      <TEMPLATE_PARAM SOURCE="self.SERVICE_ACCOUNT_CREDENTIALS_FILE"   TARGET="tBQBE.SERVICE_ACCOUNT_CREDENTIALS_FILE" />
-      <TEMPLATE_PARAM SOURCE="self.CLIENT_ID"   TARGET="tBQBE.CLIENT_ID" />
-      <TEMPLATE_PARAM SOURCE="self.CLIENT_SECRET" TARGET="tBQBE.CLIENT_SECRET" />
-      <TEMPLATE_PARAM SOURCE="self.PROJECT_ID" TARGET="tBQBE.PROJECT_ID" />
-      <TEMPLATE_PARAM SOURCE="self.AUTHORIZATION_CODE"   TARGET="tBQBE.AUTHORIZATION_CODE"/>
-      <TEMPLATE_PARAM SOURCE="self.DATASET"   TARGET="tBQBE.DATASET" />
-      <TEMPLATE_PARAM SOURCE="self.TABLE"   TARGET="tBQBE.TABLE" />
-      <TEMPLATE_PARAM SOURCE="self.CREATE_TABLE_IF_NOT_EXIST"  TARGET="tBQBE.CREATE_TABLE_IF_NOT_EXIST" />
-      <TEMPLATE_PARAM SOURCE="self.ACTION_ON_DATA"  TARGET="tBQBE.ACTION_ON_DATA" />
-      <TEMPLATE_PARAM SOURCE="self.GS_ACCESS_KEY"  TARGET="tBQBE.GS_ACCESS_KEY" />
-      <TEMPLATE_PARAM SOURCE="self.GS_SECRET_KEY"  TARGET="tBQBE.GS_SECRET_KEY" />
-      <TEMPLATE_PARAM SOURCE="self.BUCKET_NAME"  TARGET="tBQBE.BUCKET_NAME" />
-      <TEMPLATE_PARAM SOURCE="self.GS_FILE"  TARGET="tBQBE.GS_FILE" />
-      <TEMPLATE_PARAM SOURCE="self.GS_FILE_HEADER"  TARGET="tBQBE.GS_FILE_HEADER" />
-      <TEMPLATE_PARAM SOURCE="self.TOKEN_NAME"  TARGET="tBQBE.TOKEN_NAME" />
-      <TEMPLATE_PARAM SOURCE="self.SET_FIELD_DELIMITER"  TARGET="tBQBE.SET_FIELD_DELIMITER" />
-      <TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR"  TARGET="tBQBE.FIELD_DELIMITER" />
-      <TEMPLATE_PARAM SOURCE="self.BULK_ALREADY_EXIST"  TARGET="tBQBE.BULK_ALREADY_EXIST" />
-      <TEMPLATE_PARAM SOURCE="self.DIE_ON_ERROR"  TARGET="tBQBE.DIE_ON_ERROR" />
-
-      <TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQOB.FILENAME" />
-      <TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQBE.GS_LOCAL_FILE" />
-
-      <TEMPLATE_PARAM SOURCE="self.ROWSEPARATOR" TARGET="tBQOB.ROWSEPARATOR" />
-
-      <TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR"  TARGET="tBQOB.FIELDSEPARATOR" />
-      <TEMPLATE_PARAM SOURCE="self.DROP"  TARGET="tBQBE.DROP" />
-
-      <TEMPLATE_PARAM SOURCE="self.ESCAPE_CHAR"  TARGET="tBQOB.ESCAPE_CHAR" />
-
-      <TEMPLATE_PARAM SOURCE="self.TEXT_ENCLOSURE"  TARGET="tBQOB.TEXT_ENCLOSURE" />
-
-      <TEMPLATE_PARAM SOURCE="self.SCHEMA"   TARGET="tBQOB.SCHEMA" />
-      <TEMPLATE_PARAM SOURCE="self.SCHEMA"   TARGET="tBQBE.SCHEMA" />
-
-      <TEMPLATE_PARAM SOURCE="self.ENCODING"   TARGET="tBQOB.ENCODING" />
-      <TEMPLATE_PARAM SOURCE="self.ENCODING"   TARGET="tBQBE.ENCODING" />
-
-      <TEMPLATE_PARAM SOURCE="self.RECORDS_CONTAIN_NULL_VALUE"   TARGET="tBQOB.RECORDS_CONTAIN_NULL_VALUE" />
-
-      <TEMPLATE_PARAM SOURCE="self.APPEND"   TARGET="tBQOB.APPEND" />
-      <TEMPLATE_PARAM SOURCE="self.CREATE" TARGET="tBQOB.CREATE" />
-      <TEMPLATE_PARAM SOURCE="self.FLUSHONROW"   TARGET="tBQOB.FLUSHONROW" />
-      <TEMPLATE_PARAM SOURCE="self.FLUSHONROW_NUM"   TARGET="tBQOB.FLUSHONROW_NUM" />
-
-      <TEMPLATE_PARAM SOURCE="self.DATA_ACTION"  TARGET="tBQBE.DATA_ACTION" />
-    </TEMPLATES>
-  </CODEGENERATION>
-  <RETURNS/>
-</COMPONENT>
+<COMPONENT>
+  <HEADER
+    PLATEFORM="ALL"
+    SERIAL=""
+    VERSION="0.102"
+    STATUS="ALPHA"
+
+    COMPATIBILITY="ALL"
+    AUTHOR="Talend"
+    RELEASE_DATE="20050320A"
+    STARTABLE="false"
+	PARTITIONING="NONE"
+  >
+    <SIGNATURE/>
+  </HEADER>
+
+  <FAMILIES>
+    <FAMILY>Big Data/Google BigQuery</FAMILY>
+  </FAMILIES>
+
+  <DOCUMENTATION>
+    <URL/>
+  </DOCUMENTATION>
+
+  <CONNECTORS>
+    <CONNECTOR CTYPE="FLOW" MAX_INPUT="1" MAX_OUTPUT="0"/>
+    <CONNECTOR CTYPE="ITERATE" MAX_OUTPUT="1" MAX_INPUT="1"/>
+    <CONNECTOR CTYPE="SUBJOB_OK" MAX_INPUT="1" />
+    <CONNECTOR CTYPE="SUBJOB_ERROR" MAX_INPUT="1" />
+    <CONNECTOR CTYPE="COMPONENT_OK"/>
+    <CONNECTOR CTYPE="COMPONENT_ERROR"/>
+    <CONNECTOR CTYPE="RUN_IF"/>
+  </CONNECTORS>
+
+  <PARAMETERS>
+    <PARAMETER NAME="PROPERTY" FIELD="PROPERTY_TYPE" SHOW="true" NUM_ROW="10" REPOSITORY_VALUE="DELIMITED" />
+
+    <PARAMETER NAME="FILENAME" FIELD="FILE" NUM_ROW="11" REQUIRED="true" REPOSITORY_VALUE="FILE_PATH">
+      <DEFAULT>"__COMP_DEFAULT_FILE_DIR__/biquery_bulk.txt"</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="APPEND" FIELD="CHECK" REQUIRED="true" NUM_ROW="12" REPOSITORY_VALUE="APPEND">
+      <DEFAULT>false</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="SCHEMA" FIELD="SCHEMA_TYPE" REQUIRED="true" NUM_ROW="13">
+      <DEFAULT/>
+    </PARAMETER>
+    
+    <PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="15" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
+			<ITEMS DEFAULT="SERVICEACCOUNT">
+				<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
+				<ITEM NAME="OAUTH" VALUE="OAUTH" />
+			</ITEMS>
+	</PARAMETER>
+
+	<PARAMETER 
+			NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE" 
+			REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
+			FIELD="FILE" 
+			NUM_ROW="18" 
+			REQUIRED="true" 
+			SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
+			GROUP="AUTHENTICATION"
+			>
+			<DEFAULT>""</DEFAULT>
+	</PARAMETER>
+
+	<PARAMETER
+      NAME="CLIENT_ID"
+      FIELD="TEXT"
+      NUM_ROW="20"
+      REQUIRED="true"
+      SHOW_IF="AUTH_MODE == 'OAUTH'"
+      GROUP="AUTHENTICATION"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+
+	<PARAMETER
+      NAME="CLIENT_SECRET"
+      FIELD="PASSWORD"
+      NUM_ROW="21"
+      REQUIRED="true"
+      SHOW_IF="AUTH_MODE == 'OAUTH'"
+      GROUP="AUTHENTICATION"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+
+	<PARAMETER
+      NAME="AUTHORIZATION_CODE"
+      FIELD="TEXT"
+      NUM_ROW="30"
+      REQUIRED="true"
+      SHOW_IF="AUTH_MODE == 'OAUTH'"
+      GROUP="AUTHENTICATION"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+
+	<PARAMETER
+      NAME="PROJECT_ID"
+      FIELD="TEXT"
+      NUM_ROW="40"
+      REQUIRED="true"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER
+      NAME="DATASET"
+      FIELD="TEXT"
+      NUM_ROW="49"
+      REQUIRED="true"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="TABLE"
+      FIELD="TEXT"
+      NUM_ROW="50"
+      REQUIRED="true"
+    >
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="CREATE_TABLE_IF_NOT_EXIST" FIELD="CHECK" NUM_ROW="50" REQUIRED="true">
+	<DEFAULT>true</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER
+      NAME="ACTION_ON_DATA"
+      FIELD="CLOSED_LIST"
+      NUM_ROW="51"
+      REQUIRED="true"
+    >
+      	<ITEMS DEFAULT="TRUNCATE">
+		<ITEM NAME="TRUNCATE" VALUE="TRUNCATE" />
+		<ITEM NAME="APPEND" VALUE="APPEND" />
+		<ITEM NAME="EMPTY" VALUE="EMPTY" />
+	</ITEMS>
+    </PARAMETER>
+    <PARAMETER NAME="BULK_FILE_ALREADY_EXIST" FIELD="CHECK" NUM_ROW="59" REQUIRED="true" SHOW="false">
+	<DEFAULT>false</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="GS_ACCESS_KEY" FIELD="TEXT" NUM_ROW="60" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="GS_SECRET_KEY" FIELD="PASSWORD" NUM_ROW="61" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="BUCKET_NAME" FIELD="TEXT" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
+      <DEFAULT>""</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="GS_FILE"
+      FIELD="TEXT"
+      NUM_ROW="70"
+      REQUIRED="true"
+      GROUP="GS_CONF"
+    >
+      <DEFAULT>"gs://"</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="GS_FILE_HEADER"
+      FIELD="TEXT"
+      NUM_ROW="71"
+      REQUIRED="true"
+      GROUP="GS_CONF"
+    >
+      <DEFAULT>0</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="DIE_ON_ERROR" FIELD="CHECK" NUM_ROW="80" REQUIRED="true">
+	<DEFAULT>false</DEFAULT>
+    </PARAMETER>
+   </PARAMETERS>
+
+   <ADVANCED_PARAMETERS>
+    <PARAMETER NAME="TOKEN_NAME" FIELD="FILE" NUM_ROW="10"
+        REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
+        <DEFAULT>"__COMP_DEFAULT_FILE_DIR__/token.properties"</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="SET_FIELD_DELIMITER" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" SHOW="false">
+        <DEFAULT>true</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="FIELD_DELIMITER" FIELD="TEXT" REQUIRED="true" NUM_ROW="20"
+        SHOW="false">
+        <DEFAULT>","</DEFAULT>
+    </PARAMETER>
+	<PARAMETER NAME="ROWSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="ROW_SEPARATOR" SHOW="false">
+      <DEFAULT>"\n"</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="FIELDSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="FIELD_SEPARATOR">
+      <DEFAULT>","</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="33" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
+      <DEFAULT>false</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="ESCAPE_CHAR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="ESCAPE_CHAR" SHOW="false">
+      <DEFAULT>"\\"</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="TEXT_ENCLOSURE" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="TEXT_ENCLOSURE" SHOW="false">
+      <DEFAULT>"\""</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="CREATE" FIELD="CHECK" REQUIRED="true" NUM_ROW="40">
+      <DEFAULT>true</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER NAME="FLUSHONROW" FIELD="CHECK" REQUIRED="true" NUM_ROW="45">
+      <DEFAULT>false</DEFAULT>
+    </PARAMETER>
+
+    <PARAMETER
+	NAME="FLUSHONROW_NUM"
+	FIELD="TEXT"
+	REQUIRED="true"
+	NUM_ROW="45"
+	SHOW_IF="FLUSHONROW == 'true'"
+    >
+      <DEFAULT>1</DEFAULT>
+    </PARAMETER>
+     <PARAMETER
+      NAME="RECORDS_CONTAIN_NULL_VALUE"
+      FIELD="CHECK"
+      SHOW="false"
+      NUM_ROW="50"
+    >
+      <DEFAULT>true</DEFAULT>
+    </PARAMETER>
+    <PARAMETER
+      NAME="CHECK_DISK_SPACE"
+      FIELD="CHECK"
+      SHOW="true"
+      NUM_ROW="50"
+    >
+      <DEFAULT>false</DEFAULT>
+    </PARAMETER>
+    <PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="60" REQUIRED="true" REPOSITORY_VALUE="ENCODING">
+      <DEFAULT>"ISO-8859-15"</DEFAULT>
+    </PARAMETER>
+   </ADVANCED_PARAMETERS>
+
+  <CODEGENERATION>
+    <TEMPLATES INPUT="tBQOB" OUTPUT="tBQBE">
+
+      <TEMPLATE NAME="tBQOB" COMPONENT="tBigQueryOutputBulk">
+        <LINK_TO NAME="tBQBE" CTYPE="ROWS_END"/>
+      </TEMPLATE>
+
+      <TEMPLATE NAME="tBQBE" COMPONENT="tBigQueryBulkExec"/>
+
+      <TEMPLATE_PARAM SOURCE="self.AUTH_MODE"   TARGET="tBQBE.AUTH_MODE" />
+      <TEMPLATE_PARAM SOURCE="self.SERVICE_ACCOUNT_CREDENTIALS_FILE"   TARGET="tBQBE.SERVICE_ACCOUNT_CREDENTIALS_FILE" />
+      <TEMPLATE_PARAM SOURCE="self.CLIENT_ID"   TARGET="tBQBE.CLIENT_ID" />
+      <TEMPLATE_PARAM SOURCE="self.CLIENT_SECRET" TARGET="tBQBE.CLIENT_SECRET" />
+      <TEMPLATE_PARAM SOURCE="self.PROJECT_ID" TARGET="tBQBE.PROJECT_ID" />
+      <TEMPLATE_PARAM SOURCE="self.AUTHORIZATION_CODE"   TARGET="tBQBE.AUTHORIZATION_CODE"/>
+      <TEMPLATE_PARAM SOURCE="self.DATASET"   TARGET="tBQBE.DATASET" />
+      <TEMPLATE_PARAM SOURCE="self.TABLE"   TARGET="tBQBE.TABLE" />
+      <TEMPLATE_PARAM SOURCE="self.CREATE_TABLE_IF_NOT_EXIST"  TARGET="tBQBE.CREATE_TABLE_IF_NOT_EXIST" />
+      <TEMPLATE_PARAM SOURCE="self.ACTION_ON_DATA"  TARGET="tBQBE.ACTION_ON_DATA" />
+      <TEMPLATE_PARAM SOURCE="self.GS_ACCESS_KEY"  TARGET="tBQBE.GS_ACCESS_KEY" />
+      <TEMPLATE_PARAM SOURCE="self.GS_SECRET_KEY"  TARGET="tBQBE.GS_SECRET_KEY" />
+      <TEMPLATE_PARAM SOURCE="self.BUCKET_NAME"  TARGET="tBQBE.BUCKET_NAME" />
+      <TEMPLATE_PARAM SOURCE="self.GS_FILE"  TARGET="tBQBE.GS_FILE" />
+      <TEMPLATE_PARAM SOURCE="self.GS_FILE_HEADER"  TARGET="tBQBE.GS_FILE_HEADER" />
+      <TEMPLATE_PARAM SOURCE="self.TOKEN_NAME"  TARGET="tBQBE.TOKEN_NAME" />
+      <TEMPLATE_PARAM SOURCE="self.SET_FIELD_DELIMITER"  TARGET="tBQBE.SET_FIELD_DELIMITER" />
+      <TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR"  TARGET="tBQBE.FIELD_DELIMITER" />
+      <TEMPLATE_PARAM SOURCE="self.BULK_ALREADY_EXIST"  TARGET="tBQBE.BULK_ALREADY_EXIST" />
+      <TEMPLATE_PARAM SOURCE="self.DIE_ON_ERROR"  TARGET="tBQBE.DIE_ON_ERROR" />
+
+      <TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQOB.FILENAME" />
+      <TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQBE.GS_LOCAL_FILE" />
+
+      <TEMPLATE_PARAM SOURCE="self.ROWSEPARATOR" TARGET="tBQOB.ROWSEPARATOR" />
+
+      <TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR"  TARGET="tBQOB.FIELDSEPARATOR" />
+      <TEMPLATE_PARAM SOURCE="self.DROP"  TARGET="tBQBE.DROP" />
+
+      <TEMPLATE_PARAM SOURCE="self.ESCAPE_CHAR"  TARGET="tBQOB.ESCAPE_CHAR" />
+
+      <TEMPLATE_PARAM SOURCE="self.TEXT_ENCLOSURE"  TARGET="tBQOB.TEXT_ENCLOSURE" />
+
+      <TEMPLATE_PARAM SOURCE="self.SCHEMA"   TARGET="tBQOB.SCHEMA" />
+      <TEMPLATE_PARAM SOURCE="self.SCHEMA"   TARGET="tBQBE.SCHEMA" />
+
+      <TEMPLATE_PARAM SOURCE="self.ENCODING"   TARGET="tBQOB.ENCODING" />
+      <TEMPLATE_PARAM SOURCE="self.ENCODING"   TARGET="tBQBE.ENCODING" />
+
+      <TEMPLATE_PARAM SOURCE="self.RECORDS_CONTAIN_NULL_VALUE"   TARGET="tBQOB.RECORDS_CONTAIN_NULL_VALUE" />
+
+      <TEMPLATE_PARAM SOURCE="self.APPEND"   TARGET="tBQOB.APPEND" />
+      <TEMPLATE_PARAM SOURCE="self.CREATE" TARGET="tBQOB.CREATE" />
+      <TEMPLATE_PARAM SOURCE="self.FLUSHONROW"   TARGET="tBQOB.FLUSHONROW" />
+      <TEMPLATE_PARAM SOURCE="self.FLUSHONROW_NUM"   TARGET="tBQOB.FLUSHONROW_NUM" />
+
+      <TEMPLATE_PARAM SOURCE="self.DATA_ACTION"  TARGET="tBQBE.DATA_ACTION" />
+    </TEMPLATES>
+  </CODEGENERATION>
+  <RETURNS/>
+</COMPONENT>

main/plugins/org.talend.designer.components.localprovider/components/tFTPConnection/tFTPConnection_begin.javajet

@@ -228,8 +228,8 @@ if (!sftp && !ftps) { // *** ftp *** //
 		}
 
 		ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
-		ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+		ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 	} catch (Exception e) {
 		<%if(isLog4jEnabled){%>
 			log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPDelete/tFTPDelete_begin.javajet

@@ -249,8 +249,8 @@ int nb_file_<%=cid%> = 0;
 					throw new RuntimeException("Login failed");
 				}
 
-				ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 				ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+				ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			} catch (Exception e) {
 				<%if(isLog4jEnabled){%>
 					log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPFileExist/tFTPFileExist_begin.javajet

@@ -235,8 +235,8 @@ if (sftp) { // *** sftp *** //
 					throw new RuntimeException("Login failed");
 				}
 
-				ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 				ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+				ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		} catch (Exception e) {
 			<%if(isLog4jEnabled){%>
 				log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPFileList/tFTPFileList_begin.javajet

@@ -317,8 +317,8 @@ if (sftp) {// *** sftp *** //
 				throw new RuntimeException("Login failed");
 			}
 
-			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		} catch (Exception e) {
 			<%if(isLog4jEnabled){%>
 				log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPFileProperties/tFTPFileProperties_begin.javajet

@@ -284,8 +284,8 @@ if (sftp) {  // *** sftp *** //
 			}
 
 			ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE); //to calculate md5 for binary files correctly
-			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		} catch (Exception e) {
 			<%if(isLog4jEnabled){%>
 				log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPGet/tFTPGet_begin.javajet

@@ -732,8 +732,8 @@ if (sftp) { // *** sftp *** //
 			}
 
 			ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
-			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		} catch (Exception e) {
 			<%if(isLog4jEnabled){%>
 				log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPPut/tFTPPut_begin.javajet

@@ -415,8 +415,8 @@ int nb_file_<%=cid%> = 0;
 				}
 
 				ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
-				ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 				ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+				ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			} catch (Exception e) {
 				<%if(isLog4jEnabled){%>
 					log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPRename/tFTPRename_begin.javajet

@@ -254,8 +254,8 @@ if (sftp) {  // *** sftp *** //
 				throw new RuntimeException("Login failed");
 			}
 
-			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		} catch (Exception e) {
 			<%if(isLog4jEnabled){%>
 				log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tFTPTruncate/tFTPTruncate_begin.javajet

@@ -252,8 +252,8 @@ if (sftp) {// *** sftp *** //
 				throw new RuntimeException("Login failed");
 			}
 
-			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 			ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
+			ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
 		} catch (Exception e) {
 			<%if(isLog4jEnabled){%>
 				log.error("<%=cid%> - Can't create connection: " + e.getMessage());

main/plugins/org.talend.designer.components.localprovider/components/tGSBucketCreate/tGSBucketCreate_java.xml

@@ -152,8 +152,8 @@
 		<IMPORTS>
 			<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
 	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSBucketDelete/tGSBucketDelete_java.xml

@@ -99,8 +99,8 @@
 	    <IMPORTS>
 			<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
 	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSBucketExist/tGSBucketExist_java.xml

@@ -99,8 +99,8 @@
 	    <IMPORTS>
 			<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
 	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSBucketList/tGSBucketList_java.xml

@@ -102,8 +102,8 @@
 	    <IMPORTS>
 	      	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
    	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSConnection/tGSConnection_java.xml

@@ -62,8 +62,8 @@
 	    <IMPORTS>
 	       	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED="true" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED="true" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED="true" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED="true" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED="true" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED="true" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED="true" />
 		</IMPORTS>
 	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSCopy/tGSCopy_java.xml

@@ -169,8 +169,8 @@
 		<IMPORTS>
 	 		<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 		</IMPORTS>
 	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSDelete/tGSDelete_java.xml

@@ -161,8 +161,8 @@
 	    <IMPORTS>
 	      	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
    	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSGet/tGSGet_java.xml

@@ -198,8 +198,8 @@
 	    <IMPORTS>
 	      	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
    	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSList/tGSList_java.xml

@@ -161,8 +161,8 @@
 	    <IMPORTS>
 	      	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 	    </IMPORTS>
    	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tGSPut/tGSPut_java.xml

@@ -142,8 +142,8 @@
       	<IMPORT NAME="GoogleStorageUtils" MODULE="GoogleStorageUtils.jar" MVN="mvn:org.talend.libraries/GoogleStorageUtils/6.0.0"  REQUIRED="true" />
       	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')" />
     </IMPORTS>
    </CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tHiveRow/tHiveRow_main.javajet

@@ -266,7 +266,9 @@ if(hiveDistrib.isExecutedThroughWebHCat()) {
 			}
 		}
 		%>
-		log.info("Query sent to HiveServer2 to be executed with [Application Name(MapReduce)/DAG Name(Tez): " + queryIdentifier_<%=cid %> + "]");
+		<%if(isLog4jEnabled){%>
+			log.info("Query sent to HiveServer2 to be executed with [Application Name(MapReduce)/DAG Name(Tez): " + queryIdentifier_<%=cid%> + "]");
+		<%}%>
 	    } catch (java.lang.Exception e) {
 	        whetherReject_<%=cid%> = true;
 	        <%

main/plugins/org.talend.designer.components.localprovider/components/tNetezzaNzLoad/tNetezzaNzLoad_begin.javajet

@@ -398,10 +398,15 @@ skeleton="../templates/db_output_bulk.skeleton"
 				
 				// added "print command" feature
 				<% if (printCommand == true) { %>
-				    System.out.println("INFO: executing nzload command:");
-				    for( String cmd : cmds ) {
-					       System.out.print(cmd + " ");
-					 }
+					System.out.println("INFO: executing nzload command:");
+					for(int i = 0; i < cmds.length; i++) {
+						if ("-pw".equals(cmds[i]) && decryptedPwd_<%=cid %>.equals(cmds[i+1])) {
+							System.out.print(cmds[i] + " " + routines.system.PasswordEncryptUtil.PASSWORD_FOR_LOGS_VALUE + " ");
+							i++;
+						} else {
+							System.out.print(cmds[i] + " ");
+						}
+					}
 				<% } %>
 				
 				Runtime rt = Runtime.getRuntime();

main/plugins/org.talend.designer.core.generic/plugin.xml

@@ -104,7 +104,7 @@
        </projecttask>
 		<projecttask
              beforeLogon="false"
-               breaks="7.0.1"
+               breaks="7.0.0"
                class="org.talend.designer.core.generic.model.migration.NewGoogleDriveMigrationTask"
                description="Migration task for jobs using Google Drive components to TCOMP"
                id="org.talend.designer.core.generic.model.migration.NewGoogleDriveMigrationTask"

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/model/Component.java

@@ -1469,9 +1469,11 @@ public class Component extends AbstractBasicComponent {
         }
         if (GenericTypeUtils.isStringType(property)
                 && property.getTaggedValue(IGenericConstants.LINE_SEPARATOR_REPLACED_TO) != null) {
-            String replacedTo = String.valueOf(property.getTaggedValue(IGenericConstants.LINE_SEPARATOR_REPLACED_TO));
-            // "Win", "Linux/Unix", "Mac"
-            return value.replaceAll("\r\n", replacedTo).replaceAll("\n", replacedTo).replaceAll("\r", replacedTo);
+            //process for the sql field for jdbc, snowflake, salesforce, LINE_SEPARATOR_REPLACED_TO key can tell us which a sql type field,
+            //as sql type value may have newline and return characters, which make compiler issue in java code, 
+            //so have to convert the newline characters to visible "\r", "\n" for pass the compiler issue and can't only convert them to white space as TDI-41898
+            //jdbc drivers, salesforce driver can work like that sql : select * \nfrom Account, so it is ok 
+            return NodeUtil.replaceCRLFInMEMO_SQL(value);
         }
         if (GenericTypeUtils.isSchemaType(property)) {
             // Handles embedded escaped quotes which might occur

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/model/GenericTableUtils.java

@@ -62,6 +62,9 @@ public class GenericTableUtils {
                 if(dbService != null){
                     for(String v:values){
                         if(param.getName().equals(EConnectionParameterName.GENERIC_DRIVER_JAR.getDisplayName())){
+                        	if(v == null || v.length() <= 0 || TalendQuoteUtils.removeQuotesIfExist(v).length() <= 0){
+                        		continue;
+                        	}
                             v = dbService.getMVNPath(v);
                         }
                         valueList.add(v);

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/model/migration/NewGoogleDriveMigrationTask.java

@@ -6,12 +6,39 @@ import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.Properties;
 
+import org.talend.core.model.properties.Item;
+import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.designer.core.generic.utils.ParameterUtilTool;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
+import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 
 public class NewGoogleDriveMigrationTask extends org.talend.designer.core.generic.model.migration.NewComponentFrameworkMigrationTask {
 
+    public static final String GOOGLE_DRIVE_PREFIX = "tGoogleDrive";
+
+    public static final String JAVAJET_VERSION = "0.102";
+
+    @Override
+    public ExecutionResult execute(Item item) {
+        ProcessType processType = getProcessType(item);
+        if (processType != null) {
+            for (Object obj : processType.getNode()) {
+                if (obj != null && obj instanceof NodeType) {
+                    String componentName = ((NodeType) obj).getComponentName();
+                    String version = ((NodeType) obj).getComponentVersion();
+                    if (componentName != null && componentName.startsWith(GOOGLE_DRIVE_PREFIX)) {
+                        if (!JAVAJET_VERSION.equals(version)) {
+                            // not a javajet component (tcompv0), we skip this migration for not corrupting comps.
+                            return ExecutionResult.NOTHING_TO_DO;
+                        }
+                    }
+                }
+            }
+        }
+        return super.execute(item);
+    }
+
     @Override
     public Date getOrder() {
         return new GregorianCalendar(2017, 10, 23, 10, 15, 0).getTime();
@@ -33,44 +60,43 @@ public class NewGoogleDriveMigrationTask extends org.talend.designer.core.generi
 
     @Override
     protected ElementParameterType getParameterType(NodeType node, String paramName) {
+        // sanitize inexistent props
+        ElementParameterType oAuthMethod = ParameterUtilTool.findParameterType(node, "OAUTH_METHOD");
+        if (oAuthMethod == null) {
+            ParameterUtilTool.addParameterType(node, "TEXT", "OAUTH_METHOD", "AccessToken");
+        }
+        ElementParameterType dsPath = ParameterUtilTool.findParameterType(node, "DATASTORE_PATH");
+        if (dsPath == null) {
+            String datastorePath = (System.getProperty("user.home", ".") + "/.credentials/talend-googledrive")
+                    .replace("\\", "/");
+            ParameterUtilTool.addParameterType(node, "TEXT", "DATASTORE_PATH",
+                    TalendQuoteUtils.addQuotesIfNotExist(datastorePath));
+        }
         ElementParameterType paramType = ParameterUtilTool.findParameterType(node, paramName);
         if (node != null && paramType != null) {
-            Object value = ParameterUtilTool.convertParameterValue(paramType);
             String componentName = node.getComponentName();
-            //
             if ("tGoogleDriveCopy".equals(componentName)) {
-                if ("FILE_MODE".equals(paramName)) {
-                    if ("true".equals(String.valueOf(value))) {
-                        paramType.setValue("File");
-                    } else {
-                        paramType.setValue("Folder");
-                    }
-                }
-                if ("FILE_NAME".equals(paramName)) {
-                    ElementParameterType fileMode = ParameterUtilTool.findParameterType(node, "FILE_MODE");
-                    Object fileModeValue = ParameterUtilTool.convertParameterValue(fileMode);
-                    if ("true".equals(String.valueOf(fileModeValue))) {
-                        paramType.setValue(String.valueOf(value));
-                    } else {
-                        ElementParameterType folderName = ParameterUtilTool.findParameterType(node, "FOLDER_NAME");
-                        Object folderNameValue = ParameterUtilTool.convertParameterValue(folderName);
-                        paramType.setValue(String.valueOf(folderNameValue));
-                    }
+                Object copyModeParam = ParameterUtilTool.findParameterType(node, "COPY_MODE");
+                if (copyModeParam == null) {
+                    Object fileMode = ParameterUtilTool.getParameterValue(node, "FILE_MODE");
+                    Object fileName = ParameterUtilTool.getParameterValue(node, "FILE_NAME");
+                    Object folderName = ParameterUtilTool.getParameterValue(node, "FOLDER_NAME");
+                    String copyMode = "true".equals(fileMode) ? "File" : "Folder";
+                    String source = "File".equals(copyMode) ? String.valueOf(fileName) : String.valueOf(folderName);
+                    ParameterUtilTool.addParameterType(node, "TEXT", "COPY_MODE", copyMode);
+                    ParameterUtilTool.addParameterType(node, "TEXT", "SOURCE", source);
                 }
             }
             if ("tGoogleDrivePut".equals(componentName)) {
-                if ("READ_CONTENT_FROM_INPUT".equals(paramName)) {
-                    if ("true".equals(String.valueOf(value))) {
-                        paramType.setValue("READ_CONTENT_FROM_INPUT");
-                    } else {
-                        ElementParameterType upl = ParameterUtilTool.findParameterType(node, "UPLOAD_LOCAL_FILE");
-                        Object uplv = ParameterUtilTool.convertParameterValue(upl);
-                        if ("true".equals(String.valueOf(uplv))) {
-                            paramType.setValue("UPLOAD_LOCAL_FILE");
-                        } else {
-                            paramType.setValue("EXPOSE_OUTPUT_STREAM");
-                        }
+                Object uploadModeParam = ParameterUtilTool.findParameterType(node, "UPLOAD_MODE");
+                if (uploadModeParam == null) {
+                    Object isReadContentFromInput = ParameterUtilTool.getParameterValue(node, "READ_CONTENT_FROM_INPUT");
+                    Object uploadLocalFile = ParameterUtilTool.getParameterValue(node, "UPLOAD_LOCAL_FILE");
+                    String uploadMode = "READ_CONTENT_FROM_INPUT";
+                    if (!"true".equals(isReadContentFromInput)) {
+                        uploadMode = "true".equals(uploadLocalFile) ? "UPLOAD_LOCAL_FILE" : "EXPOSE_OUTPUT_STREAM";
                     }
+                    ParameterUtilTool.addParameterType(node, "TEXT", "UPLOAD_MODE", uploadMode);
                 }
             }
         }

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/model/migration/NewGoogleDriveMigrationTask.properties

@@ -9,11 +9,25 @@ tGoogleDriveConnection.useSSL=USE_SSL
 tGoogleDriveConnection.sslAlgorithm=SSL_ALGORITHM
 tGoogleDriveConnection.sslTrustStore=SSL_TRUST_STORE
 tGoogleDriveConnection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDriveConnection.oAuthMethod=OAUTH_METHOD
+tGoogleDriveConnection.datastorePath=DATASTORE_PATH
 
 # tGoogleDriveCopy
 tGoogleDriveCopy=tGoogleDriveCopy
-tGoogleDriveCopy.copyMode=FILE_MODE
-tGoogleDriveCopy.source=FILE_NAME
+tGoogleDriveCopy.connection.referencedComponent=CONNECTION
+tGoogleDriveCopy.connection.applicationName=APPLICATION_NAME
+tGoogleDriveCopy.connection.accessToken=ACCESS_TOKEN
+tGoogleDriveCopy.connection.useProxy=USE_PROXY
+tGoogleDriveCopy.connection.proxyHost=PROXY_HOST
+tGoogleDriveCopy.connection.proxyPort=PROXY_PORT
+tGoogleDriveCopy.connection.useSSL=USE_SSL
+tGoogleDriveCopy.connection.sslAlgorithm=SSL_ALGORITHM
+tGoogleDriveCopy.connection.sslTrustStore=SSL_TRUST_STORE
+tGoogleDriveCopy.connection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDriveCopy.connection.oAuthMethod=OAUTH_METHOD
+tGoogleDriveCopy.connection.datastorePath=DATASTORE_PATH
+tGoogleDriveCopy.copyMode=COPY_MODE
+tGoogleDriveCopy.source=SOURCE
 tGoogleDriveCopy.destinationFolder=DESTINATION_FOLDER_NAME
 tGoogleDriveCopy.rename=RENAME
 tGoogleDriveCopy.newName=DESTINATION_FILE_NAME
@@ -21,16 +35,52 @@ tGoogleDriveCopy.deleteSourceFile=REMOVE_SOURCE_FILE
 
 # tGoogleDriveCreate
 tGoogleDriveCreate=tGoogleDriveCreate
+tGoogleDriveCreate.connection.referencedComponent=CONNECTION
+tGoogleDriveCreate.connection.applicationName=APPLICATION_NAME
+tGoogleDriveCreate.connection.accessToken=ACCESS_TOKEN
+tGoogleDriveCreate.connection.useProxy=USE_PROXY
+tGoogleDriveCreate.connection.proxyHost=PROXY_HOST
+tGoogleDriveCreate.connection.proxyPort=PROXY_PORT
+tGoogleDriveCreate.connection.useSSL=USE_SSL
+tGoogleDriveCreate.connection.sslAlgorithm=SSL_ALGORITHM
+tGoogleDriveCreate.connection.sslTrustStore=SSL_TRUST_STORE
+tGoogleDriveCreate.connection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDriveCreate.connection.oAuthMethod=OAUTH_METHOD
+tGoogleDriveCreate.connection.datastorePath=DATASTORE_PATH
 tGoogleDriveCreate.parentFolder=FOLDER_NAME
 tGoogleDriveCreate.newFolder=NEW_FOLDER_NAME
 
 # tGoogleDriveDelete
 tGoogleDriveDelete=tGoogleDriveDelete
+tGoogleDriveDelete.connection.referencedComponent=CONNECTION
+tGoogleDriveDelete.connection.applicationName=APPLICATION_NAME
+tGoogleDriveDelete.connection.accessToken=ACCESS_TOKEN
+tGoogleDriveDelete.connection.useProxy=USE_PROXY
+tGoogleDriveDelete.connection.proxyHost=PROXY_HOST
+tGoogleDriveDelete.connection.proxyPort=PROXY_PORT
+tGoogleDriveDelete.connection.useSSL=USE_SSL
+tGoogleDriveDelete.connection.sslAlgorithm=SSL_ALGORITHM
+tGoogleDriveDelete.connection.sslTrustStore=SSL_TRUST_STORE
+tGoogleDriveDelete.connection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDriveDelete.connection.oAuthMethod=OAUTH_METHOD
+tGoogleDriveDelete.connection.datastorePath=DATASTORE_PATH
 tGoogleDriveDelete.file=FILE_NAME
 tGoogleDriveDelete.useTrash=USE_TRASH
 
 # tGoogleDriveGet
 tGoogleDriveGet=tGoogleDriveGet
+tGoogleDriveGet.connection.referencedComponent=CONNECTION
+tGoogleDriveGet.connection.applicationName=APPLICATION_NAME
+tGoogleDriveGet.connection.accessToken=ACCESS_TOKEN
+tGoogleDriveGet.connection.useProxy=USE_PROXY
+tGoogleDriveGet.connection.proxyHost=PROXY_HOST
+tGoogleDriveGet.connection.proxyPort=PROXY_PORT
+tGoogleDriveGet.connection.useSSL=USE_SSL
+tGoogleDriveGet.connection.sslAlgorithm=SSL_ALGORITHM
+tGoogleDriveGet.connection.sslTrustStore=SSL_TRUST_STORE
+tGoogleDriveGet.connection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDriveGet.connection.oAuthMethod=OAUTH_METHOD
+tGoogleDriveGet.connection.datastorePath=DATASTORE_PATH
 tGoogleDriveGet.file=FILE_NAME
 tGoogleDriveGet.storeToLocal=STORE_TO_LOCAL
 tGoogleDriveGet.outputFileName=OUTPUT
@@ -43,6 +93,18 @@ tGoogleDriveGet.mainSchema.schema.connector=FLOW->FLOW
 
 # tGoogleDriveList
 tGoogleDriveList=tGoogleDriveList
+tGoogleDriveList.connection.referencedComponent=CONNECTION
+tGoogleDriveList.connection.applicationName=APPLICATION_NAME
+tGoogleDriveList.connection.accessToken=ACCESS_TOKEN
+tGoogleDriveList.connection.useProxy=USE_PROXY
+tGoogleDriveList.connection.proxyHost=PROXY_HOST
+tGoogleDriveList.connection.proxyPort=PROXY_PORT
+tGoogleDriveList.connection.useSSL=USE_SSL
+tGoogleDriveList.connection.sslAlgorithm=SSL_ALGORITHM
+tGoogleDriveList.connection.sslTrustStore=SSL_TRUST_STORE
+tGoogleDriveList.connection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDriveList.connection.oAuthMethod=OAUTH_METHOD
+tGoogleDriveList.connection.datastorePath=DATASTORE_PATH
 tGoogleDriveList.folder=FOLDER_NAME
 tGoogleDriveList.includeSubDirectories=INCLUDSUBDIR
 tGoogleDriveList.listMode=LIST_MODE
@@ -50,7 +112,19 @@ tGoogleDriveList.includeTrashedFiles=INCLUDTRASHFILES
 
 # tGoogleDrivePut
 tGoogleDrivePut=tGoogleDrivePut
-tGoogleDrivePut.uploadMode=READ_CONTENT_FROM_INPUT
+tGoogleDrivePut.connection.referencedComponent=CONNECTION
+tGoogleDrivePut.connection.applicationName=APPLICATION_NAME
+tGoogleDrivePut.connection.accessToken=ACCESS_TOKEN
+tGoogleDrivePut.connection.useProxy=USE_PROXY
+tGoogleDrivePut.connection.proxyHost=PROXY_HOST
+tGoogleDrivePut.connection.proxyPort=PROXY_PORT
+tGoogleDrivePut.connection.useSSL=USE_SSL
+tGoogleDrivePut.connection.sslAlgorithm=SSL_ALGORITHM
+tGoogleDrivePut.connection.sslTrustStore=SSL_TRUST_STORE
+tGoogleDrivePut.connection.sslTrustStorePassword=SSL_TRUST_STORE_PASSWORD
+tGoogleDrivePut.connection.oAuthMethod=OAUTH_METHOD
+tGoogleDrivePut.connection.datastorePath=DATASTORE_PATH
+tGoogleDrivePut.uploadMode=UPLOAD_MODE
 tGoogleDrivePut.fileName=FILE_TITLE
 tGoogleDrivePut.destinationFolder=DESTINATION_FOLDER_NAME
 tGoogleDrivePut.overwrite=OVERWRITE

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/service/GenericService.java

@@ -19,8 +19,13 @@ import java.util.Map;
 import java.util.Set;
 
 import org.talend.components.api.component.ComponentDefinition;
+import org.talend.components.api.properties.ComponentProperties;
+import org.talend.components.api.properties.ComponentReferenceProperties;
+import org.talend.core.model.process.EParameterFieldType;
 import org.talend.core.model.process.IElementParameter;
+import org.talend.core.model.process.INode;
 import org.talend.core.runtime.services.IGenericService;
+import org.talend.daikon.properties.Properties;
 import org.talend.designer.core.generic.model.GenericElementParameter;
 import org.talend.designer.core.generic.utils.ComponentsUtils;
 
@@ -52,5 +57,56 @@ public class GenericService implements IGenericService {
         }
         return genericComponents;
     }
+    
+    @Override
+    public void resetReferenceValue(INode curNode, String oldConnectionName, String newConnectionName) {
+        ComponentReferenceProperties comPro = null;
+        IElementParameter refPara = curNode.getElementParameterFromField(EParameterFieldType.COMPONENT_REFERENCE);
+        if(refPara != null){
+            comPro = ComponentsUtils.getReferencedComponent(refPara);
+        }
+        if(comPro != null){
+            Object sValue = comPro.componentInstanceId.getStoredValue();
+            if (oldConnectionName.equals(sValue)) {
+                comPro.componentInstanceId.setValue(newConnectionName);
+                return;
+            } 
+            if (sValue != null && ((String)sValue).startsWith(oldConnectionName + "_")) { //$NON-NLS-1$
+                comPro.componentInstanceId.setValue(((String)sValue).replaceFirst(oldConnectionName + "_", newConnectionName + "_")); //$NON-NLS-1$
+                return;
+            }
+        }
+        comPro = getComponentReferenceProperties(curNode);
+        if(comPro == null){
+            return;
+        }
+        Object sValue = comPro.componentInstanceId.getStoredValue();
+        if (oldConnectionName.equals(sValue)) {
+            comPro.componentInstanceId.setValue(newConnectionName);
+        } else if (sValue != null && ((String)sValue).startsWith(oldConnectionName + "_")) { //$NON-NLS-1$
+            comPro.componentInstanceId.setValue(((String)sValue).replaceFirst(oldConnectionName + "_", newConnectionName + "_")); //$NON-NLS-1$
+        }
+    }
+    
+    private ComponentReferenceProperties getComponentReferenceProperties(INode curNode){
+        ComponentProperties pros = curNode.getComponentProperties();
+        if(pros == null){
+            return null;
+        }
+        Properties ps = pros.getProperties("referencedComponent"); //$NON-NLS-1$
+        if(ps == null){
+            Properties conn = pros.getProperties("connection"); //$NON-NLS-1$
+            if(conn != null){
+                ps = conn.getProperties("referencedComponent"); //$NON-NLS-1$
+            }
+        }
+        if(ps == null){
+            return null;
+        }
+        if(ps instanceof ComponentReferenceProperties){
+            return (ComponentReferenceProperties)ps;
+        }
+        return null;
+    }
 
 }

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/utils/ComponentsUtils.java

@@ -837,6 +837,20 @@ public class ComponentsUtils {
         }
         return nals;
     }
+    
+    public static ComponentReferenceProperties getReferencedComponent(IElementParameter refPara){
+        if (!(refPara instanceof GenericElementParameter)) {
+            return null;
+        }
+        Widget widget = ((GenericElementParameter) refPara).getWidget();
+        NamedThing widgetProperty = widget.getContent();
+        if (widgetProperty instanceof ComponentReferenceProperties
+                && Widget.COMPONENT_REFERENCE_WIDGET_TYPE.equals(widget.getWidgetType())) {
+            ComponentReferenceProperties props = (ComponentReferenceProperties) widgetProperty;
+            return props;
+        }
+        return null;
+    }
 
     public static void initReferencedComponent(IElementParameter refPara, String newValue) {
 

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/model/process/statsandlogs/StatsAndLogsManager.java

@@ -737,6 +737,12 @@ public class StatsAndLogsManager {
 
                         if (isGeneric) {// reset the show if
                             resetShowIf(connectionNode);
+                            if (checkUrlContainsAutoCommit(connectionNode)) {
+                                IElementParameter autoCommitParam = connectionNode.getElementParameter("autocommit");//$NON-NLS-1$
+                                if (autoCommitParam != null) {
+                                    autoCommitParam.setValue(Boolean.TRUE);
+                                }
+                            }
                         }
 
                         if (connectionComponentName.contains("Oracle")) {//$NON-NLS-1$
@@ -764,13 +770,44 @@ public class StatsAndLogsManager {
                 }
             }
         }
-        DataConnection dataConnec = createDataConnectionForSubJobOK(dataNode, commitNode);
-        ((List<IConnection>) dataNode.getOutgoingConnections()).add(dataConnec);
-        ((List<IConnection>) commitNode.getIncomingConnections()).add(dataConnec);
-
+        boolean noCommitNode = false;
+        if (checkUrlContainsAutoCommit(connectionNode)) {
+            IElementParameter autoCommitParam = connectionNode.getElementParameter("autocommit");//$NON-NLS-1$
+            if (autoCommitParam != null && autoCommitParam.getValue() != null) {
+                noCommitNode = Boolean.parseBoolean(autoCommitParam.getValue().toString());
+                if (noCommitNode && nodeList.contains(commitNode)) {
+                    nodeList.remove(commitNode);
+                }
+            }
+        }
+        if (!noCommitNode) {
+            DataConnection dataConnec = createDataConnectionForSubJobOK(dataNode, commitNode);
+            ((List<IConnection>) dataNode.getOutgoingConnections()).add(dataConnec);
+            ((List<IConnection>) commitNode.getIncomingConnections()).add(dataConnec);
+        }
         return connectionNode;
     }
 
+    private static boolean checkUrlContainsAutoCommit(DataNode connectionNode) {
+        if (connectionNode != null) {
+            boolean isGeneric = connectionNode.getComponent().getComponentType() == EComponentType.GENERIC;
+            if (isGeneric) {
+                IElementParameter urlParam = connectionNode
+                        .getElementParameter(EConnectionParameterName.GENERIC_URL.getDisplayName());
+                if (urlParam != null) {
+                    Object obj = urlParam.getValue();
+                    if (obj != null && obj instanceof String) {
+                        String url = (String) obj;
+                        if (url != null && url.toLowerCase().contains("autocommit=true")) {//$NON-NLS-1$
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
     private static void resetShowIf(DataNode connectionNode) {
         ComponentProperties tcomp_properties = connectionNode.getComponentProperties();
         if (tcomp_properties != null) {

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/ui/editor/nodes/Node.java

@@ -1883,7 +1883,15 @@ public class Node extends Element implements IGraphicalNode {
 
         if (id.equals(EParameterName.CONNECTION_FORMAT.getName())) {
             connectionToParse = (String) value;
-            setConnectionName(ElementParameterParser.parse(this, connectionToParse));
+            boolean isTestCase = false;
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
+                ITestContainerProviderService testContainerService = (ITestContainerProviderService) GlobalServiceRegister
+                        .getDefault().getService(ITestContainerProviderService.class);
+                isTestCase = getProcess() != null && testContainerService.isTestContainerProcess(getProcess());
+            }
+            if (!isTestCase) {
+                setConnectionName(ElementParameterParser.parse(this, connectionToParse));
+            }
             // to check
             // String newValue = ElementParameterParser.parse(this, connectionToParse);
             // setConnectionName(newValue);

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/ui/editor/properties/controllers/ComponentListController.java

@@ -48,6 +48,7 @@ import org.talend.core.model.process.IElement;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess2;
+import org.talend.core.runtime.services.IGenericService;
 import org.talend.core.ui.IJobletProviderService;
 import org.talend.core.ui.process.IGraphicalNode;
 import org.talend.core.ui.properties.tab.IDynamicProperty;
@@ -209,6 +210,20 @@ public class ComponentListController extends AbstractElementPropertySectionContr
                     } else if (value != null && value.startsWith(oldConnectionName + "_")) {
                         curParam.setValue(value.replaceFirst(oldConnectionName + "_", newConnectionName + "_"));
                     }
+                }else if(curParam.getFieldType().equals(EParameterFieldType.COMPONENT_REFERENCE)){
+                    String value = (String)curParam.getValue();
+                    if (oldConnectionName.equals(value)) {
+                        curParam.setValue(newConnectionName);
+                    } else if (value != null && value.startsWith(oldConnectionName + "_")) {
+                        curParam.setValue(value.replaceFirst(oldConnectionName + "_", newConnectionName + "_"));
+                    }
+                    if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericService.class)) {
+                        IGenericService genericService = (IGenericService) GlobalServiceRegister.getDefault().getService(
+                                IGenericService.class);
+                        if(genericService != null){
+                            genericService.resetReferenceValue(curNode, oldConnectionName, newConnectionName);
+                        }
+                    }
                 } else if (curParam.getFieldType().equals(EParameterFieldType.TABLE)) {
                     final Object[] itemsValue = curParam.getListItemsValue();
                     for (Object element : itemsValue) {

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/utils/BigDataJobUtil.java

@@ -10,27 +10,29 @@
 // 9 rue Pages 92150 Suresnes, France
 //
 // ============================================================================
-package org.talend.designer.runprocess.bigdata;
+package org.talend.designer.core.utils;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
+import java.util.Collection;
 import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Set;
 
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
-import org.eclipse.emf.common.util.EList;
 import org.osgi.framework.Bundle;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.hadoop.HadoopConstants;
 import org.talend.core.hadoop.version.EHadoopDistributions;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.model.properties.ProcessItem;
-import org.talend.core.model.properties.Property;
+import org.talend.core.model.process.IElementParameter;
+import org.talend.core.model.process.IProcess;
+import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.repository.ERepositoryObjectType;
-import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 
 /**
  * created by nrousseau on Mar 24, 2018 Detailled comment
@@ -38,10 +40,10 @@ import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
  */
 public class BigDataJobUtil {
 
-    private ProcessItem processItem;
+    private IProcess process;
 
-    public BigDataJobUtil(ProcessItem processItem) {
-        this.processItem = processItem;
+    public BigDataJobUtil(IProcess process) {
+        this.process = process;
     }
 
     /**
@@ -68,11 +70,10 @@ public class BigDataJobUtil {
     public boolean isSparkWithHDInsight() {
         boolean isSparkWithHDInsight = false;
         if (isBDJobWithFramework(ERepositoryObjectType.PROCESS_MR, HadoopConstants.FRAMEWORK_SPARK)
-                || isBDJobWithFramework(ERepositoryObjectType.PROCESS_STORM,
-                        HadoopConstants.FRAMEWORK_SPARKSTREAMING)) {
-            EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
+                || isBDJobWithFramework(ERepositoryObjectType.PROCESS_STORM, HadoopConstants.FRAMEWORK_SPARKSTREAMING)) {
+            List<? extends IElementParameter> parameters = process.getElementParametersWithChildrens();
             boolean modeParameterVisited = false;
-            for (ElementParameterType pt : parameters) {
+            for (IElementParameter pt : parameters) {
                 if (pt.getName().equals("SPARK_LOCAL_MODE")) { //$NON-NLS-1$
                     modeParameterVisited = true;
                     if ("true".equals(pt.getValue())) { //$NON-NLS-1$
@@ -96,11 +97,11 @@ public class BigDataJobUtil {
 
     public boolean isMRWithHDInsight() {
         Boolean isMRWithHDInsight = false;
-        if (processItem != null) {
+        if (process != null) {
             isMRWithHDInsight = false;
             if (isBDJobWithFramework(ERepositoryObjectType.PROCESS_MR, HadoopConstants.FRAMEWORK_MAPREDUCE)) {
-                EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
-                for (ElementParameterType pt : parameters) {
+                List<? extends IElementParameter> parameters = process.getElementParametersWithChildrens();
+                for (IElementParameter pt : parameters) {
                     if (pt.getName().equals("DISTRIBUTION") //$NON-NLS-1$
                             && EHadoopDistributions.MICROSOFT_HD_INSIGHT.getName().equals(pt.getValue())) {
                         isMRWithHDInsight = true;
@@ -119,11 +120,10 @@ public class BigDataJobUtil {
         Boolean isSparkInYarnClusterMode = false;
         // Test if we are in Spark or Spark streaming
         if (isBDJobWithFramework(ERepositoryObjectType.PROCESS_MR, HadoopConstants.FRAMEWORK_SPARK)
-                || isBDJobWithFramework(ERepositoryObjectType.PROCESS_STORM,
-                        HadoopConstants.FRAMEWORK_SPARKSTREAMING)) {
+                || isBDJobWithFramework(ERepositoryObjectType.PROCESS_STORM, HadoopConstants.FRAMEWORK_SPARKSTREAMING)) {
 
-            EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
-            for (ElementParameterType pt : parameters) {
+            List<? extends IElementParameter> parameters = process.getElementParametersWithChildrens();
+            for (IElementParameter pt : parameters) {
                 if (HadoopConstants.SPARK_MODE.equals(pt.getName())
                         && HadoopConstants.SPARK_MODE_YARN_CLUSTER.equals(pt.getValue())) {
                     isSparkInYarnClusterMode = true;
@@ -135,31 +135,23 @@ public class BigDataJobUtil {
     }
 
     private boolean isBDJobWithFramework(ERepositoryObjectType objectType, String frameworkName) {
-        if (processItem != null) {
-            // Storm/SparkStreaming(PROCESS_STORM), MR/Spark(PROCESS_MR)
-            if (ERepositoryObjectType.getItemType(processItem).equals(objectType)) { // have same type
-                Property property = processItem.getProperty();
-                if (property != null && property.getAdditionalProperties() != null
-                        && frameworkName.equals(property.getAdditionalProperties().get(HadoopConstants.FRAMEWORK))) {
-                    return true;
-                }
-            }
+        // Storm/SparkStreaming(PROCESS_STORM), MR/Spark(PROCESS_MR)
+        if (process != null && process instanceof IProcess2 && ((IProcess2) process).getAdditionalProperties() != null
+                && frameworkName.equals(((IProcess2) process).getAdditionalProperties().get(HadoopConstants.FRAMEWORK))) {
+            return true;
         }
         return false;
     }
 
     /** Find the distribution where the generated jar rquired to have the context files inside **/
     public boolean needsToHaveContextInsideJar() {
-        if (processItem.getProcess() != null && processItem.getProcess().getParameters() != null) {
-            EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
-            for (ElementParameterType pt : parameters) {
+        List<? extends IElementParameter> parameters = process.getElementParametersWithChildrens();
+
+        if (process != null && parameters != null) {
+
+            for (IElementParameter pt : parameters) {
                 if (pt.getName().equals("DISTRIBUTION")) { //$NON-NLS-1$
-                    String value = pt.getValue();
-                    if ("MICROSOFT_HD_INSIGHT".equals(value) //$NON-NLS-1$
-                            || "GOOGLE_CLOUD_DATAPROC".equals(value) //$NON-NLS-1$
-                            || "CLOUDERA_ALTUS".equals(value)) { //$NON-NLS-1$
-                        return true;
-                    }
+                    return true;
                 }
             }
             if (isSparkWithYarnClusterMode()) {
@@ -169,6 +161,27 @@ public class BigDataJobUtil {
         return false;
     }
 
+    public void setExcludedModules(Collection<ModuleNeeded> modulesNeeded) {
+        if (isMRWithHDInsight() || isSparkWithHDInsight()) {
+            // we need to exclude every non-MR Required jars.
+            for (ModuleNeeded currentModule : modulesNeeded) {
+                if (currentModule.isMrRequired()) {
+                    currentModule.setExcluded(true);
+                }
+            }
+        }
+    }
+
+    public void removeExcludedModules(Collection<ModuleNeeded> modulesNeeded) {
+        Iterator<ModuleNeeded> itModules = modulesNeeded.iterator();
+        while (itModules.hasNext()) {
+            ModuleNeeded module = itModules.next();
+            if (module.isExcluded()) {
+                itModules.remove();
+            }
+        }
+    }
+
     /**
      * DOC nrousseau Comment method "getShadedModulesExclude".
      * 
@@ -182,6 +195,8 @@ public class BigDataJobUtil {
             for (ModuleNeeded currentModule : modulesNeeded) {
                 if (!currentModule.isMrRequired()) {
                     excludedModules.add(currentModule);
+                } else {
+                    currentModule.setExcluded(true);
                 }
             }
         }

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/utils/JavaProcessUtil.java

@@ -27,12 +27,9 @@ import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.Path;
 import org.talend.core.CorePlugin;
-import org.talend.core.GlobalServiceRegister;
-import org.talend.core.ILibraryManagerService;
 import org.talend.core.hadoop.IHadoopClusterService;
 import org.talend.core.hadoop.repository.HadoopRepositoryUtil;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.process.EParameterFieldType;
 import org.talend.core.model.process.IContext;
 import org.talend.core.model.process.IContextParameter;
@@ -106,6 +103,10 @@ public class JavaProcessUtil {
             }
         }
 
+        if (BitwiseOptionUtils.containOption(options, TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED)) {
+            new BigDataJobUtil(process).removeExcludedModules(modulesNeeded);
+        }
+
         return new HashSet<ModuleNeeded>(modulesNeeded);
     }
 
@@ -208,6 +209,7 @@ public class JavaProcessUtil {
         if (hadoopItemId != null) {
             useCustomConfsJarIfNeeded(modulesNeeded, hadoopItemId);
         }
+        new BigDataJobUtil(process).setExcludedModules(modulesNeeded);
     }
 
     public static String getHadoopClusterItemId(INode node) {

main/plugins/org.talend.designer.dbmap/src/main/java/org/talend/designer/dbmap/language/generation/DbGenerationManager.java

@@ -27,8 +27,12 @@ import java.util.regex.PatternSyntaxException;
 import org.talend.commons.ui.runtime.exception.ExceptionHandler;
 import org.talend.commons.utils.StringUtils;
 import org.talend.commons.utils.data.text.StringHelper;
+import org.talend.core.database.EDatabaseTypeName;
+import org.talend.core.model.metadata.Dbms;
 import org.talend.core.model.metadata.IMetadataColumn;
 import org.talend.core.model.metadata.IMetadataTable;
+import org.talend.core.model.metadata.MappingTypeRetriever;
+import org.talend.core.model.metadata.MetadataTalendType;
 import org.talend.core.model.metadata.MetadataToolHelper;
 import org.talend.core.model.process.EConnectionType;
 import org.talend.core.model.process.IConnection;
@@ -40,6 +44,7 @@ import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.model.utils.TalendTextUtils;
+import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.designer.core.model.components.EParameterName;
 import org.talend.designer.dbmap.DbMapComponent;
 import org.talend.designer.dbmap.external.data.ExternalDbMapData;
@@ -436,6 +441,7 @@ public abstract class DbGenerationManager {
             }
 
             StringBuilder sbWhere = new StringBuilder();
+            this.tabSpaceString = DEFAULT_TAB_SPACE_STRING;
             boolean isFirstClause = true;
             for (int i = 0; i < lstSizeInputTables; i++) {
                 ExternalDbMapTable inputTable = inputTables.get(i);
@@ -443,7 +449,6 @@ public abstract class DbGenerationManager {
                     isFirstClause = false;
                 }
             }
-
             /*
              * for addition conditions
              */
@@ -481,16 +486,19 @@ public abstract class DbGenerationManager {
                         }
                     }
                 }
+                
                 List<ExternalDbMapEntry> customOtherConditionsEntries = outputTable.getCustomOtherConditionsEntries();
                 if (customOtherConditionsEntries != null) {
                     for (ExternalDbMapEntry entry : customOtherConditionsEntries) {
                         String exp = initExpression(component, entry);
                         if (exp != null && !DbMapSqlConstants.EMPTY.equals(exp.trim())) {
+                            exp = replaceVariablesForExpression(component, exp);
                             otherAddition.add(exp);
                         }
                     }
                 }
             }
+            this.tabSpaceString = tabString;
 
             String whereClauses = sbWhere.toString();
 
@@ -1016,6 +1024,8 @@ public abstract class DbGenerationManager {
     }
 
     protected String initExpression(DbMapComponent component, ExternalDbMapEntry dbMapEntry) {
+    	String quote = getQuote(component);
+    	String quto_mark = TalendQuoteUtils.QUOTATION_MARK;
         String expression = dbMapEntry.getExpression();
         if (expression != null) {
             List<Map<String, String>> itemNameList = null;
@@ -1040,7 +1050,7 @@ public abstract class DbGenerationManager {
                 itemNameList = mapParser2.parseInTableEntryLocations(expression);
             }
 
-            String quoParser = "[\\\\]?\\\""; //$NON-NLS-1$
+            String quto_markParser = "[\\\\]?\\" + quto_mark; //$NON-NLS-1$
             for (Map<String, String> itemNamemap : itemNameList) {
                 Set<Entry<String, String>> set = itemNamemap.entrySet();
                 Iterator<Entry<String, String>> ite = set.iterator();
@@ -1105,13 +1115,13 @@ public abstract class DbGenerationManager {
                                             continue;
                                         }
                                         if (expression.trim().equals(tableValue + "." + oriName)) {
-                                            expression = tableValue + "." + getColumnName(iconn, oriName);
-                                            expression = expression.replaceAll(quoParser,"\\\\\""); //$NON-NLS-1$
+                                            expression = tableValue + "." + getColumnName(iconn, oriName, quote);
+                                            expression = expression.replaceAll(quto_markParser,"\\\\" +quto_mark); //$NON-NLS-1$
                                             continue;
                                         }
                                         if (expression.trim().equals(originaltableName + "." + oriName)) {
-                                            expression = originaltableName + "." + getColumnName(iconn, oriName);
-                                            expression = expression.replaceAll(quoParser,"\\\\\""); //$NON-NLS-1$
+                                            expression = originaltableName + "." + getColumnName(iconn, oriName, quote);
+                                            expression = expression.replaceAll(quto_markParser,"\\\\" +quto_mark); //$NON-NLS-1$
                                             continue;
                                         }
                                         // if it is temp delived table, use label to generate sql
@@ -1119,13 +1129,13 @@ public abstract class DbGenerationManager {
                                             continue;
                                         }
                                         if (!isRefTableConnection(iconn) && isUseDelimitedIdentifiers()) {
-                                            oriName = getColumnName(iconn, oriName);
+                                            oriName = getColumnName(iconn, oriName, quote);
                                         } else {
                                             oriName = oriName.replaceAll("\\$", "\\\\\\$"); //$NON-NLS-1$ //$NON-NLS-2$
                                         }
-                                        expression = expression.replaceFirst("\\." + co.getLabel(), //$NON-NLS-1$
-                                                "\\." + oriName); //$NON-NLS-1$
-                                        expression = expression.replaceAll(quoParser,"\\\\\""); //$NON-NLS-1$
+                                        expression = expression.replaceFirst(tableValue + "\\." + co.getLabel(), //$NON-NLS-1$
+                                        		tableValue + "\\." + oriName); //$NON-NLS-1$
+                                        expression = replaceAuotes(expression, quto_markParser, quto_mark);
                                     }
                                 }
 
@@ -1139,6 +1149,59 @@ public abstract class DbGenerationManager {
 
         return expression;
     }
+    
+    protected String replaceAuotes(String expression, String quoParser, String quote){
+        if(!expression.contains("'")){
+            return expression.replaceAll(quoParser,"\\\\" +quote); //$NON-NLS-1$
+        }
+        
+        List<Integer> indexs = new ArrayList<>();
+        for(int i=0;i<expression.length();i++){
+            if("'".equals(String.valueOf(expression.charAt(i)))){
+                indexs.add(i);
+            }
+        }
+        StringBuffer result = new StringBuffer();
+        int start = 0;
+        for(int i=0;i<indexs.size();i++){
+            if(i == 0){
+                if(indexs.size() == 1 && indexs.get(i) == 0){
+                    result.append(expression.substring(0, indexs.get(i)+1)); 
+                }else{
+                    String exp = expression.substring(start, indexs.get(i));
+                    result.append(exp.replaceAll(quoParser,"\\\\" +quote)); //$NON-NLS-1$
+                }
+            }
+            if(i % 2 == 0 && i != indexs.size() - 1){
+                result.append(expression.substring(indexs.get(i), indexs.get(i+1)+1));
+                start = indexs.get(i+1)+1;
+            }else if(i == indexs.size() - 1){
+                String exp = expression.substring(indexs.get(i)+1, expression.length());
+                result.append(exp.replaceAll(quoParser,"\\\\" +quote)); //$NON-NLS-1$
+            }
+        }
+        return result.toString();
+    }
+    
+    private String getQuote(DbMapComponent component){
+    	String quote = TalendQuoteUtils.QUOTATION_MARK;
+    	IElementParameter mappingPara = component.getElementParameter(EParameterName.MAPPING.getName());
+    	if(mappingPara == null){
+    		return quote;
+    	}
+    	String mapping = (String) mappingPara.getValue();
+    	if(mapping == null){
+    		return quote;
+    	}
+		MappingTypeRetriever mappingTypeRetriever = MetadataTalendType.getMappingTypeRetriever(mapping);
+		if (mappingTypeRetriever == null) {
+			return quote;
+		}
+		Dbms dbms = mappingTypeRetriever.getDbms();
+		String product = dbms.getProduct();
+		EDatabaseTypeName type = EDatabaseTypeName.getTypeFromProductName(product);
+		return TalendQuoteUtils.getQuoteByDBType(type);
+    }
 
     private String getOriginalColumnName(String entryName, DbMapComponent component, ExternalDbMapTable table) {
         List<IConnection> inputConnections = (List<IConnection>) component.getIncomingConnections();
@@ -1247,6 +1310,14 @@ public abstract class DbGenerationManager {
             return name;
         }
     }
+    
+    protected String getColumnName(IConnection conn, String name, String quote) {
+        if (!isRefTableConnection(conn) && isUseDelimitedIdentifiers()) {
+            return getNameWithDelimitedIdentifier(name, quote);
+        } else {
+            return name;
+        }
+    }
 
     protected boolean isRefTableConnection(IConnection conn) {
         return conn != null && conn.getLineStyle() == EConnectionType.TABLE_REF;
@@ -1259,6 +1330,13 @@ public abstract class DbGenerationManager {
         newName = delimitedIdentifier + newName + delimitedIdentifier;
         return newName;
     }
+    
+    protected String getNameWithDelimitedIdentifier(String name, String quote) {
+        String newName = name;
+        newName = newName.replaceAll("\"", "\"\"");
+        newName = quote + newName + quote;
+        return newName;
+    }
 
     protected String getDelimitedIdentifiers() {
         return "\""; //$NON-NLS-1$

main/plugins/org.talend.designer.dbmap/src/main/java/org/talend/designer/dbmap/language/hive/HiveGenerationManager.java

@@ -12,6 +12,9 @@
 // ============================================================================
 package org.talend.designer.dbmap.language.hive;
 
+import java.util.List;
+import java.util.Set;
+
 import org.talend.designer.dbmap.DbMapComponent;
 import org.talend.designer.dbmap.external.data.ExternalDbMapData;
 import org.talend.designer.dbmap.external.data.ExternalDbMapTable;
@@ -53,4 +56,36 @@ public class HiveGenerationManager extends DbGenerationManager {
         }
         return query;
     }
+    
+    @Override
+    protected String replaceVariablesForExpression(DbMapComponent component, String expression) {
+        if (expression == null) {
+            return null;
+        }
+
+            List<String> contextList = getContextList(component);
+            boolean haveReplace = false;
+            for (String context : contextList) {
+                if (expression.contains(context)) {
+                    expression = expression.replaceAll("\\b" + context + "\\b", "\" +" + context + "+ \""); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+                    haveReplace = true;
+                }
+            }
+            if (!haveReplace) {
+                List<String> connContextList = getConnectionContextList(component);
+                for (String context : connContextList) {
+                    if (expression.contains(context)) {
+                        expression = expression.replaceAll("\\b" + context + "\\b", "\" +" + context + "+ \""); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+                    }
+                }
+            }
+            Set<String> globalMapList = getGlobalMapList(component, expression);
+            for (String globalMapStr : globalMapList) {
+                String regex = parser.getGlobalMapExpressionRegex(globalMapStr);
+                String replacement = parser.getGlobalMapReplacement(globalMapStr);
+                expression = expression.replaceAll(regex, "\" +" + replacement + "+ \""); //$NON-NLS-1$ //$NON-NLS-2$ 
+            }
+        
+        return expression;
+    }
 }

main/plugins/org.talend.designer.dbmap/src/main/java/org/talend/designer/dbmap/language/oracle/OracleGenerationManager.java

@@ -194,6 +194,7 @@ public class OracleGenerationManager extends DbGenerationManager {
             }
 
             StringBuilder sbWhere = new StringBuilder();
+            this.tabSpaceString = DEFAULT_TAB_SPACE_STRING;
             boolean isFirstClause = true;
             for (int i = 0; i < lstSizeInputTables; i++) {
                 ExternalDbMapTable inputTable = inputTables.get(i);
@@ -201,6 +202,7 @@ public class OracleGenerationManager extends DbGenerationManager {
                     isFirstClause = false;
                 }
             }
+            this.tabSpaceString = tabString;
 
             appendSqlQuery(sb, DbMapSqlConstants.NEW_LINE);
             appendSqlQuery(sb, tabSpaceString);
@@ -281,6 +283,7 @@ public class OracleGenerationManager extends DbGenerationManager {
             List<String> otherAddition = new ArrayList<String>();
 
             if (outputTable != null) {
+                this.tabSpaceString = DEFAULT_TAB_SPACE_STRING;
                 List<ExternalDbMapEntry> customWhereConditionsEntries = outputTable.getCustomWhereConditionsEntries();
                 if (customWhereConditionsEntries != null) {
                     for (ExternalDbMapEntry entry : customWhereConditionsEntries) {
@@ -308,10 +311,12 @@ public class OracleGenerationManager extends DbGenerationManager {
                     for (ExternalDbMapEntry entry : customOtherConditionsEntries) {
                         String exp = initExpression(component, entry);
                         if (exp != null && !DbMapSqlConstants.EMPTY.equals(exp.trim())) {
+                            exp = replaceVariablesForExpression(component, exp);
                             otherAddition.add(exp);
                         }
                     }
                 }
+                this.tabSpaceString = tabString;
             }
 
             String whereClauses = sbWhere.toString();

main/plugins/org.talend.designer.dbmap/src/main/java/org/talend/designer/dbmap/utils/DataMapExpressionParser.java

@@ -62,9 +62,11 @@ public class DataMapExpressionParser {
 
     // ((String)globalMap.get("tableName")).columnName
     private final static String GLOBALMAP_EXPRESSION4 = "(" + GLOBALMAP_PATTERN + COLUMN_PATTERN + ")";
-
+    
+    private final static String GLOBALMAP_EXPRESSION5 = "\\s*(\\s*(\\w+)\\s*\\.\\\\\"\\s*(.+?)\\s*\\\\\")\\s*";
+    
     private final static String GLOBALMAP_EXPRESSION = GLOBALMAP_EXPRESSION1 + "|" + GLOBALMAP_EXPRESSION2 + "|"//$NON-NLS-1$//$NON-NLS-2$
-            + GLOBALMAP_EXPRESSION3 + "|" + GLOBALMAP_EXPRESSION4;//$NON-NLS-1$
+            + GLOBALMAP_EXPRESSION3 + "|" + GLOBALMAP_EXPRESSION4 + "|" + GLOBALMAP_EXPRESSION5;//$NON-NLS-1$
 
     private final static String EXPRESSION_PATTERN = "(\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*)" //$NON-NLS-1$
             + "|(\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(\\w+)\\s*)" //$NON-NLS-1$
@@ -143,6 +145,10 @@ public class DataMapExpressionParser {
                 } else if (matchResult.group(35) != null) {
                     // ((String)globalMap.get("tableName")).columnName
                     location = new TableEntryLocation(matchResult.group(36), matchResult.group(38));
+                }else if (matchResult.group(39) != null) {
+                 // table.\"column\"
+                	// Case NVL(Keys.\"ORG_SCD_ID\",-99) When -99 Then 'N' Else 'Y' End
+                    location = new TableEntryLocation(matchResult.group(40), matchResult.group(41));
                 }
                 if (location != null) {
                     resultList.add(location);

main/plugins/org.talend.designer.mapper/src/main/java/org/talend/designer/mapper/MapperComponent.java

@@ -478,12 +478,18 @@ public class MapperComponent extends AbstractMapComponent implements IHashableIn
         ExternalMapperData data = mapperMain.buildExternalData();
         if (mapperMain != null && data != null) {
             if (externalData != null) {
-                this.externalData = data;// fwang fixed bug TDI-8027
+                if(!isConnectionEmpty()) {
+                    this.externalData = data;// fwang fixed bug TDI-8027
+                }
                 MapperHelper.saveDataToEmf(data, emfMapperData);
             }
         }
         return emfMapperData;
     }
+    
+    private boolean isConnectionEmpty() {
+        return getIncomingConnections().isEmpty() && getOutgoingConnections().isEmpty();
+    }
 
     public void renameInputConnection(String oldConnectionName, String newConnectionName) {
         if (oldConnectionName == null || newConnectionName == null) {

main/plugins/org.talend.designer.mapper/src/main/java/org/talend/designer/mapper/ui/visualmap/table/DataMapTableView.java

@@ -1399,7 +1399,7 @@ public abstract class DataMapTableView extends Composite implements IDataMapTabl
         if (mapperManager.componentIsReadOnly()) {
             metadataEditorView.setReadOnly(true);
         } else {
-            metadataEditorView.setReadOnly(isRepository || metadataEditorView.isReadOnly());
+            metadataEditorView.setReadOnly(isRepository);
         }
     }
 

main/plugins/org.talend.designer.maven.job/resources/templates/osgi/assembly_job_template.xml

@@ -13,7 +13,7 @@
             <directory>${project.build.outputDirectory}</directory>
             <outputDirectory>${file.separator}</outputDirectory>
             <includes>
-                <include>${talend.job.path}/**</include>
+                <include>${talend.job.path}/**/*.class</include>
                 <include>__tdm/**</include>
             </includes>
         </fileSet>
@@ -55,15 +55,5 @@
             </includes>
         </fileSet>
     </fileSets>
-    <dependencySets>
-        <!-- for all jobs -->
-        <dependencySet>
-            <includes>
-                <!--@JobIncludes@-->
-            </includes>
-            <outputFileNameMapping>${artifact.build.finalName}.${artifact.extension}</outputFileNameMapping>
-            <useProjectArtifact>true</useProjectArtifact>
-            <unpack>true</unpack>
-        </dependencySet>
-    </dependencySets>
+    <dependencySets></dependencySets>
 </assembly>

main/plugins/org.talend.designer.maven.job/resources/templates/osgi/assembly_route_template.xml

@@ -15,7 +15,7 @@
 			<directory>${project.build.outputDirectory}</directory>
 			<outputDirectory>${file.separator}</outputDirectory>
 			<includes>
-				<include>${talend.job.path}/**</include>
+				<include>${talend.job.path}/**/*.class</include>
 				<include>__tdm/**</include>
 			</includes>
 		</fileSet>

main/plugins/org.talend.designer.maven.job/resources/templates/standalone/assembly_job_template.xml

@@ -209,32 +209,5 @@
 			</includes>
 		</fileSet>
 	</fileSets>
-	<dependencySets>
-		<dependencySet>
-			<outputDirectory>lib</outputDirectory>
-			<includes>
-				<!--@TalendLibIncludes@-->
-			</includes>
-			<!-- without version for talend libraries -->
-			<outputFileNameMapping>${artifact.artifactId}.${artifact.extension}
-			</outputFileNameMapping>
-			<useProjectArtifact>false</useProjectArtifact>
-		</dependencySet>
-		<dependencySet>
-			<outputDirectory>lib</outputDirectory>
-			<includes>
-				<!--@3rdPartyLibIncludes@-->
-			</includes>
-			<useProjectArtifact>false</useProjectArtifact>
-		</dependencySet>
-		<!-- for all jobs -->
-		<dependencySet>
-			<outputDirectory>${talend.job.name}</outputDirectory>
-			<includes>
-				<!--@JobIncludes@-->
-			</includes>
-			<outputFileNameMapping>${artifact.build.finalName}.${artifact.extension}</outputFileNameMapping>
-			<useProjectArtifact>true</useProjectArtifact>
-		</dependencySet>
-	</dependencySets>
+	<dependencySets></dependencySets>
 </assembly>

main/plugins/org.talend.designer.publish.core/src/org/talend/designer/publish/core/actions/AbstractPublishJobAction.java

@@ -112,6 +112,9 @@ public abstract class AbstractPublishJobAction implements IRunnableWithProgress
             // TDI-32861, because for publish job, so means, must be binaries
             exportChoiceMap.put(ExportChoice.binaries, true);
             exportChoiceMap.put(ExportChoice.includeLibs, true);
+            
+            // TESB-26145/TESB-26086 adding context to published job
+            exportChoiceMap.put(ExportChoice.needContext, true);
 
             ProcessItem processItem = (ProcessItem) node.getObject().getProperty().getItem();
             exportItemForDQComponent(processItem);

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/DefaultRunProcessService.java

@@ -712,7 +712,19 @@ public class DefaultRunProcessService implements IRunProcessService {
 
     @Override
     public ITalendProcessJavaProject getTalendCodeJavaProject(ERepositoryObjectType type, String projectTechName) {
-        return TalendJavaProjectManager.getTalendCodeJavaProject(type, projectTechName);
+        ITalendProcessJavaProject codeProject = TalendJavaProjectManager.getTalendCodeJavaProject(type, projectTechName);
+        if (codeProject == null) {
+            // try to recover from any damage of pom.
+            try {
+                AggregatorPomsHelper helper = new AggregatorPomsHelper(projectTechName);
+                IFile pomFile = helper.getCodeFolder(type).getFile(TalendMavenConstants.POM_FILE_NAME);
+                helper.updateCodeProjectPom(new NullProgressMonitor(), type, pomFile);
+                codeProject = TalendJavaProjectManager.getTalendCodeJavaProject(type, projectTechName);
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return codeProject;
     }
 
     @Override
@@ -742,6 +754,10 @@ public class DefaultRunProcessService implements IRunProcessService {
         return ProcessorUtilities.isExportConfig();
     }
 
+    public boolean isdebug() {
+        return ProcessorUtilities.isdebug();
+    }
+
     @Override
     public void buildCodesJavaProject(IProgressMonitor monitor) {
         try {

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/RunProcessContext.java

@@ -54,6 +54,7 @@ import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.ComponentUtilities;
 import org.talend.core.model.process.IConnection;
 import org.talend.core.model.process.IContext;
+import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IPerformance;
@@ -66,7 +67,6 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.runprocess.IEclipseProcessor;
 import org.talend.core.model.runprocess.data.PerformanceData;
-import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.model.utils.TalendTextUtils;
 import org.talend.core.runtime.process.TalendProcessArgumentConstant;
 import org.talend.designer.core.model.components.EParameterName;
@@ -1911,37 +1911,61 @@ public class RunProcessContext {
      * Foe ESB related processes, add a message to the console indicating the endpoint (domain + port) currently used.
      */
     private void addEndpointURL() {
+
+        String defaultRestUri = Platform
+                .getPreferencesService()
+                .getString("org.talend.designer.esb.components.rs.provider", "restServiceDefaultUri",
+                        "http://127.0.0.1:8090/", null);
+
         Collection<NodeType> restComponents = EmfModelUtils
                 .getComponentsByName((ProcessItem) process.getProperty().getItem(), "cREST", "tRESTRequest");
         if (!restComponents.isEmpty() && running) {
             NodeType restComponent = restComponents.iterator().next();
             String endpoint;
             String url = null;
+
             if ("cREST".equals(restComponent.getComponentName()))
                 endpoint = ComponentUtilities.getNodePropertyValue(restComponent, "URL");
             else
                 endpoint = ComponentUtilities.getNodePropertyValue(restComponent, "REST_ENDPOINT");
+
+            String decodedEndpoint = "";
+
             if (!StringUtils.isEmpty(endpoint)) {
-                if (TalendTextUtils.removeQuotes(endpoint).startsWith("http"))
-                    url = TalendTextUtils.removeQuotes(endpoint);
-                else if (ContextParameterUtils.containContextVariables(endpoint)) {
-                    String variable = ContextParameterUtils.getVariableFromCode(endpoint);
-                    if (selectedContext != null) {
-                        url = TalendTextUtils.removeQuotes(selectedContext.getContextParameter(variable).getValue());
+
+                String[] allStrings = endpoint.split("[\\+]");
+
+                for (String endpointElement : allStrings) {
+
+                    endpointElement = endpointElement.trim();
+
+                    if (endpointElement.startsWith("context.")) {
+
+                        // Context parameter
+                        String contextParamId = endpointElement.replaceFirst("context.", "");
+
+                        for (IContextParameter param : selectedContext.getContextParameterList()) {
+                            if (param.getName().equals(contextParamId)) {
+                                decodedEndpoint += TalendTextUtils
+                                        .removeQuotes(selectedContext.getContextParameter(param.getName()).getValue());
+                                break;
+                            }
+                        }
+
+                    } else {
+                        decodedEndpoint += TalendTextUtils.removeQuotes(endpointElement);
                     }
+                }
+
+                if (decodedEndpoint.startsWith("http")) {
+                    url = decodedEndpoint;
                 } else {
-                    String defaultRestUri = Platform
-                            .getPreferencesService()
-                            .getString("org.talend.designer.esb.components.rs.provider", "restServiceDefaultUri",
-                                    "http://127.0.0.1:8090/", null);
-                    endpoint = TalendTextUtils.removeQuotes(endpoint);
-                    if (endpoint.startsWith("/"))
-                        endpoint = endpoint.substring(1);
-                    String fullURL = defaultRestUri + endpoint;
+                    String fullURL = defaultRestUri + decodedEndpoint;
                     url = fullURL.replaceAll("(?<!(http:|https:))//", "/");
                 }
+
                 if (url != null)
-                    addMessage(new ProcessMessage(MsgType.CORE_OUT, "Endpoint deployed at: " + url));
+                    addMessage(new ProcessMessage(MsgType.STD_OUT, "Endpoint deployed at: " + url));
             }
 
         }

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/RunProcessService.java

@@ -478,4 +478,10 @@ public class RunProcessService implements IRunProcessService {
         delegateService.handleJobDependencyLoop(mainJobInfo, listJobs, progressMonitor);
     }
 
+    @Override
+    public boolean isdebug() {
+        
+        return delegateService.isdebug();
+    }
+
 }

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/bigdata/BigDataJavaProcessor.java

@@ -41,6 +41,7 @@ import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.process.TalendProcessOptionConstants;
 import org.talend.core.runtime.repository.build.IMavenPomCreator;
+import org.talend.designer.core.utils.BigDataJobUtil;
 import org.talend.designer.maven.tools.creator.CreateMavenJobPom;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IBigDataProcessor;
@@ -224,7 +225,7 @@ public abstract class BigDataJavaProcessor extends MavenJavaProcessor implements
         // StringBuffer.
         boolean needAllLibJars = true;
         if (needsShade()) {
-            BigDataJobUtil bdUtil = new BigDataJobUtil((ProcessItem) this.getProperty().getItem());
+            BigDataJobUtil bdUtil = new BigDataJobUtil(process);
             if (bdUtil.isMRWithHDInsight()) {
                 needAllLibJars = false;
             }
@@ -410,7 +411,7 @@ public abstract class BigDataJavaProcessor extends MavenJavaProcessor implements
     @Override
     public boolean needsShade() {
         if (needsShade == null && property != null) {
-            needsShade = new BigDataJobUtil((ProcessItem) property.getItem()).needsShade();
+            needsShade = new BigDataJobUtil(process).needsShade();
         }
         return needsShade;
     }
@@ -430,7 +431,7 @@ public abstract class BigDataJavaProcessor extends MavenJavaProcessor implements
                     modulesNeeded);
         }
 
-        return new BigDataJobUtil((ProcessItem) property.getItem()).getShadedModulesExclude(modulesNeeded);
+        return new BigDataJobUtil(getProcess()).getShadedModulesExclude(modulesNeeded);
     }
 
 }

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/java/JavaProcessor.java

@@ -136,6 +136,7 @@ import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 import org.talend.designer.core.ui.editor.CodeEditorFactory;
 import org.talend.designer.core.ui.editor.nodes.Node;
 import org.talend.designer.core.ui.editor.process.Process;
+import org.talend.designer.core.utils.BigDataJobUtil;
 import org.talend.designer.maven.utils.ClasspathsJarGenerator;
 import org.talend.designer.maven.utils.MavenVersionHelper;
 import org.talend.designer.maven.utils.PomUtil;
@@ -145,7 +146,6 @@ import org.talend.designer.runprocess.ProcessorException;
 import org.talend.designer.runprocess.ProcessorUtilities;
 import org.talend.designer.runprocess.RunProcessContext;
 import org.talend.designer.runprocess.RunProcessPlugin;
-import org.talend.designer.runprocess.bigdata.BigDataJobUtil;
 import org.talend.designer.runprocess.i18n.Messages;
 import org.talend.designer.runprocess.prefs.RunProcessPrefsConstants;
 import org.talend.designer.runprocess.utils.JobVMArgumentsUtil;
@@ -356,7 +356,7 @@ public class JavaProcessor extends AbstractJavaProcessor implements IJavaBreakpo
             boolean needsToHaveContextInsideJar = true;
 
             if (property != null && property.getItem() instanceof ProcessItem) {
-                needsToHaveContextInsideJar = !new BigDataJobUtil((ProcessItem) property.getItem()).needsToHaveContextInsideJar();
+                needsToHaveContextInsideJar = !new BigDataJobUtil(process).needsToHaveContextInsideJar();
             }
 
             if (ProcessorUtilities.isExportConfig() && property != null && needsToHaveContextInsideJar) {
@@ -1229,7 +1229,7 @@ public class JavaProcessor extends AbstractJavaProcessor implements IJavaBreakpo
         libsStr = StringUtils.replace(libsStr, " ", "%20"); //$NON-NLS-1$ //$NON-NLS-2$
 
         // create classpath.jar
-        if (!isExportConfig() && !isSkipClasspathJar()) {
+        if (!isExportConfig() && !isSkipClasspathJar() && isCorrespondingOS()) {
             try {
                 libsStr = ClasspathsJarGenerator.createJar(getProperty(), libsStr, classPathSeparator);
             } catch (Exception e) {
@@ -1240,6 +1240,16 @@ public class JavaProcessor extends AbstractJavaProcessor implements IJavaBreakpo
         return libsStr;
     }
 
+    private boolean isCorrespondingOS() {
+        if (Platform.getOS().equals(Platform.OS_WIN32) && isWinTargetPlatform()) {
+            return true;
+        }
+        if (!Platform.getOS().equals(Platform.OS_WIN32) && !isWinTargetPlatform()) {
+            return true;
+        }
+        return false;
+    }
+
     protected String getBasePathClasspath() throws ProcessorException {
         final String classPathSeparator = extractClassPathSeparator();
         final String rootWorkingDir = getRootWorkingDir(false);
@@ -1356,7 +1366,12 @@ public class JavaProcessor extends AbstractJavaProcessor implements IJavaBreakpo
         final String classPathSeparator = extractClassPathSeparator();
         final String libPrefixPath = getRootWorkingDir(true);
 
-        Set<ModuleNeeded> neededModules = getNeededModules(TalendProcessOptionConstants.MODULES_WITH_CHILDREN);
+        int option = TalendProcessOptionConstants.MODULES_WITH_CHILDREN;
+
+        if (isExportConfig() || isSkipClasspathJar()) {
+            option = option | TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED;
+        }
+        Set<ModuleNeeded> neededModules = getNeededModules(option);
         JavaProcessorUtilities.checkJavaProjectLib(neededModules);
 
         // Ignore hadoop confs jars in lib path.
@@ -1386,10 +1401,12 @@ public class JavaProcessor extends AbstractJavaProcessor implements IJavaBreakpo
         } else {
             for (ModuleNeeded neededModule : neededModules) {
                 MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(neededModule.getMavenUri());
-                if ("sapjco3".equals(artifact.getArtifactId())) { //$NON-NLS-1$
-                    String jarPath = JavaProcessorUtilities.getJavaProjectLibFolder2().getFile("sapjco3.jar").getLocation() //$NON-NLS-1$
+                boolean isSapidoc3 = "sapidoc3".equals(artifact.getArtifactId());
+                if ("sapjco3".equals(artifact.getArtifactId()) || isSapidoc3) { //$NON-NLS-1$ //$NON-NLS-2$
+                    String jarPath = JavaProcessorUtilities.getJavaProjectLibFolder2().getFile(artifact.getArtifactId() + ".jar") //$NON-NLS-1$
+                            .getLocation()
                             .toPortableString();
-                    if (compareSapjco3Version(jarPath) > 0) {
+                    if (compareSapjco3Version(jarPath) > 0 || isSapidoc3) {
                         libPath.append(jarPath).append(classPathSeparator);
                     } else {
                         libPath.append(PomUtil.getAbsArtifactPathAsCP(artifact)).append(classPathSeparator);

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/java/JavaProcessorUtilities.java

@@ -72,6 +72,7 @@ import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 import org.talend.designer.core.ui.editor.process.Process;
+import org.talend.designer.core.utils.BigDataJobUtil;
 import org.talend.designer.core.utils.JavaProcessUtil;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IRunProcessService;
@@ -104,7 +105,8 @@ public class JavaProcessorUtilities {
      * @return
      */
     public static Set<ModuleNeeded> extractLibsOnlyForMapperAndReducer(IProcess process) {
-        int options = TalendProcessOptionConstants.MODULES_WITH_CHILDREN | TalendProcessOptionConstants.MODULES_FOR_MR;
+        int options = TalendProcessOptionConstants.MODULES_WITH_CHILDREN | TalendProcessOptionConstants.MODULES_FOR_MR
+                | TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED;
         Set<ModuleNeeded> allModules = JavaProcessUtil.getNeededModules(process, options);
         return allModules;
     }
@@ -241,6 +243,11 @@ public class JavaProcessorUtilities {
                 }
             }
         }
+
+        if (BitwiseOptionUtils.containOption(options, TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED)) {
+            new BigDataJobUtil(process).removeExcludedModules(neededLibraries);
+        }
+
         // move high priority modules to front.
         Set<ModuleNeeded> highPriorityModuleNeeded = LastGenerationInfo.getInstance().getHighPriorityModuleNeeded();
         if (!highPriorityModuleNeeded.isEmpty()) {

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/java/TalendJavaProjectManager.java

@@ -479,6 +479,11 @@ public class TalendJavaProjectManager {
             public void run(IProgressMonitor monitor) throws CoreException {
                 IProject[] projects = ResourcesPlugin.getWorkspace().getRoot().getProjects();
                 for (IProject project : projects) {
+                    if (project.hasNature("com.oaklandsw.transform.runtime.nature")) { //$NON-NLS-1$
+                        // never delete TDM Builtin, Examples and Examples EDI project ref
+                        // com.oaklandsw.data.transform.builtin
+                        continue;
+                    }
                     if (project.getLocation() == null || !project.getLocation().toFile().exists()
                             || !project.getFile(IProjectDescription.DESCRIPTION_FILE_NAME).getLocation().toFile().exists()) {
                         project.delete(false, true, monitor);

main/plugins/org.talend.repository/src/main/java/messages.properties

@@ -389,6 +389,8 @@ JavaJobScriptsExportWSWizardPage.PETALSESB=Petals ESB (Deprecated)
 JavaJobScriptsExportWSWizardPage.OSGI=OSGI Bundle For ESB
 JavaJobScriptsExportWSWizardPage.MSESB=Microservice (Spring Boot) For ESB
 JavaJobScriptsExportWSWizardPage.IMAGE=Docker Image
+JavaJobScriptsExportWSWizardPage.ROUTE=ESB Route
+JavaJobScriptsExportWSWizardPage.SERVICE=Service
 JavaJobScriptsExportWSWizardPage.DOCKER.optionGroup=Docker Options
 JavaJobScriptsExportWSWizardPage.DOCKER.dockerHost=Docker host
 JavaJobScriptsExportWSWizardPage.DOCKER.localHost=Local
@@ -407,7 +409,7 @@ JavaJobScriptsExportWSWizardPage.banServiceJob=Service Job can't build independe
 LimitRESTRequestJobChecker.limit_tRESTRequest=ESB Job contains "{0}" should only build using one of [OSGI Bundle For ESB],[Microservice For ESB].
 LimitRESTRequestJobChecker.limit_ESBJobForMS=Only ESB Job with "{0}" could build using [Microservice (Spring Boot) For ESB] type.
 LimitRESTRequestJobChecker.limit_ESBJobForOSGI=Job without ESB components could not build using [OSGI Bundle For ESB] type.
-LimitESBConsumerJobChecker.limit_tRESTClient_tESBConsumer=ESB Job contains "{0}" should only build using one of [Standalone Job],[OSGI Bundle For ESB].
+LimitESBConsumerJobChecker.limit_tRESTClient_tESBConsumer=ESB Job contains "{0}" should only build using one of [Standalone Job],[OSGI Bundle For ESB],[Docker Image].
 LimitESBConsumerJobChecker.limitFeaturesWithStandalone.locator=ESB Job contains "{0}" shouldn't specify feature Service Locator when build using [Standalone Job]
 LimitESBConsumerJobChecker.limitFeaturesWithStandalone.SAM=ESB Job contains "{0}" shouldn't specify feature Service Activity Monitor when build using [Standalone Job]
 LimitESBConsumerJobChecker.limitFeaturesWithStandalone.SR=ESB Job contains "{0}" shouldn't specify feature Service Registry when build using [Standalone Job]
@@ -823,10 +825,13 @@ connections.form.field.description=Description
 connections.form.field.imcomplete=Incomplete
 connections.form.field.name=Name
 connections.form.field.password=User Password
+connections.form.field.token=Token
+connections.form.field.token.browse=Open the token page in browse
 connections.form.field.repository=Repository
 connections.form.field.username=User E-mail
 connections.form.field.usernameNoMail=User Name
 connections.form.malformedField.username=The username must be a valid email.
+connections.form.getTokenUrl.failed=Please check the server url.
 dataset.error.populateXMLTree=An invalid file is inputed, please input valid file.
 repository.title=Repository
 

main/plugins/org.talend.repository/src/main/java/org/talend/repository/RepositoryService.java

@@ -423,6 +423,7 @@ public class RepositoryService implements IRepositoryService, IRepositoryContext
                 repositoryContext.setUser(userInfo);
                 repositoryContext.setClearPassword(password);
                 repositoryContext.setFields(bean.getDynamicFields());
+                repositoryContext.setToken(bean.isToken());
 
                 repositoryFactory.setRepositoryFactoryFromProvider(RepositoryFactoryProvider.getRepositoriyById(bean
                         .getRepositoryId()));

main/plugins/org.talend.repository/src/main/java/org/talend/repository/model/migration/AutoUpdateRelationsMigrationTask.java

@@ -57,7 +57,9 @@ public class AutoUpdateRelationsMigrationTask extends AbstractJobMigrationTask {
                 Resource processResource = ((ProcessItem) item).getProcess().eResource();
                 Resource propertyResource = item.eResource();
                 if (processResource != null && processResource.isLoaded() && (processResource instanceof XMIResource)) {
-                    propertyResource.unload();
+                    if (propertyResource != null) {
+                        propertyResource.unload();
+                    }
                     processResource.unload();
                     rM.resourceSet.getResources().remove(propertyResource);
                     rM.resourceSet.getResources().remove(processResource);

main/plugins/org.talend.repository/src/main/java/org/talend/repository/model/migration/UseOracle11VersionInsteadOfRemoved.java

@@ -23,9 +23,6 @@ public class UseOracle11VersionInsteadOfRemoved extends AbstractJobMigrationTask
     private static final String ORACLE_VERSION_PROPERTY_NAME = "DB_VERSION";
     private static final String REMOVED_ORACLE_VERSION = "ORACLE_11-6";
 
-    private static final String USE_CONNECTION_PROPERTY_NAME = "USE_EXISTING_CONNECTION";
-
-    private static final String MOVE_TO_DIR_PROPERTY_NAME = "MOVE_TO_THE_CURRENT_DIRECTORY";
 
     @Override
     public Date getOrder() {
@@ -83,12 +80,12 @@ public class UseOracle11VersionInsteadOfRemoved extends AbstractJobMigrationTask
     private class OracleVersionConversion implements IComponentConversion {
         @Override
         public void transform(NodeType node) {
-            if ((!"true".equals(ComponentUtilities.getNodePropertyValue(node, USE_CONNECTION_PROPERTY_NAME)))) {
-                String oracleVersion =  ComponentUtilities.getNodePropertyValue(node, ORACLE_VERSION_PROPERTY_NAME);
-                if (REMOVED_ORACLE_VERSION.equals(oracleVersion)) {
-                    ComponentUtilities.setNodeValue(node, ORACLE_VERSION_PROPERTY_NAME, "ORACLE_11");
-                }
+            
+            String oracleVersion =  ComponentUtilities.getNodePropertyValue(node, ORACLE_VERSION_PROPERTY_NAME);
+            if (REMOVED_ORACLE_VERSION.equals(oracleVersion)) {
+                ComponentUtilities.setNodeValue(node, ORACLE_VERSION_PROPERTY_NAME, "ORACLE_11");
             }
+            
 
         }
     }

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/login/LoginHelper.java

@@ -374,6 +374,7 @@ public class LoginHelper {
         if (connBean != null) {
             repositoryContext.setFields(connBean.getDynamicFields());
             password = connBean.getPassword();
+            repositoryContext.setToken(connBean.isToken());
         }
         repositoryContext.setClearPassword(password);
         if (project != null) {

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/login/connections/ConnectionFormComposite.java

@@ -13,15 +13,18 @@
 package org.talend.repository.ui.login.connections;
 
 import java.io.File;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.draw2d.ColorConstants;
+import org.eclipse.jface.dialogs.MessageDialog;
 import org.eclipse.jface.layout.GridDataFactory;
 import org.eclipse.jface.viewers.ArrayContentProvider;
 import org.eclipse.jface.viewers.ComboViewer;
@@ -39,6 +42,7 @@ import org.eclipse.swt.events.SelectionListener;
 import org.eclipse.swt.graphics.Point;
 import org.eclipse.swt.layout.GridData;
 import org.eclipse.swt.layout.GridLayout;
+import org.eclipse.swt.program.Program;
 import org.eclipse.swt.widgets.Button;
 import org.eclipse.swt.widgets.Combo;
 import org.eclipse.swt.widgets.Composite;
@@ -54,6 +58,7 @@ import org.talend.commons.ui.swt.formtools.LabelText;
 import org.talend.commons.ui.swt.formtools.LabelledCombo;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.general.ConnectionBean;
+import org.talend.core.model.utils.TalendTextUtils;
 import org.talend.core.repository.model.DynamicButtonBean;
 import org.talend.core.repository.model.DynamicChoiceBean;
 import org.talend.core.repository.model.DynamicFieldBean;
@@ -90,8 +95,24 @@ public class ConnectionFormComposite extends Composite {
 
     private Text userText;
 
+    private Label tpPlaceHolder;
+
+    private Composite tpCompo;
+
+    private Button tokenButton;
+
+    private Composite tokenCompo;
+
+    private Text tokenText;
+
+    private Button tokenBrowseButton;
+
+    private Button passwordButton;
+
     private Text passwordText;
 
+    private String password;
+
     private Text workSpaceText;
 
     private Button workSpaceButton;
@@ -112,8 +133,9 @@ public class ConnectionFormComposite extends Composite {
 
     public static final String URL_FIELD_NAME = "url"; //$NON-NLS-1$
 
-    Label passwordLabel = null;
+    private static final String TOKEN_URL_SUFFIX = "/user/access-tokens"; //$NON-NLS-1$
 
+    Label passwordLabel = null;
     /**
      * DOC smallet ConnectionsComposite constructor comment.
      * 
@@ -171,15 +193,44 @@ public class ConnectionFormComposite extends Composite {
         userText = toolkit.createText(formBody, "", SWT.BORDER); //$NON-NLS-1$
         formDefaultFactory.copy().grab(true, false).span(2, 1).applyTo(userText);
 
-        // Password
         passwordLabel = toolkit.createLabel(formBody, Messages.getString("connections.form.field.password")); //$NON-NLS-1$
         formDefaultFactory.copy().applyTo(passwordLabel);
 
-        passwordText = toolkit.createText(formBody, "", SWT.PASSWORD | SWT.BORDER); //$NON-NLS-1$
+        tpPlaceHolder = new Label(formBody, SWT.NONE);
+        tpCompo = toolkit.createComposite(formBody);
+        GridLayout tCompoLayout = new GridLayout(3, false);
+        tCompoLayout.marginHeight = 0;
+        tCompoLayout.marginWidth = 0;
+        tpCompo.setLayout(tCompoLayout);
+        formDefaultFactory.copy().grab(true, false).span(2, 1).applyTo(tpCompo);
+
+        // Password
+        passwordButton = toolkit.createButton(tpCompo, Messages.getString("connections.form.field.password"), SWT.RADIO); //$NON-NLS-1$
+        formDefaultFactory.copy().applyTo(passwordButton);
+
+        passwordText = toolkit.createText(tpCompo, "", SWT.PASSWORD | SWT.BORDER); //$NON-NLS-1$
         formDefaultFactory.copy().grab(true, false).span(2, 1).applyTo(passwordText);
 
         Label workSpaceLabel = toolkit.createLabel(formBody, Messages.getString("ConnectionFormComposite.WORKSPACE")); //$NON-NLS-1$
         formDefaultFactory.copy().applyTo(workSpaceLabel);
+        // Token
+        tokenButton = toolkit.createButton(tpCompo, Messages.getString("connections.form.field.token"), SWT.RADIO); //$NON-NLS-1$
+        formDefaultFactory.copy().applyTo(tokenButton);
+
+        tokenCompo = toolkit.createComposite(tpCompo);
+        GridLayout tokenCompoLayout = new GridLayout(3, false);
+        tokenCompoLayout.marginHeight = 0;
+        tokenCompoLayout.marginWidth = 0;
+        tokenCompo.setLayout(tokenCompoLayout);
+        formDefaultFactory.copy().grab(true, false).span(2, 1).applyTo(tokenCompo);
+
+        tokenText = toolkit.createText(tokenCompo, "", SWT.PASSWORD | SWT.BORDER); //$NON-NLS-1$
+        formDefaultFactory.copy().grab(true, false).span(2, 1).applyTo(tokenText);
+
+        tokenBrowseButton = toolkit.createButton(tokenCompo, null, SWT.PUSH);
+        tokenBrowseButton.setToolTipText(Messages.getString("connections.form.field.token.browse")); //$NON-NLS-1$
+        tokenBrowseButton.setImage(ImageProvider.getImage(EImage.THREE_DOTS_ICON));
+        GridDataFactory.fillDefaults().applyTo(tokenBrowseButton);
 
         Composite wsCompo = toolkit.createComposite(formBody);
         GridLayout wsCompoLayout = new GridLayout(2, false);
@@ -275,6 +326,7 @@ public class ConnectionFormComposite extends Composite {
         deleteProjectsButton.setLayoutData(deleteButtonGridData);
 
         addListeners();
+        addTokenBrowseListener();
         addWorkSpaceListener();
         fillLists();
         showHideDynamicsControls();
@@ -414,32 +466,67 @@ public class ConnectionFormComposite extends Composite {
     }
 
     private void showHideTexts() {
-        if (passwordText != null && !passwordText.isDisposed()) {
-            boolean enablePasswordField = false;
-            if (connection != null) {
-                IRepositoryFactory factory = RepositoryFactoryProvider.getRepositoriyById(connection.getRepositoryId());
-                if (factory != null && factory.isAuthenticationNeeded()) {
-                    enablePasswordField = true;
-                }
-            } else if (getRepository() != null && LoginHelper.isRemotesRepository(getRepository().getId())) {
+        boolean token = false;
+        if (connection != null && tokenText != null && !tokenText.isDisposed()) {
+            token = connection.isToken();
+        }
+        // Check enable field or not
+        boolean enablePasswordField = false;
+        boolean enableTokenField = false;
+        if (connection != null) {
+            IRepositoryFactory factory = RepositoryFactoryProvider.getRepositoriyById(connection.getRepositoryId());
+            if (factory != null && factory.isAuthenticationNeeded()) {
                 enablePasswordField = true;
+                if (LoginHelper.isCloudConnection(connection)) {
+                    enableTokenField = true;
+                }
             }
+        } else if (getRepository() != null && LoginHelper.isRemotesRepository(getRepository().getId())) {
+            enablePasswordField = true;
+            if (LoginHelper.isCloudRepository(getRepository().getId())) {
+                enableTokenField = true;
+            }
+        }
 
-            if (enablePasswordField) {
+        // password
+        boolean hidePasswordLabel = !enablePasswordField || enableTokenField;
+        boolean enablePasswordText = passwordButton.getSelection() || !hidePasswordLabel;
+        if (passwordText != null && !passwordText.isDisposed()) {
+            if (enablePasswordField && enablePasswordText) {
                 passwordText.setBackground(LoginDialogV2.WHITE_COLOR);
             } else {
                 passwordText.setText(""); //$NON-NLS-1$
                 passwordText.setBackground(LoginDialogV2.GREY_COLOR);
             }
-            passwordText.setEnabled(enablePasswordField);
-            passwordText.setEditable(enablePasswordField);
-
+            passwordText.setEnabled(enablePasswordField && enablePasswordText);
+            passwordText.setEditable(enablePasswordField && enablePasswordText);
             hideControl(passwordText, !enablePasswordField, false);
-            hideControl(passwordLabel, !enablePasswordField, false);
-
+            hideControl(passwordLabel, hidePasswordLabel, false);
+            hideControl(passwordButton, !(enablePasswordField && enableTokenField), false);
             passwordText.getParent().layout();
-
+            passwordText.getParent().getParent().layout();
         }
+        // token
+        if (tokenText != null && !tokenText.isDisposed()) {
+            if (enableTokenField && tokenButton.getSelection()) {
+                tokenText.setBackground(LoginDialogV2.WHITE_COLOR);
+            } else {
+                tokenText.setText(""); //$NON-NLS-1$
+                password = ""; //$NON-NLS-1$
+                tokenText.setBackground(LoginDialogV2.GREY_COLOR);
+            }
+            tokenText.setEnabled(enableTokenField && tokenButton.getSelection());
+            tokenText.setEditable(enableTokenField && tokenButton.getSelection());
+            tokenBrowseButton.setEnabled(enableTokenField && tokenButton.getSelection());
+            hideControl(tokenButton, !enableTokenField, false);
+            hideControl(tokenCompo, !enableTokenField, false);
+            tokenText.getParent().getParent().layout();
+            tokenText.getParent().getParent().getParent().layout();
+        }
+
+        hideControl(tpPlaceHolder, !enableTokenField, false);
+        hideControl(tpCompo, !enablePasswordField, false);
+        tpCompo.getParent().layout();
     }
 
     private void hideControl(Control control, boolean hide, boolean autoLayout) {
@@ -480,7 +567,7 @@ public class ConnectionFormComposite extends Composite {
     }
 
     private String getPassword() {
-        return passwordText.getText();
+        return password;
     }
 
     private String getWorkspace() {
@@ -501,7 +588,16 @@ public class ConnectionFormComposite extends Composite {
                     userText.setSelection(selection);
                 }
             }
-            fillBean(false);
+            fillBean(false, false);
+            validateFields();
+        }
+    };
+
+    ModifyListener standardTokenPasswordTextListener = new ModifyListener() {
+
+        @Override
+        public void modifyText(ModifyEvent e) {
+            fillBean(false, true);
             validateFields();
         }
     };
@@ -512,7 +608,7 @@ public class ConnectionFormComposite extends Composite {
         public void selectionChanged(SelectionChangedEvent e) {
             showHideDynamicsControls();
             validateFields();
-            fillBean(true);
+            fillBean(true, true);
             showHideTexts();
             changeUserLabel();
         }
@@ -536,12 +632,45 @@ public class ConnectionFormComposite extends Composite {
         }
     };
 
+    SelectionListener passwordClickListener = new SelectionListener() {
+
+        @Override
+        public void widgetSelected(SelectionEvent e) {
+            tokenButton.setSelection(!passwordButton.getSelection());
+            passwordButton.setSelection(passwordButton.getSelection());
+            passwordText.setEnabled(passwordButton.getSelection());
+            showHideTexts();
+        }
+
+        @Override
+        public void widgetDefaultSelected(SelectionEvent e) {
+            // nothing need to do
+        }
+    };
+
+    SelectionListener tokenClickListener = new SelectionListener() {
+
+        @Override
+        public void widgetSelected(SelectionEvent e) {
+            passwordButton.setSelection(!tokenButton.getSelection());
+            tokenButton.setSelection(tokenButton.getSelection());
+            tokenText.setEnabled(tokenButton.getSelection());
+            showHideTexts();
+        }
+
+        @Override
+        public void widgetDefaultSelected(SelectionEvent e) {
+            // nothing need to do
+        }
+    };
+
     private void addListeners() {
         repositoryCombo.addPostSelectionChangedListener(repositoryListener);
         nameText.addModifyListener(standardTextListener);
         descriptionText.addModifyListener(standardTextListener);
         userText.addModifyListener(standardTextListener);
-        passwordText.addModifyListener(standardTextListener);
+        passwordText.addModifyListener(standardTokenPasswordTextListener);
+        tokenText.addModifyListener(standardTokenPasswordTextListener);
         workSpaceText.addModifyListener(standardTextListener);
 
         for (IRepositoryFactory f : dynamicControls.keySet()) {
@@ -555,7 +684,8 @@ public class ConnectionFormComposite extends Composite {
                 control.getCombo().addModifyListener(standardTextListener);
             }
         }
-
+        passwordButton.addSelectionListener(passwordClickListener);
+        tokenButton.addSelectionListener(tokenClickListener);
         deleteProjectsButton.addSelectionListener(deleteProjectClickListener);
     }
 
@@ -587,12 +717,50 @@ public class ConnectionFormComposite extends Composite {
         });
     }
 
+    private void addTokenBrowseListener() {
+        tokenBrowseButton.addSelectionListener(new SelectionAdapter() {
+
+            @Override
+            public void widgetSelected(SelectionEvent e) {
+                String currentUrl = "";
+                if (getRepository() != null) {
+                    for (LabelText currentUrlLabel : dynamicRequiredControls.get(getRepository()).values()) {
+                        if (currentUrlLabel.getText().length() != 0) {
+                            currentUrl = currentUrlLabel.getText();
+                        }
+                    }
+                }
+                if (StringUtils.isNotBlank(currentUrl)) {
+                    try {
+                        String tokenUrl = getTokenUrl(currentUrl);
+                        openBrower(tokenUrl);
+                    } catch (Exception exception) {
+                        MessageDialog.openError(getShell(), getShell().getText(),
+                                Messages.getString("connections.form.getTokenUrl.failed"));
+                    }
+                }
+            }
+        });
+    }
+
+    private String getTokenUrl(String serverUrlStr) throws Exception {
+        URL serverUrl = new URL(serverUrlStr);
+        String host = serverUrl.getHost().replace("tmc.", "");
+        URL tokenUrl = new URL(serverUrl.getProtocol(), host, TOKEN_URL_SUFFIX);
+        return tokenUrl.toString();
+    }
+
+    private void openBrower(String url) {
+        Program.launch(url);
+    }
+
     private void removeListeners() {
         repositoryCombo.removePostSelectionChangedListener(repositoryListener);
         nameText.removeModifyListener(standardTextListener);
         descriptionText.removeModifyListener(standardTextListener);
         userText.removeModifyListener(standardTextListener);
-        passwordText.removeModifyListener(standardTextListener);
+        passwordText.removeModifyListener(standardTokenPasswordTextListener);
+        tokenText.removeModifyListener(standardTokenPasswordTextListener);
         workSpaceText.removeModifyListener(standardTextListener);
 
         for (IRepositoryFactory f : dynamicControls.keySet()) {
@@ -606,10 +774,12 @@ public class ConnectionFormComposite extends Composite {
                 control.getCombo().removeModifyListener(standardTextListener);
             }
         }
+        passwordButton.removeSelectionListener(passwordClickListener);
+        tokenButton.removeSelectionListener(tokenClickListener);
         deleteProjectsButton.removeSelectionListener(deleteProjectClickListener);
     }
 
-    private void fillBean(boolean cleanDynamicValue) {
+    private void fillBean(boolean cleanDynamicValue, boolean enableHidePassword) {
         if (connection != null) {
             IRepositoryFactory repository = getRepository();
             if (repository != null) {
@@ -662,7 +832,18 @@ public class ConnectionFormComposite extends Composite {
             connection.setName(nameText.getText());
             connection.setDescription(descriptionText.getText());
             connection.setUser(userText.getText());
-            connection.setPassword(passwordText.getText());
+            if (enableHidePassword) {
+                String password = getPassword(); // $NON-NLS-1$
+                if (StringUtils.isEmpty(password)) {
+                    if (tokenButton.getSelection()) {
+                        password = tokenText.getText();
+                    } else {
+                        password = passwordText.getText();
+                    }
+                }
+                connection.setPassword(password);
+                connection.setToken(tokenButton.getSelection());
+            }
             connection.setWorkSpace(workSpaceText.getText());
 
             connectionsListComposite.refresh(connection);
@@ -734,7 +915,17 @@ public class ConnectionFormComposite extends Composite {
             nameText.setText((connection.getName() == null ? "" : connection.getName())); //$NON-NLS-1$
             descriptionText.setText((connection.getDescription() == null ? "" : connection.getDescription())); //$NON-NLS-1$
             userText.setText((connection.getUser() == null ? "" : connection.getUser())); //$NON-NLS-1$
-            passwordText.setText((connection.getPassword() == null ? "" : connection.getPassword())); //$NON-NLS-1$
+            String passwordStr = TalendTextUtils.hidePassword(connection.getPassword());
+            tokenButton.setSelection(connection.isToken());
+            tokenText.setEnabled(connection.isToken());
+            passwordButton.setSelection(!connection.isToken());
+            passwordText.setEnabled(!connection.isToken());
+            if (connection.isToken()) {
+                tokenText.setText(passwordStr);
+            } else {
+                passwordText.setText(passwordStr);
+            }
+            password = connection.getPassword() == null ? "" : connection.getPassword(); // $NON-NLS-1$
             workSpaceText
                     .setText(("".equals(connection.getWorkSpace()) || connection.getWorkSpace() == null) ? getRecentWorkSpace() : connection.getWorkSpace());//$NON-NLS-1$
             addListeners();

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/login/sandboxProject/CreateSandboxProjectDialog.java

@@ -251,6 +251,7 @@ public class CreateSandboxProjectDialog extends TitleAreaDialog {
                             RepositoryContext repositoryContext = new RepositoryContext();
                             repositoryContext.setFields(new HashMap<String, String>());
                             repositoryContext.setClearPassword(originRepCtx.getClearPassword());
+                            repositoryContext.setToken(bean != null ? bean.isToken() : false);
                             repositoryContext.setUser(originRepCtx.getUser());
                             repositoryContext.getFields().put(RepositoryConstants.REPOSITORY_URL, urlText.getText());
                             ctx.putProperty(Context.REPOSITORY_CONTEXT_KEY, repositoryContext);

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/JavaJobScriptsExportWSWizardPage.java

@@ -91,7 +91,9 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
         POJO(Messages.getString("JavaJobScriptsExportWSWizardPage.POJO"), false), //$NON-NLS-1$
         OSGI(Messages.getString("JavaJobScriptsExportWSWizardPage.OSGI"), false), //$NON-NLS-1$
         MSESB(Messages.getString("JavaJobScriptsExportWSWizardPage.MSESB"), false), //$NON-NLS-1$
-        IMAGE(Messages.getString("JavaJobScriptsExportWSWizardPage.IMAGE"), false); //$NON-NLS-1$
+        IMAGE(Messages.getString("JavaJobScriptsExportWSWizardPage.IMAGE"), false), //$NON-NLS-1$
+        ROUTE(Messages.getString("JavaJobScriptsExportWSWizardPage.ROUTE"), false), //$NON-NLS-1$
+        SERVICE(Messages.getString("JavaJobScriptsExportWSWizardPage.SERVICE"), false); //$NON-NLS-1$
 
         public final String label;
 
@@ -129,6 +131,10 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             }
         }
 
+        public static boolean isImageType(JobExportType type) {
+            return type == IMAGE;
+        }
+
     }
 
     public static final String ESBTYPE_JBOSS_MQ = "JBoss MQ"; //$NON-NLS-1$
@@ -203,6 +209,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
 
     private Label hostLabel, imageLabel, tagLabel;
 
+    private boolean isValid;
+
     public JavaJobScriptsExportWSWizardPage(IStructuredSelection selection, String exportType) {
         super(selection);
         // there assign the manager again
@@ -229,10 +237,11 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             // set default export type according to system properties
             String exportType = dialogSettings.get(STORE_EXPORTTYPE_ID);
             String defaultExportType = System.getProperty("talend.export.job.default.type"); //$NON-NLS-1$
-            if ((exportType == null || exportType.equals("")) && defaultExportType != null && !defaultExportType.equals("")) { //$NON-NLS-1$ //$NON-NLS-2$
+            if ((exportType == null || exportType.equals("")) && defaultExportType != null //$NON-NLS-1$
+                    && !defaultExportType.equals("")) { //$NON-NLS-1$
                 dialogSettings.put(STORE_EXPORTTYPE_ID, defaultExportType);
             }
-        }// else ignors it
+        } // else ignors it
     }
 
     @Override
@@ -243,7 +252,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
         GridLayout layout = new GridLayout();
         layout.numColumns = 3;
         destinationSelectionGroup.setLayout(layout);
-        destinationSelectionGroup.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_FILL | GridData.VERTICAL_ALIGN_FILL));
+        destinationSelectionGroup
+                .setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_FILL | GridData.VERTICAL_ALIGN_FILL));
         destinationSelectionGroup.setFont(font);
 
         destinationLabel = new Label(destinationSelectionGroup, SWT.NONE);
@@ -351,31 +361,30 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
 
     @Override
     public void createControl(Composite parent) {
-        
+
         initializeDialogUnits(parent);
         SashForm sash = createExportTree(parent);
 
-        
         // Added a scrolled composite by Marvin Wang on Feb. 27, 2012 for bug TDI-19198.
         scrolledComposite = new ScrolledComposite(sash, SWT.H_SCROLL | SWT.V_SCROLL);
         scrolledComposite.setExpandHorizontal(true);
         scrolledComposite.setExpandVertical(true);
         GridDataFactory.fillDefaults().grab(true, true).applyTo(scrolledComposite);
-        
+
         pageComposite = new Composite(scrolledComposite, SWT.BORDER);
         GridDataFactory.fillDefaults().grab(true, true).applyTo(pageComposite);
-        
+
         GridLayout gdlPageComposite = new GridLayout();
         pageComposite.setLayout(gdlPageComposite);
         pageComposite.setFont(parent.getFont());
-        
+
         createDestinationGroup(pageComposite);
 
         // this.getDestinationValue()
         // createExportTree(pageComposite);
         if (!isMultiNodes()) {
-            IBrandingService brandingService = (IBrandingService) GlobalServiceRegister.getDefault()
-                    .getService(IBrandingService.class);
+            IBrandingService brandingService =
+                    (IBrandingService) GlobalServiceRegister.getDefault().getService(IBrandingService.class);
             boolean allowVerchange = brandingService.getBrandingConfiguration().isAllowChengeVersion();
             if (allowVerchange) {
                 createJobVersionGroup(pageComposite);
@@ -393,10 +402,9 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
         updateWidgetEnablements();
         setPageComplete(determinePageCompletion());
 
-        
         setControl(sash);
         sash.setWeights(new int[] { 0, 1, 23 });
-        
+
         giveFocusToDestination();
 
         pageComposite.setSize(pageComposite.computeSize(SWT.DEFAULT, SWT.DEFAULT));
@@ -407,7 +415,7 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
     protected void createExportTypeGroup(Composite parent) {
         // options group
         Group optionsGroup = new Group(parent, SWT.NONE);
-        
+
         GridLayout layout = new GridLayout();
         optionsGroup.setLayout(layout);
         optionsGroup.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_FILL | GridData.GRAB_HORIZONTAL));
@@ -418,7 +426,7 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
 
         Composite left = new Composite(optionsGroup, SWT.NONE);
         left.setLayoutData(new GridData(SWT.LEFT, SWT.TOP, true, false));
-        
+
         GridLayout gdlLeft = new GridLayout(3, false);
         gdlLeft.marginHeight = 0;
         gdlLeft.marginWidth = 0;
@@ -459,8 +467,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
         // if the build type was set, try to select by default
         if (nodes != null && nodes.length == 1) { // deal with one node only.
             ProcessItem item = getProcessItem();
-            final Object buildType = item.getProperty().getAdditionalProperties()
-                    .get(TalendProcessArgumentConstant.ARG_BUILD_TYPE);
+            final Object buildType =
+                    item.getProperty().getAdditionalProperties().get(TalendProcessArgumentConstant.ARG_BUILD_TYPE);
             if (buildType != null) {
                 Map<JobExportType, String> map = BuildJobFactory.oldBuildTypeMap;
                 for (JobExportType t : map.keySet()) {
@@ -489,7 +497,7 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             chkButton.setVisible(true);
             zipOption = String.valueOf(chkButton.getSelection());
         }
-        
+
         chkButton.addSelectionListener(new SelectionAdapter() {
 
             @Override
@@ -498,7 +506,7 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
                 zipOption = String.valueOf(chkButton.getSelection());
             }
         });
-        
+
         exportTypeCombo.addSelectionListener(new SelectionAdapter() {
 
             @Override
@@ -537,10 +545,12 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
         Map<ExportChoice, Object> exportChoiceMap = getExportChoiceMap();
         String log4jLevel = "";
         String launcher = (getCurrentExportType1() == JobExportType.POJO) ? launcherCombo.getText() : "all";
-        String context = (contextCombo == null || contextCombo.isDisposed()) ? IContext.DEFAULT : contextCombo.getText();
+        String context =
+                (contextCombo == null || contextCombo.isDisposed()) ? IContext.DEFAULT : contextCombo.getText();
 
-        JobScriptsManager manager = JobScriptsManagerFactory.createManagerInstance(exportChoiceMap, context, launcher,
-                IProcessor.NO_STATISTICS, IProcessor.NO_TRACES, getCurrentExportType1());
+        JobScriptsManager manager = JobScriptsManagerFactory
+                .createManagerInstance(exportChoiceMap, context, launcher, IProcessor.NO_STATISTICS,
+                        IProcessor.NO_TRACES, getCurrentExportType1());
         manager.setDestinationPath(getDestinationValue());
         if (log4jLevelCombo != null && !log4jLevelCombo.isDisposed()) {
             if (log4jLevelCombo.isEnabled()) {
@@ -630,8 +640,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
 
             if (str.equals(s)) {
                 if (getDefaultFileName().get(1) != null && !"".equals(getDefaultFileName().get(1))) {
-                    selectedFileName = b + ((JobExportType.OSGI.equals(jobExportType)) ? "-" : "_") + getDefaultFileName().get(1)
-                            + idealSuffix;
+                    selectedFileName = b + ((JobExportType.OSGI.equals(jobExportType)) ? "-" : "_")
+                            + getDefaultFileName().get(1) + idealSuffix;
                 } else {
                     selectedFileName = b + idealSuffix;
                 }
@@ -718,7 +728,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
                 // destination
                 for (int i = 0; i < directoryNames.length; i++) {
                     if (directoryNames[i].toLowerCase().endsWith(FileConstants.ZIP_FILE_SUFFIX)) {
-                        directoryNames[i] = (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
+                        directoryNames[i] =
+                                (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
                         addDestinationItem(directoryNames[i]);
                     }
                 }
@@ -753,7 +764,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
                 // destination
                 for (int i = 0; i < directoryNames.length; i++) {
                     if (directoryNames[i].toLowerCase().endsWith(FileConstants.ESB_FILE_SUFFIX)) {
-                        directoryNames[i] = (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
+                        directoryNames[i] =
+                                (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
                         addDestinationItem(directoryNames[i]);
                     }
                 }
@@ -796,8 +808,10 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
 
         if (getProcessItem() != null && contextCombo != null) {
             try {
-                setProcessItem((ProcessItem) ProxyRepositoryFactory.getInstance()
-                        .getUptodateProperty(getProcessItem().getProperty()).getItem());
+                setProcessItem((ProcessItem) ProxyRepositoryFactory
+                        .getInstance()
+                        .getUptodateProperty(getProcessItem().getProperty())
+                        .getItem());
             } catch (PersistenceException e) {
                 e.printStackTrace();
             }
@@ -817,7 +831,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
                 String fileName = getDefaultFileNameWithType();
                 for (int i = 0; i < directoryNames.length; i++) {
                     if (directoryNames[i].toLowerCase().endsWith(FileConstants.JAR_FILE_SUFFIX)) {
-                        directoryNames[i] = (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
+                        directoryNames[i] =
+                                (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
                         addDestinationItem(directoryNames[i]);
                     }
                 }
@@ -851,11 +866,13 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
                     // destination = dirPath.append(fileName).toOSString();
                     // }
                     if (directoryNames[i].toLowerCase().endsWith(FileConstants.ZIP_FILE_SUFFIX)) {
-                        directoryNames[i] = (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
+                        directoryNames[i] =
+                                (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
                         addDestinationItem(directoryNames[i]);
                     }
                 }
-                setDestinationValue(directoryNames[0].substring(0, (directoryNames[0].lastIndexOf("\\") + 1)) + fileName);//$NON-NLS-1$
+                setDestinationValue(
+                        directoryNames[0].substring(0, (directoryNames[0].lastIndexOf("\\") + 1)) + fileName);//$NON-NLS-1$
             } else {
                 setDefaultDestination();
             }
@@ -924,12 +941,14 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             if (directoryNames != null && directoryNames.length > 0) {
                 for (int i = 0; i < directoryNames.length; i++) {
                     if (directoryNames[i].toLowerCase().endsWith(FileConstants.ZIP_FILE_SUFFIX)) {
-                        directoryNames[i] = (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
+                        directoryNames[i] =
+                                (directoryNames[i].charAt(0) + "").toUpperCase() + directoryNames[i].substring(1);//$NON-NLS-1$
                         addDestinationItem(directoryNames[i]);
                         break;
                     }
                 }
-                setDestinationValue(directoryNames[0].substring(0, (directoryNames[0].lastIndexOf("\\") + 1)) + fileName);//$NON-NLS-1$
+                setDestinationValue(
+                        directoryNames[0].substring(0, (directoryNames[0].lastIndexOf("\\") + 1)) + fileName);//$NON-NLS-1$
             } else {
                 setDefaultDestination();
             }
@@ -1054,9 +1073,9 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             exportChoiceMap.put(ExportChoice.needJobItem, false);
             exportChoiceMap.put(ExportChoice.needSourceCode, false);
             exportChoiceMap.put(ExportChoice.binaries, !isAddMavenScript());
-            if(exportMSAsZipButton!=null) {
+            if (exportMSAsZipButton != null) {
                 exportChoiceMap.put(ExportChoice.needAssembly, exportMSAsZipButton.getSelection());
-                exportChoiceMap.put(ExportChoice.needLauncher, exportMSAsZipButton.getSelection()); 
+                exportChoiceMap.put(ExportChoice.needLauncher, exportMSAsZipButton.getSelection());
             }
             if (addBSButton != null) {
                 exportChoiceMap.put(ExportChoice.needMavenScript, addBSButton.getSelection());
@@ -1143,20 +1162,19 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
     }
 
     protected void createOptionsGroupButtons(Composite parent) {
-        
-        
+
         optionsGroupComposite = new Composite(parent, SWT.NONE);
         GridDataFactory.fillDefaults().grab(true, false).applyTo(optionsGroupComposite);
-        
+
         GridLayout gdlOptionsGroupComposite = new GridLayout();
         gdlOptionsGroupComposite.marginHeight = 0;
         gdlOptionsGroupComposite.marginWidth = 0;
         optionsGroupComposite.setLayout(gdlOptionsGroupComposite);
-        
+
         // options group
         Group optionsGroup = new Group(optionsGroupComposite, SWT.NONE);
         GridDataFactory.fillDefaults().grab(true, false).applyTo(optionsGroup);
-        
+
         optionsGroup.setText(Messages.getString("IDEWorkbenchMessages.WizardExportPage_options")); //$NON-NLS-1$
         optionsGroup.setFont(parent.getFont());
 
@@ -1219,7 +1237,7 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
     @Override
     protected boolean validateOptionsGroup() {
         setErrorMessage(null);
-        boolean isValid = super.validateOptionsGroup();
+        isValid = super.validateOptionsGroup();
 
         // TESB-13867 Export limitations for ESB 'Jobs'
         // add extra checks.
@@ -1227,7 +1245,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             String errorMsg = presenter.extraCheck(getCurrentExportType1(), getCheckNodes());
             if (errorMsg != null) {
                 setErrorMessage(errorMsg);
-                return false;
+                isValid = false;
+                // return false;
             }
         }
         setPageComplete(isValid);
@@ -1447,7 +1466,8 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
     }
 
     private String getDefaultImageName(ProcessItem procesItem) {
-        IFile pomFile = AggregatorPomsHelper.getItemPomFolder(procesItem.getProperty())
+        IFile pomFile = AggregatorPomsHelper
+                .getItemPomFolder(procesItem.getProperty())
                 .getFile(TalendMavenConstants.POM_FILE_NAME);
         String projectName = PomUtil.getPomProperty(pomFile, "talend.project.name.lowercase"); //$NON-NLS-1$
         String jobFolderPath = PomUtil.getPomProperty(pomFile, "talend.job.folder"); //$NON-NLS-1$
@@ -1468,17 +1488,21 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
     }
 
     private boolean isOptionValid(Text text, String label) {
-        boolean isValid = false;
-        if (StringUtils.isBlank(text.getText())) {
-            setErrorMessage(Messages.getString("JavaJobScriptsExportWSWizardPage.DOCKER.errorMsg", label)); //$NON-NLS-1$
-            setPageComplete(false);
-            isValid = false;
-        } else {
-            setErrorMessage(null);
-            setPageComplete(true);
-            isValid = true;
+        boolean isDockerValid = false;
+
+        // If no error message is already displayed
+        if (isValid) {
+            if (StringUtils.isBlank(text.getText())) {
+                setErrorMessage(Messages.getString("JavaJobScriptsExportWSWizardPage.DOCKER.errorMsg", label)); //$NON-NLS-1$
+                setPageComplete(false);
+                isDockerValid = false;
+            } else {
+                setErrorMessage(null);
+                setPageComplete(true);
+                isDockerValid = true;
+            }
         }
-        return isValid;
+        return isDockerValid;
     }
 
     protected void createOptionsForWS(Composite optionsGroup, Font font) {
@@ -1611,7 +1635,7 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
             String errorMsg = presenter.extraCheck(getCurrentExportType1(), getCheckNodes());
             if (errorMsg != null) {
                 setErrorMessage(errorMsg);
-                return false;
+                noError = false;
             }
         }
 
@@ -1623,16 +1647,24 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
     protected void updateOptionBySelection() {
         RepositoryNode[] selectedNodes = treeViewer.getCheckNodes();
         if (selectedNodes.length > 1) {
-            imageText.setText(getDefaultImageNamePattern());
-            imageText.setEnabled(false);
-            tagText.setText(getDefaultImageTagPattern());
-            tagText.setEnabled(false);
+            if (imageText != null) {
+                imageText.setText(getDefaultImageNamePattern());
+                imageText.setEnabled(false);
+            }
+            if (tagText != null) {
+                tagText.setText(getDefaultImageTagPattern());
+                tagText.setEnabled(false);
+            }
         } else if (selectedNodes.length == 1) {
             ProcessItem selectedProcessItem = ExportJobUtil.getProcessItem(Arrays.asList(selectedNodes));
-            imageText.setText(getDefaultImageName(selectedProcessItem));
-            imageText.setEnabled(true);
-            tagText.setText(getDefaultImageTag(selectedProcessItem));
-            tagText.setEnabled(true);
+            if (imageText != null) {
+                imageText.setText(getDefaultImageName(selectedProcessItem));
+                imageText.setEnabled(true);
+            }
+            if (tagText != null) {
+                tagText.setText(getDefaultImageTag(selectedProcessItem));
+                tagText.setEnabled(true);
+            }
         }
     }
 
@@ -1643,5 +1675,4 @@ public class JavaJobScriptsExportWSWizardPage extends JavaJobScriptsExportWizard
         return super.finish();
     }
 
-
 }

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/extrachecker/LimitESBConsumerJobChecker.java

@@ -31,7 +31,7 @@ public class LimitESBConsumerJobChecker extends AbstractJobNodeChecker {
 		}
 		String componentName = nodeType.getComponentName();
 		if (isESBConsumerComponent(componentName)) {
-			if (exportType != JobExportType.POJO) {
+			if (exportType != JobExportType.POJO && exportType != JobExportType.IMAGE) {
 				return Messages.getString("LimitESBConsumerJobChecker.limit_tRESTClient_tESBConsumer", componentName);
 			}
 			return checkLimitFeaturesWhenBuildStandalone(nodeType);

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/handler/BuildJobHandler.java

@@ -104,6 +104,7 @@ public class BuildJobHandler extends AbstractBuildJobHandler {
         //
         argumentsMap.put(TalendProcessArgumentConstant.ARG_ENABLE_STATS, isOptionChoosed(ExportChoice.addStatistics));
         argumentsMap.put(TalendProcessArgumentConstant.ARG_ENABLE_TRACS, isOptionChoosed(ExportChoice.addTracs));
+        argumentsMap.put(TalendProcessArgumentConstant.ARG_AVOID_BRANCH_NAME, isOptionChoosed(ExportChoice.avoidBranchName));
         Properties prop = (Properties) exportChoice.get(ExportChoice.properties);
         if (prop != null) { // add all properties for arugment map.
             Enumeration<Object> keys = prop.keys();

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/handler/BuildOSGiBundleHandler.java

@@ -81,11 +81,10 @@ public class BuildOSGiBundleHandler extends BuildJobHandler {
         // to compute import-package for the manifest.mf. TalendJavaProjectManager.getTalendJobJavaProject is always
         // disabled MavenNature when create project(false), it will stop jdt to compile, and imporve this part will help
         // to avoid using maven in this step.
-        final Map<String, Object> argumentsMap = new HashMap<String, Object>();
-        argumentsMap.put(TalendProcessArgumentConstant.ARG_PROGRAM_ARGUMENTS, "-Dci.builder.skip=true");
         MavenPomCommandLauncher mavenLauncher = new MavenPomCommandLauncher(talendProcessJavaProject.getProjectPom(),
                 TalendMavenConstants.GOAL_COMPILE);
-        mavenLauncher.setArgumentsMap(argumentsMap);
+        mavenLauncher.setSkipCIBuilder(true);
+        mavenLauncher.setSkipTests(true);
         mavenLauncher.execute(monitor);
 
         List<ExportFileResource> resources = osgiMavenManager

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/scriptsmanager/JobScriptsManager.java

@@ -229,7 +229,8 @@ public abstract class JobScriptsManager {
         imageTag,
         pushRegistry,
         registryUsername,
-        registryPassword
+        registryPassword,
+        avoidBranchName
     }
 
     /**

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/scriptsmanager/esb/JobJavaScriptOSGIForESBManager.java

@@ -12,14 +12,17 @@
 // ============================================================================
 package org.talend.repository.ui.wizards.exportjob.scriptsmanager.esb;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Enumeration;
@@ -30,8 +33,11 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.jar.Manifest;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.collections.map.MultiKeyMap;
+import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.Status;
@@ -55,6 +61,7 @@ import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.constants.FileConstants;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
+import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.repository.build.BuildExportManager;
 import org.talend.core.ui.branding.IBrandingService;
@@ -670,10 +677,39 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
 
     private Manifest getManifest(ExportFileResource libResource, ProcessItem processItem) throws IOException {
         Analyzer analyzer = createAnalyzer(libResource, processItem);
+        
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService service = (IRunProcessService) GlobalServiceRegister.getDefault()
+                    .getService(IRunProcessService.class);
+            ITalendProcessJavaProject talendProcessJavaProject = service.getTalendJobJavaProject(processItem.getProperty());
+            if (talendProcessJavaProject != null) {
+                String optional = ";resolution:=optional";
+                String src = JavaResourcesHelper.getJobClassFilePath(processItem, true);
+                IFile srcFile = talendProcessJavaProject.getSrcFolder().getFile(src);
+                Set<String> imports = importCompiler(srcFile.getLocation().toString());
+                String[] defaultPackages = analyzer.getProperty(Analyzer.IMPORT_PACKAGE).split(",");
+                for (String dp : defaultPackages) {
+                    if (!imports.contains(dp) && !imports.contains(dp + optional)) {
+                        imports.add(dp);
+                    }
+                }
+                imports.remove("*;resolution:=optional");
+                imports.remove("routines.system");
+                imports.remove("routines.system" + optional);
+                StringBuilder importPackage = new StringBuilder();
+                for (String packageName : imports) {
+                    importPackage.append(packageName).append(',');
+                }
+                importPackage.append("*;resolution:=optional");
+                analyzer.setProperty(Analyzer.IMPORT_PACKAGE, importPackage.toString());
+            }   
+        }
+        
         // Calculate the manifest
         Manifest manifest = null;
         try {
             manifest = analyzer.calcManifest();
+            filterImportPackages(manifest);
         } catch (IOException e) {
             throw e;
         } catch (Exception e) {
@@ -683,6 +719,38 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
         }
         return manifest;
     }
+    
+    private void filterImportPackages(Manifest manifest) {
+
+        // remove import packages which are present in private packages
+
+        List<String> privatePackages = new ArrayList<String>(); 
+        String privatePackagesString = manifest.getMainAttributes().getValue(Analyzer.PRIVATE_PACKAGE);
+        if (privatePackagesString != null) {
+            String [] packages = privatePackagesString.split(",");
+            for (String p : packages) {
+                privatePackages.add(p);
+            }
+        }
+        
+        StringBuilder fileterdImportPackage = new StringBuilder();
+        String importPackagesString = manifest.getMainAttributes().getValue(Analyzer.IMPORT_PACKAGE);
+        if (importPackagesString != null) {
+            String [] packages = importPackagesString.split(",");
+            for (String p : packages) {
+                String importPackage = p.split(";")[0];
+                if (!privatePackages.contains(importPackage) || importPackage.startsWith("routines")) {
+                    fileterdImportPackage.append(p).append(",");
+                }
+            }
+        }
+        
+        String str = fileterdImportPackage.toString();
+        if (str != null && str.length() > 0 && str.endsWith(",")) {
+            str = str.substring(0, str.length() - 1);
+        }
+        manifest.getMainAttributes().putValue(Analyzer.IMPORT_PACKAGE, str);
+    }
 
     protected Analyzer createAnalyzer(ExportFileResource libResource, ProcessItem processItem) throws IOException {
         Analyzer analyzer = new Analyzer();
@@ -900,4 +968,36 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
         return processor.getProcess();
     }
 
+    private Set<String> importCompiler(String src) {
+        Set<String> imports = new HashSet<String>();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        ByteArrayOutputStream err = new ByteArrayOutputStream();
+        try {
+            org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(src + " -1.7 -nowarn  -maxProblems 100000 ", new PrintWriter(out),
+                    new PrintWriter(err), null);
+            String errString = new String(err.toByteArray());
+            String[] errBlocks = errString.split("----------");
+            String reg = "(^[a-z_0-9\\.]+)\\.";
+            Pattern pattern = Pattern.compile(reg);
+            for (String errBlock : errBlocks) {
+                String[] lines = errBlock.trim().replaceAll("\r", "").split("\n");
+                if (lines.length == 4) {
+                    if (lines[3].endsWith("cannot be resolved to a type") || lines[3].endsWith("cannot be resolved")) {
+                        int markerPos = lines[2].indexOf('^');
+                        Matcher m = pattern.matcher(lines[1].substring(markerPos));
+                        if (m.find()) {
+                            if (m.groupCount() == 1 && m.group(1).indexOf('.') > 0) {
+                                imports.add(m.group(1) + ";resolution:=optional");
+                            }
+                        }
+                    }
+                }
+            }
+            out.close();
+            err.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return imports;
+    }
 }

test/plugins/org.talend.designer.core.generic.test/src/main/java/org/talend/designer/core/generic/service/GenericServiceTest.java

@@ -0,0 +1,66 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.core.generic.service;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.talend.commons.utils.VersionUtils;
+import org.talend.components.api.properties.ComponentProperties;
+import org.talend.components.api.properties.ComponentReferenceProperties;
+import org.talend.core.CorePlugin;
+import org.talend.core.context.Context;
+import org.talend.core.context.RepositoryContext;
+import org.talend.core.model.components.ComponentCategory;
+import org.talend.core.model.components.IComponent;
+import org.talend.core.model.properties.PropertiesFactory;
+import org.talend.core.model.properties.Property;
+import org.talend.core.ui.component.ComponentsFactoryProvider;
+import org.talend.designer.core.ui.editor.nodes.Node;
+import org.talend.designer.core.ui.editor.process.Process;
+
+/**
+ * DOC hwang  class global comment. Detailled comment
+ */
+public class GenericServiceTest {
+    
+    @Test
+    public void testResetReferenceValue() {
+          GenericService service = new GenericService();
+        //---first
+          String oldConnectionName = "joblet3_1";
+          String newConnectionName = "joblet1_1_joblet2_1_joblet3_1";
+          
+          IComponent component = ComponentsFactoryProvider.getInstance().get("tJDBCInput",
+                  ComponentCategory.CATEGORY_4_DI.getName());
+          Property property = PropertiesFactory.eINSTANCE.createProperty();
+          property.setAuthor(((RepositoryContext) CorePlugin.getContext().getProperty(Context.REPOSITORY_CONTEXT_KEY)).getUser());
+          property.setVersion(VersionUtils.DEFAULT_VERSION);
+          Process process = new Process(property);
+          Node node = new Node(component, process);
+          
+          ComponentProperties pros = node.getComponentProperties();
+          ComponentReferenceProperties comPro = (ComponentReferenceProperties) pros.getProperties("referencedComponent"); //$NON-NLS-1$
+          comPro.componentInstanceId.setValue("joblet3_1_joblet4_1_joblet5_1_tJDBCConnection_1");
+          
+          service.resetReferenceValue(node, oldConnectionName, newConnectionName);
+          assertTrue(comPro.componentInstanceId.getStoredValue().equals("joblet1_1_joblet2_1_joblet3_1_joblet4_1_joblet5_1_tJDBCConnection_1"));
+          //---second
+          oldConnectionName = "tJDBCConnection_1";
+          newConnectionName = "joblet1_1_joblet2_1_tJDBCConnection_1";
+          comPro.componentInstanceId.setValue("tJDBCConnection_1_joblet4_1_joblet5_1_tJDBCConnection_1");
+          service.resetReferenceValue(node, oldConnectionName, newConnectionName);
+          assertTrue(comPro.componentInstanceId.getStoredValue().equals("joblet1_1_joblet2_1_tJDBCConnection_1_joblet4_1_joblet5_1_tJDBCConnection_1"));
+    }
+
+}

test/plugins/org.talend.designer.core.test/src/org/talend/designer/core/model/process/DataNodeTest.java

@@ -0,0 +1,106 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.core.model.process;
+
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.talend.core.model.components.ComponentCategory;
+import org.talend.core.model.components.IComponent;
+import org.talend.core.model.process.EConnectionType;
+import org.talend.core.model.process.IConnection;
+import org.talend.core.model.process.INode;
+import org.talend.core.ui.component.ComponentsFactoryProvider;
+
+/**
+ * DOC jding  class global comment. Detailled comment
+ */
+public class DataNodeTest {
+
+    private static final String TEST_COMPONENT_NAME = "tMysqlInput";
+
+    private DataNode mainStartNode;
+
+    private DataNode lookupNode;
+
+    private DataNode tmapNode;
+
+    private DataNode logRowNode;
+
+    @Before
+    public void before() {
+        IComponent testComponent = ComponentsFactoryProvider.getInstance().get(TEST_COMPONENT_NAME,
+                ComponentCategory.CATEGORY_4_DI.getName());
+        IComponent tmapComponent = ComponentsFactoryProvider.getInstance().get("tMap", ComponentCategory.CATEGORY_4_DI.getName());
+        IComponent logrowComponent = ComponentsFactoryProvider.getInstance().get("tLogRow",
+                ComponentCategory.CATEGORY_4_DI.getName());
+        mainStartNode = new DataNode(testComponent, TEST_COMPONENT_NAME + "_1");
+        lookupNode = new DataNode(testComponent, TEST_COMPONENT_NAME + "_2");
+        tmapNode = new DataNode(tmapComponent, "tMap_1");
+        logRowNode = new DataNode(logrowComponent, "tLogRow_1");
+
+        // mainStartNode income mainStartNode->tmapNode (main row)
+        ((List<IConnection>) mainStartNode.getOutgoingConnections())
+                .add(createDataConnection(mainStartNode, tmapNode, EConnectionType.FLOW_MAIN));
+        // lookupNode outcome lookupNode->tmapNode (lookup row)
+        ((List<IConnection>) lookupNode.getOutgoingConnections())
+                .add(createDataConnection(lookupNode, tmapNode, EConnectionType.FLOW_REF));
+        ((List<IConnection>) lookupNode.getIncomingConnections())
+                .add(createDataConnection(mainStartNode, lookupNode, EConnectionType.RUN_AFTER));
+        // tmapNode income mainStartNode->tmapNode (main row) lookupNode->tmapNode (lookup row)
+        ((List<IConnection>) tmapNode.getIncomingConnections())
+                .add(createDataConnection(mainStartNode, tmapNode, EConnectionType.FLOW_MAIN));
+        ((List<IConnection>) tmapNode.getIncomingConnections())
+                .add(createDataConnection(lookupNode, tmapNode, EConnectionType.FLOW_REF));
+        // tmapNode outcome tmapNode->logRowNode (main row)
+        ((List<IConnection>) tmapNode.getOutgoingConnections())
+                .add(createDataConnection(tmapNode, logRowNode, EConnectionType.FLOW_MAIN));
+        // logRowNode income tmapNode->logRowNode (main row)
+        ((List<IConnection>) logRowNode.getIncomingConnections())
+                .add(createDataConnection(tmapNode, logRowNode, EConnectionType.FLOW_MAIN));
+
+    }
+
+    @Test
+    public void testGetSubProcessStartNode() {
+        INode startOftmap = tmapNode.getSubProcessStartNode(false);
+        Assert.assertEquals(startOftmap, mainStartNode);
+        INode startOflogrow = logRowNode.getSubProcessStartNode(false);
+        Assert.assertEquals(startOflogrow, mainStartNode);
+        INode startOflookup = lookupNode.getSubProcessStartNode(false);
+        Assert.assertEquals(startOflookup, mainStartNode);
+    }
+
+    @After
+    public void after() {
+        mainStartNode = null;
+        lookupNode = null;
+        tmapNode = null;
+        logRowNode = null;
+    }
+
+    private DataConnection createDataConnection(DataNode source, DataNode target, EConnectionType lineStyle) {
+        DataConnection dataConnec = new DataConnection();
+        dataConnec.setActivate(true);
+        dataConnec.setLineStyle(EConnectionType.FLOW_MAIN);
+        dataConnec.setConnectorName("FLOW");
+        dataConnec.setName("row_" + source.getUniqueName());
+        dataConnec.setSource(source);
+        dataConnec.setTarget(target);
+        return dataConnec;
+    }
+
+}

test/plugins/org.talend.designer.dbmap.test/src/org/talend/designer/dbmap/language/generation/DbGenerationManagerTest.java

@@ -1,7 +1,8 @@
 package org.talend.designer.dbmap.language.generation;
 
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -17,18 +18,22 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.IComponent;
+import org.talend.core.model.context.JobContext;
 import org.talend.core.model.context.JobContextManager;
+import org.talend.core.model.context.JobContextParameter;
 import org.talend.core.model.metadata.IMetadataColumn;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.MetadataColumn;
 import org.talend.core.model.metadata.MetadataTable;
 import org.talend.core.model.process.EConnectionType;
 import org.talend.core.model.process.IConnection;
+import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.process.INodeConnector;
 import org.talend.core.model.properties.PropertiesFactory;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.utils.TalendTextUtils;
 import org.talend.core.ui.component.ComponentsFactoryProvider;
+import org.talend.designer.core.model.components.EParameterName;
 import org.talend.designer.core.model.components.ElementParameter;
 import org.talend.designer.core.model.components.NodeConnector;
 import org.talend.designer.core.ui.editor.connections.Connection;
@@ -134,11 +139,85 @@ public class DbGenerationManagerTest extends DbGenerationManagerTestHelper {
 
     @Test
     public void testInitExpression() {
-        checkValue("t1.\\\"id\\\"", extMapEntry);
+    	checkValue("t1.\\\"id\\\"", extMapEntry);
         
         ExternalDbMapEntry extMapEntry2 = new ExternalDbMapEntry("multiple", "t1.id + t1.name");
         tableEntries.add(extMapEntry2);
         checkValue("t1.\\\"id\\\" + t1.\\\"name\\\"", extMapEntry2);
+        
+        testWithQuote();
+    }
+    
+    @Test
+    public void testReplaceAuotes() {
+        String quote = "\"";
+        String quoParser = "[\\\\]?\\" + quote; //$NON-NLS-1$
+        DbGenerationManager dbManager = new GenericDbGenerationManager();
+        
+        String expression = "'\"" + "+" + "(String)globalMap.get(\"BusinessDateStr\")" + "+" + "\"' BETWEEN PROD_GRP_DA.EFF_FRM_DT AND PROD_GRP_DA.EFF_TO_DT";
+        
+        assertEquals(expression.trim(), dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "'\"" + "+" + "(String)globalMap.get(\"BusinessDateStr\")" + "+" + "\" BETWEEN PROD_GRP_DA.EFF_FRM_DT AND PROD_GRP_DA.EFF_TO_DT";
+        String expect = "'\\\"" + "+" + "(String)globalMap.get(\\\"BusinessDateStr\\\")" + "+" + "\\\" BETWEEN PROD_GRP_DA.EFF_FRM_DT AND PROD_GRP_DA.EFF_TO_DT";
+        
+        assertEquals(expect.trim(), dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "BETWEEN PROD_GRP_DA.EFF_FRM_DT AND PROD_GRP_DA.EFF_TO_DT";
+        expect = "BETWEEN PROD_GRP_DA.EFF_FRM_DT AND PROD_GRP_DA.EFF_TO_DT";
+        assertEquals(expect.trim(), dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "'" +"+"+"context.param1"+"+"+ "aaa";
+        expect = "'" +"+"+"context.param1"+"+"+ "aaa";
+        
+        assertEquals(expect.trim(), dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "'\"" +"+"+"context.param1"+"+"+ "\"'aaa";
+        expect = "'\"" +"+"+"context.param1"+"+"+ "\"'aaa";
+        
+        assertEquals(expect.trim(), dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "'context.param1'";
+        assertEquals(expression, dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "table1.name = 'context.param2'";
+        assertEquals(expression, dbManager.replaceAuotes(expression, quoParser, quote).trim());
+        
+        expression = "table1.name = 'context.param2' aa";
+        assertEquals(expression, dbManager.replaceAuotes(expression, quoParser, quote).trim());
+    }
+    
+    private void testWithQuote(){
+    	dbManager.setUseDelimitedIdentifiers(true);
+    	List<IConnection> incomingConnections = new ArrayList<IConnection>();
+        String[] columns = new String[] { "id",  "name"};
+        String[] labels = new String[] { "id",  "name"};
+        incomingConnections.add(createConnection("t1", "t1", labels, columns));
+        dbMapComponent.setIncomingConnections(incomingConnections);
+        
+        ElementParameter param = new ElementParameter(dbMapComponent);
+    	param.setName(EParameterName.MAPPING.getName());
+    	param.setValue("mysql_id");
+    	List<ElementParameter> list = new ArrayList<>();
+    	list.add(param);
+    	dbMapComponent.setElementParameters(list);
+    	checkValue("t1.`id`", extMapEntry);
+    	
+    	ExternalDbMapEntry extMapEntry3 = new ExternalDbMapEntry("multiple", "if(t1.id<2500,\"<2500\",\">=2500\")");
+        tableEntries.add(extMapEntry3);
+        checkValue("if(t1.`id`<2500,\\\"<2500\\\",\\\">=2500\\\")", extMapEntry3);
+    	
+    	ExternalDbMapEntry extMapEntry2 = new ExternalDbMapEntry("multiple", "t1.id + t1.name");
+        tableEntries.add(extMapEntry2);
+        checkValue("t1.`id` + t1.`name`", extMapEntry2);
+    	
+    	param.setValue("oracle_id");
+    	checkValue("t1.\\\"id\\\"", extMapEntry);
+    	checkValue("t1.\\\"id\\\" + t1.\\\"name\\\"", extMapEntry2);
+    	
+    	param.setValue("mssql_id");
+    	checkValue("t1.\\\"id\\\"", extMapEntry);
+    	checkValue("t1.\\\"id\\\" + t1.\\\"name\\\"", extMapEntry2);
     }
 
     public void checkValue(String expression, ExternalDbMapEntry entry) {
@@ -381,4 +460,155 @@ public class DbGenerationManagerTest extends DbGenerationManagerTestHelper {
     	dbManager.appendSqlQuery(sb, "table1", false);
     	Assert.assertTrue(dbManager.getQuerySegments().isEmpty());
     }
+    
+    @Test
+    public void testBuildSqlSelect() {
+        runBuildSql("","table1","table2","table3");
+        
+    }
+    
+    private void runBuildSql(String schema, String table1, String table2, String table3) {
+        List<IConnection> incomingConnections = new ArrayList<IConnection>();
+        String[] mainTableEntities = new String[] { "id", "name" };
+        incomingConnections.add(mockConnection(schema, table1, mainTableEntities));
+        dbMapComponent.setIncomingConnections(incomingConnections);
+
+        ExternalDbMapData externalData = new ExternalDbMapData();
+        List<ExternalDbMapTable> inputs = new ArrayList<ExternalDbMapTable>();
+        List<ExternalDbMapTable> outputs = new ArrayList<ExternalDbMapTable>();
+        // main table
+        ExternalDbMapTable inputTable = new ExternalDbMapTable();
+        String mainTableName = "".equals(schema) ? table1 : schema + "." + table1;
+        // quote will be removed in the ui for connections ,so we do the same for test
+        String mainTableNameNoQuote = TalendTextUtils.removeQuotes(mainTableName);
+        inputTable.setTableName(mainTableNameNoQuote);
+        inputTable.setName(mainTableName);
+        List<ExternalDbMapEntry> entities = getMetadataEntities(mainTableEntities, new String[3]);
+        inputTable.setMetadataTableEntries(entities);
+        inputs.add(inputTable);
+
+        // output
+        ExternalDbMapTable outputTable = new ExternalDbMapTable();
+        mainTableName = "".equals(schema) ? table2 : schema + "." + table2;
+        // quote will be removed in the ui for connections ,so we do the same for test
+        mainTableNameNoQuote = TalendTextUtils.removeQuotes(mainTableName);
+        outputTable.setTableName(mainTableNameNoQuote);
+        outputTable.setName("table2");
+        String[] names = new String[] { "id", "name" };
+        String mainTable = mainTableName;
+        String[] expressions = new String[] { "table1.id", "table1.name"};
+        outputTable.setMetadataTableEntries(getMetadataEntities(names, expressions));
+        
+        String[] whereNames = new String[]{"whereFilter"};
+        String[] whereExps = new String[]{"table1.name = " + "'context.param2'"};
+        outputTable.setCustomWhereConditionsEntries(getMetadataEntities(whereNames, whereExps));
+        
+        String[] otherNames = new String[]{"otherFilter"};
+        String[] otherExps = new String[]{"table1.id = " + "'context.param1'"};
+        outputTable.setCustomOtherConditionsEntries(getMetadataEntities(otherNames, otherExps));
+        outputs.add(outputTable);
+
+        externalData.setInputTables(inputs);
+        externalData.setOutputTables(outputs);
+        dbMapComponent.setExternalData(externalData);
+        List<IMetadataTable> metadataList = new ArrayList<IMetadataTable>();
+        MetadataTable metadataTable = getMetadataTable(names);
+        metadataTable.setLabel("table2");
+        metadataList.add(metadataTable);
+        dbMapComponent.setMetadataList(metadataList);
+        JobContext newContext = new JobContext("Default");
+        List<IContextParameter> newParamList = new ArrayList<IContextParameter>();
+        newContext.setContextParameterList(newParamList);
+        JobContextParameter param = new JobContextParameter();
+        param.setName("schema");
+        newParamList.add(param);
+        process = mock(Process.class);
+        JobContextManager contextManger = new JobContextManager();
+        contextManger.setDefaultContext(newContext);
+        when(process.getContextManager()).thenReturn(contextManger);
+        dbMapComponent.setProcess(process);
+        
+        ExternalDbMapData externalData2 = new ExternalDbMapData();
+        DbMapComponent dbMapComponent2 = new DbMapComponent();
+        dbMapComponent2.setExternalData(externalData2);
+        mainTableEntities = new String[] {"id", "name"};
+        
+        List<IConnection> outgoingConnections = new ArrayList<IConnection>();
+        Node map1 = mockNode(dbMapComponent);
+        outgoingConnections.add(mockConnection(map1, schema, table2, mainTableEntities));
+        dbMapComponent.setOutgoingConnections(outgoingConnections);
+        dbMapComponent2.setIncomingConnections(outgoingConnections);
+
+        inputs = new ArrayList<ExternalDbMapTable>();
+        outputs = new ArrayList<ExternalDbMapTable>();
+        // main table
+        inputTable = new ExternalDbMapTable();
+        mainTableName = "".equals(schema) ? table2 : schema + "." + table2;
+        // quote will be removed in the ui for connections ,so we do the same for test
+        mainTableNameNoQuote = TalendTextUtils.removeQuotes(mainTableName);
+        inputTable.setTableName(mainTableNameNoQuote);
+        inputTable.setName("table2");
+        entities = getMetadataEntities(mainTableEntities, new String[3]);
+        inputTable.setMetadataTableEntries(entities);
+        inputs.add(inputTable);
+
+        // output
+        outputTable = new ExternalDbMapTable();
+        mainTableName = "".equals(schema) ? table3 : schema + "." + table3;
+        // quote will be removed in the ui for connections ,so we do the same for test
+        mainTableNameNoQuote = TalendTextUtils.removeQuotes(mainTableName);
+        outputTable.setTableName(mainTableNameNoQuote);
+        outputTable.setName("table3");
+        names = new String[] { "id", "name"};
+        mainTable = mainTableName;
+        expressions = new String[] { "table2.id", "table2.name"};
+        outputTable.setMetadataTableEntries(getMetadataEntities(names, expressions));
+        outputs.add(outputTable);
+
+        externalData2.setInputTables(inputs);
+        externalData2.setOutputTables(outputs);
+        dbMapComponent2.setExternalData(externalData2);
+        metadataList = new ArrayList<IMetadataTable>();
+        metadataTable = getMetadataTable(names);
+        metadataTable.setLabel("table3");
+        metadataList.add(metadataTable);
+        dbMapComponent2.setMetadataList(metadataList);
+        if (dbMapComponent2.getElementParameters() == null) {
+            dbMapComponent2.setElementParameters(Collections.EMPTY_LIST);
+        }
+        JobContextParameter param1 = new JobContextParameter();
+        param1.setName("param1");
+        newParamList.add(param1);
+        
+        JobContextParameter param2 = new JobContextParameter();
+        param2.setName("param2");
+        newParamList.add(param2);
+        dbMapComponent2.setProcess(process);
+        outgoingConnections = new ArrayList<IConnection>();
+        outgoingConnections.add(mockConnection(schema, table3, mainTableEntities));
+        dbMapComponent2.setOutgoingConnections(outgoingConnections);
+        
+        ElementParameter comName = new ElementParameter(dbMapComponent);
+        comName.setName("COMPONENT_NAME");
+        comName.setValue("tELTMap");
+        List<ElementParameter> list = new ArrayList<>();
+        list.add(comName);
+        dbMapComponent.setElementParameters(list);
+        
+        dbManager = new GenericDbGenerationManager();
+        String query = dbManager.buildSqlSelect(dbMapComponent2, "table3").replaceAll("\n", "");
+        String exceptQuery = "\"SELECT\n"
+                + "table2.id, table2.name\n"
+                + "FROM\n"
+                + " (\n"
+                +"  SELECT\n"
+                +"  table1.id AS id, table1.name AS name\n"
+                +"  FROM\n"
+                +"   table1\n"
+                +"  WHERE table1.name = '\" +context.param2+ \"'\n"
+                +"  table1.id = '\" +context.param1+ \"'\n"
+                +"  \n"
+                +" ) table2\"";
+        assertEquals(exceptQuery.replaceAll("\n", "").trim(), query.trim());
+    }
 }

test/plugins/org.talend.designer.dbmap.test/src/org/talend/designer/dbmap/language/generation/DbGenerationManagerTestHelper.java

@@ -17,6 +17,8 @@ import static org.mockito.Mockito.*;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.talend.core.model.components.ComponentCategory;
+import org.talend.core.model.components.IComponent;
 import org.talend.core.model.context.JobContext;
 import org.talend.core.model.context.JobContextManager;
 import org.talend.core.model.context.JobContextParameter;
@@ -27,6 +29,7 @@ import org.talend.core.model.metadata.MetadataTable;
 import org.talend.core.model.process.IConnection;
 import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.utils.TalendTextUtils;
+import org.talend.core.ui.component.ComponentsFactoryProvider;
 import org.talend.designer.core.model.components.ElementParameter;
 import org.talend.designer.core.ui.editor.connections.Connection;
 import org.talend.designer.core.ui.editor.nodes.Node;
@@ -43,6 +46,8 @@ import org.talend.designer.dbmap.external.data.ExternalDbMapTable;
 public class DbGenerationManagerTestHelper {
 
     protected DbMapComponent dbMapComponent;
+    
+    protected Process process;
 
     protected void init(String schema, String main_table, String main_alias, String lookup_table, String lookup_alias) {
         List<IConnection> incomingConnections = new ArrayList<IConnection>();
@@ -116,7 +121,7 @@ public class DbGenerationManagerTestHelper {
         param = new JobContextParameter();
         param.setName("lookup");
         newParamList.add(param);
-        Process process = mock(Process.class);
+        process = mock(Process.class);
         JobContextManager contextManger = new JobContextManager();
         contextManger.setDefaultContext(newContext);
         when(process.getContextManager()).thenReturn(contextManger);
@@ -124,9 +129,15 @@ public class DbGenerationManagerTestHelper {
 
     }
 
-    private IConnection mockConnection(String schemaName, String tableName, String[] columns) {
+    protected IConnection mockConnection(String schemaName, String tableName, String[] columns) {
+        return mockConnection(null, schemaName, tableName, columns);
+    }
+    
+    protected IConnection mockConnection(Node node, String schemaName, String tableName, String[] columns) {
         Connection connection = mock(Connection.class);
-        Node node = mock(Node.class);
+        if(node == null){
+            node = mock(Node.class);
+        }
         ElementParameter param = new ElementParameter(node);
         param.setName("ELT_SCHEMA_NAME");
         param.setValue(schemaName);
@@ -155,6 +166,15 @@ public class DbGenerationManagerTestHelper {
 
         return connection;
     }
+    
+    protected Node mockNode(DbMapComponent mapCom) {
+        Node node = mock(Node.class);
+        when(node.isELTComponent()).thenReturn(true);
+        IComponent component = ComponentsFactoryProvider.getInstance().get("tELTMap", ComponentCategory.CATEGORY_4_DI.getName());
+        when(node.getComponent()).thenReturn(component);
+        when(node.getExternalNode()).thenReturn(mapCom);
+        return node;
+    }
 
     protected List<ExternalDbMapEntry> getMetadataEntities(String[] entitiesName, String[] expressions) {
         List<ExternalDbMapEntry> entities = new ArrayList<ExternalDbMapEntry>();

test/plugins/org.talend.designer.dbmap.test/src/org/talend/designer/dbmap/utils/DataMapExpressionParserTest.java

@@ -43,6 +43,17 @@ public class DataMapExpressionParserTest {
         verifyParseResult("schema. ((String)globalMap.get(\"main_table\")).column ");
         verifyParseResult("((String)globalMap.get(\"schema\")). table.column ");
         verifyParseResult("((String)globalMap.get(\"main_table\")).column ");
+        
+        verifyIdentifiersResult("table.\\\"column\\\"");
+        verifyIdentifiersResult("Case NVL(table.\\\"column\\\",-99) When -99 Then 'N' Else 'Y' End");
+    }
+    
+    private void verifyIdentifiersResult(String expression) {
+    	TableEntryLocation[] locations = parser.parseTableEntryLocations(expression);
+        for (TableEntryLocation location : locations) {
+            assertEquals("table", location.tableName);
+            assertEquals("column", location.columnName);
+        }
     }
 
     private void verifyParseResult(String expression) {

test/plugins/org.talend.repository.test/src/org/talend/repository/handler/BuildJobHandlerTest.java

@@ -234,6 +234,8 @@ public class BuildJobHandlerTest {
 
             Properties jobInfoProp = new Properties();
             jobInfoProp.load(jobInfoStream);
+            jobInfoStream.close();
+
             assertEquals(property.getId(), jobInfoProp.getProperty("jobId"));
             assertEquals(jobName, jobInfoProp.getProperty("job"));
             assertEquals(jobVersion, jobInfoProp.getProperty("jobVersion"));
@@ -245,6 +247,14 @@ public class BuildJobHandlerTest {
 
             ZipEntry libEntry = zip.getEntry("lib");
             assertNotNull("No lib folder", libEntry);
+            if (jobItem == jobWithChildrenItem) {
+                String dependencyFromParent = "commons-lang-2.5.jar";
+                ZipEntry dependencyEntry = zip.getEntry("lib/" + dependencyFromParent);
+                assertNotNull("No parent job dependency in lib folder", dependencyEntry);
+                String dependencyFromChild = "commons-lang-2.6.jar";
+                dependencyEntry = zip.getEntry("lib/" + dependencyFromChild);
+                assertNotNull("No child job dependency in lib folder", dependencyEntry);
+            }
             if (jobItem == jobWithJobletItem) {
                 String dependencyFromJoblet = "commons-beanutils-1.9.2.jar";
                 ZipEntry dependencyEntry = zip.getEntry("lib/" + dependencyFromJoblet);