TESB-30694: cawssqs fails in Runtime with error Could not find a suitable setter for property: amazonSQSClient (#5405)

Remove phantom libraries before export. Can happen when building project after patch application.

main/plugins/org.talend.designer.codegen/jet_stub/generic/component_util_indexedrecord_to_rowstruct.javajet

@@ -62,6 +62,8 @@ class IndexedRecordToRowStructGenerator {
             %>
             boolean <%=codeVarIsDynamicInitialized%> = false;
             routines.system.Dynamic <%=codeVarDynamic%> = new routines.system.Dynamic();
+			//Workaround for TDI-44051, TcimpV0 do not need DbmsID
+			<%=codeVarDynamic%>.setDbmsId("<%=cid%>");
             <%
         }
 
@@ -115,10 +117,11 @@ class IndexedRecordToRowStructGenerator {
                 for (org.apache.avro.Schema.Field dynamicField_<%=cid%> : dynSchema_<%=cid%>.getFields()){
                     routines.system.DynamicMetadata dynamicMetadata_<%=cid%> = new routines.system.DynamicMetadata();
                     org.apache.avro.Schema dynamicFieldSchema_<%=cid%> = dynamicField_<%=cid%>.schema();
+					String dbName = dynamicField_<%=cid%>.getProp("talend.field.dbColumnName");
                     // set name
                     dynamicMetadata_<%=cid%>.setName(dynamicField_<%=cid%>.name());
                     // set db name
-                    dynamicMetadata_<%=cid%>.setDbName(dynamicField_<%=cid%>.name());
+                    dynamicMetadata_<%=cid%>.setDbName(dbName==null?dynamicField_<%=cid%>.name():dbName);
                     // set nullable
                     if (org.talend.daikon.avro.AvroUtils.isNullable(dynamicFieldSchema_<%=cid%>)) {
                         dynamicMetadata_<%=cid%>.setNullable(true);

main/plugins/org.talend.designer.components.libs/libs_src/talendsap/pom.xml

@@ -1,10 +1,10 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
-	<groupId>org.talend.libraries</groupId>
+	<groupId>org.talend.components.lib</groupId>
 	<artifactId>talendsap</artifactId>
 	<name>talendsap</name>
-	<version>1.0.1</version>
+	<version>1.0.2</version>
 
 	<properties>
 		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>

main/plugins/org.talend.designer.components.libs/libs_src/talendsap/src/main/java/org/talend/sap/DocumentExtractor.java

@@ -90,6 +90,7 @@ public class DocumentExtractor {
 		List<Element> tablesAndChangingElements = new ArrayList<Element>(3);
 		tablesAndChangingElements.add(functionElement.element("TABLES"));
 		tablesAndChangingElements.add(functionElement.element("CHANGING"));
+		tablesAndChangingElements.add(functionElement.element("OUTPUT"));
 
 		for(Element tablesOrChangingElement : tablesAndChangingElements) {
 			if (tablesOrChangingElement == null) {
@@ -121,6 +122,10 @@ public class DocumentExtractor {
 				}
 				result.add(row);
 			}
+			
+			if(!result.isEmpty()) {
+				return result;
+			}
 		}
 		
 		return result;

main/plugins/org.talend.designer.components.libs/libs_src/talendsap/src/main/java/org/talend/sap/DocumentHelper.java

@@ -49,11 +49,11 @@ public class DocumentHelper {
 		}
 	}
 
-	public void addSingleParameter(String name, String value, boolean isChanging) {
+	public void addSingleParameter(String name, String value, SAPParameterType parameter_type) {
 		if(value == null) {
 			value = "";
 		}
-		if (isChanging) {
+		if (parameter_type == SAPParameterType.CHANGING) {
 			correctChanging();
 			changing.addElement(name).setText(value);
 		} else {
@@ -62,8 +62,8 @@ public class DocumentHelper {
 		}
 	}
 
-	public void addStructParameter(String name, boolean isChanging) {
-		if (isChanging) {
+	public void addStructParameter(String name, SAPParameterType parameter_type) {
+		if (parameter_type == SAPParameterType.CHANGING) {
 			correctChanging();
 			currentStruct = changing.addElement(name);
 		} else {
@@ -79,13 +79,16 @@ public class DocumentHelper {
 		currentStruct.addElement(name).setText(value);
 	}
 
-	public void addTableParameter(String name, boolean isChanging) {
-		if (isChanging) {
+	public void addTableParameter(String name, SAPParameterType parameter_type) {
+		if (parameter_type == SAPParameterType.CHANGING) {
 			correctChanging();
 			currentTable = changing.addElement(name);
-		} else {
+		} else if(parameter_type == SAPParameterType.TABLES) {
 			correctTables();
 			currentTable = tables.addElement(name);
+		} else {
+			correctInput();
+			currentTable = input.addElement(name);
 		}
 	}
 
@@ -108,18 +111,18 @@ public class DocumentHelper {
 		DocumentHelper helper = new DocumentHelper();
 		helper.setFunctionName("READ_TABLE_FUNCTION");
 
-		helper.addSingleParameter("ID", "1", true);
-		helper.addSingleParameter("NAME", "gaoyan", false);
+		helper.addSingleParameter("ID", "1", SAPParameterType.CHANGING);
+		helper.addSingleParameter("NAME", "gaoyan", SAPParameterType.IMPORT);
 
-		helper.addStructParameter("INFO", true);
+		helper.addStructParameter("INFO", SAPParameterType.CHANGING);
 		helper.addStructChildParameter("ID", "2");
 		helper.addStructChildParameter("NAME", "wangwei");
 
-		helper.addStructParameter("INFO1", false);
+		helper.addStructParameter("INFO1", SAPParameterType.IMPORT);
 		helper.addStructChildParameter("ID1", "4");
 		helper.addStructChildParameter("NAME1", "momo");
 
-		helper.addTableParameter("TABLE1", false);
+		helper.addTableParameter("TABLE1", SAPParameterType.TABLES);
 		for (int i = 0; i < 200000; i++) {
 			helper.addTableRow();
 			helper.addTableRowChildParameter("c1", i + "");
@@ -132,7 +135,7 @@ public class DocumentHelper {
 			helper.addTableRowChildParameter("c8", "wangwei" + i);
 		}
 
-		helper.addTableParameter("TABLE2", false);
+		helper.addTableParameter("TABLE2", SAPParameterType.TABLES);
 		for (int i = 0; i < 2; i++) {
 			helper.addTableRow();
 			helper.addTableRowChildParameter("ID4", i + "");

main/plugins/org.talend.designer.components.libs/libs_src/talendsap/src/main/java/org/talend/sap/SAPParameterType.java

@@ -0,0 +1,8 @@
+package org.talend.sap;
+
+public enum SAPParameterType {
+	IMPORT,
+	CHANGING,
+	TABLES,
+	EXPORT
+}

main/plugins/org.talend.designer.components.localprovider/components/tBigQueryInput/BigQueryInputQueryHelper.javajet

@@ -21,7 +21,7 @@ class BigQueryUtil_<%=cid%> {
 	private String genTempName(String prefix){
 		return "temp_" + prefix + java.util.UUID.randomUUID().toString().replaceAll("-", "") + "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a") + Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
 	}
-	
+
 	public void cleanup() throws Exception{
 		if(useLargeResult){
 			bigqueryclient.tables().delete(projectId, tempDataset, tempTable).execute();
@@ -38,11 +38,11 @@ class BigQueryUtil_<%=cid%> {
 		job.setConfiguration(config);
 		List<com.google.api.services.bigquery.model.TableReference> referencedTables = bigqueryclient.jobs().insert(projectId, job).execute().getStatistics().getQuery().getReferencedTables();
 		if(referencedTables != null && !referencedTables.isEmpty()) {
-			location = bigqueryclient.tables().get(projectId, referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
+			location = bigqueryclient.tables().get(referencedTables.get(0).getProjectId(), referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
 		}
 		return location;
 	}
-	
+
 	private void createDataset(String location) throws Exception {
 		com.google.api.services.bigquery.model.Dataset dataset = new com.google.api.services.bigquery.model.Dataset().setDatasetReference(new com.google.api.services.bigquery.model.DatasetReference().setProjectId(projectId).setDatasetId(tempDataset));
 		if(location != null) {
@@ -53,11 +53,11 @@ class BigQueryUtil_<%=cid%> {
     	dataset.setDescription(description);
     	bigqueryclient.datasets().insert(projectId, dataset).execute();
 	}
-	
+
 	public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult) throws Exception{
 		return executeQuery(query, useLargeResult, true);
 	}
-	
+
 	public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult, boolean useLegacySql) throws Exception{
 		com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig = new com.google.api.services.bigquery.model.JobConfigurationQuery();
 		queryConfig.setQuery(query);
@@ -73,13 +73,13 @@ class BigQueryUtil_<%=cid%> {
 											.setDatasetId(tempDataset)
 											.setTableId(tempTable));
 		}
-		
+
 		com.google.api.services.bigquery.model.JobConfiguration config = new com.google.api.services.bigquery.model.JobConfiguration();
 		config.setQuery(queryConfig);
-		
+
 		com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job();
 		job.setConfiguration(config);
-		
+
 		com.google.api.services.bigquery.model.Job insert = null;
 		com.google.api.services.bigquery.model.JobReference jobId = null;
 		try {
@@ -121,7 +121,7 @@ class BigQueryUtil_<%=cid%> {
 			}
 			throw e;
 		}
-		
+
 		<%
 		if(isLog4jEnabled){
 		%>
@@ -145,7 +145,7 @@ class BigQueryUtil_<%=cid%> {
 							<%
 							}
 							%>
-							return executeQuery(query, true);
+							return executeQuery(query, true, useLegacySql);
 						}
 					<%}%>
 					// Do not throw exception to avoid behavior changed(because it may throw "duplicate" exception which do not throw before);
@@ -163,13 +163,13 @@ class BigQueryUtil_<%=cid%> {
 					}// else job successful
 					break;
 				}
-				// Pause execution for one second before polling job status again, to 
+				// Pause execution for one second before polling job status again, to
 				// reduce unnecessary calls to the BigQUery API and lower overall
 				// application bandwidth.
 				Thread.sleep(1000);
 			}
-		
+
 			return insert;
 		}
  		
-	}
+	}

main/plugins/org.talend.designer.components.localprovider/components/tDTDValidator/tDTDValidator_main.javajet

@@ -45,8 +45,8 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
 		java.lang.StringBuilder sb<%=cid %>=new java.lang.StringBuilder("");
 		
 		try{
-    		br<%=cid %> = new java.io.BufferedReader(new java.io.InputStreamReader(url<%=cid %>.openStream()));
-    		
+    		br<%=cid %> = new java.io.BufferedReader(new UnicodeReader(url<%=cid %>.openStream(), null));
+
     		char[] buffer<%=cid %> = new char[1024];
     		int length<%=cid %> = -1;
     		while ((length<%=cid %> = br<%=cid %>.read(buffer<%=cid %>)) != -1)
@@ -66,7 +66,9 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
 		dbf<%=cid %> = javax.xml.parsers.DocumentBuilderFactory.newInstance();
 		dbf<%=cid %>.setValidating(false);
 		db<%=cid %> = dbf<%=cid %>.newDocumentBuilder(); 
-		org.w3c.dom.Document doc<%=cid %> = db<%=cid %>.parse(new java.io.StringBufferInputStream(sb<%=cid%>.toString()));
+		org.w3c.dom.Document doc<%=cid %> = db<%=cid %>.parse(
+			new java.io.ByteArrayInputStream(sb<%=cid%>.toString().getBytes("UTF-8")));
+
 		String rootnode<%=cid %>=doc<%=cid %>.getDocumentElement().getNodeName();
 		
 		String encoding=null;
@@ -89,8 +91,8 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
 		
 		int offsetRoot<%=cid %>=sb<%=cid %>.indexOf("<"+rootnode<%=cid %>);
 		sb<%=cid%>.replace(0, offsetRoot<%=cid %>, reference<%=cid %>);		
-		is<%=cid %>=new java.io.StringBufferInputStream(sb<%=cid %>.toString());
-		
+		is<%=cid %> = new java.io.ByteArrayInputStream(sb<%=cid%>.toString().getBytes("UTF-8"));
+
 		class MyHandler<%=cid %> extends org.xml.sax.helpers.DefaultHandler{
 		    String errorMessage = null;
 		    public void error(org.xml.sax.SAXParseException e) throws org.xml.sax.SAXException {

main/plugins/org.talend.designer.components.localprovider/components/tELTSybaseOutput/tELTSybaseOutput_main.javajet

@@ -107,7 +107,7 @@ imports="
 		}
 		else{
 %>
-    String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+    String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 		}
             }

main/plugins/org.talend.designer.components.localprovider/components/tFileInputMail/tFileInputMail_MIME.javajet

@@ -66,25 +66,28 @@
 			}
 		}
 		DecodeString_<%=cid%> decode_<%=cid%> = new DecodeString_<%=cid%>();
+
 		try{
 			fileInput<%=cid%> = new java.io.FileInputStream(<%=filename %>);
 			javax.mail.Session session_<%=cid %> = javax.mail.Session.getInstance(System.getProperties(), null);
-		    javax.mail.internet.MimeMessage msg_<%=cid %> = new javax.mail.internet.MimeMessage(session_<%=cid %>, fileInput<%=cid%>);
+			javax.mail.internet.MimeMessage msg_<%=cid %> = new javax.mail.internet.MimeMessage(session_<%=cid %>, fileInput<%=cid%>);
 			java.util.List<String> list_<%=cid %> = new java.util.ArrayList<String>();
-		
-			for (int i_<%=cid %> =0;i_<%=cid %> < mailParts_<%=cid %>.length;i_<%=cid %>++) {				
-			String part_<%=cid %> = mailParts_<%=cid %>[i_<%=cid %>];
-	       	String sep_<%=cid%>= mailSeparator_<%=cid %>[i_<%=cid %>];
-		        if(part_<%=cid %>.equalsIgnoreCase("body")) {        
+
+			for (int i_<%=cid %> = 0; i_<%=cid %> < mailParts_<%=cid %>.length; i_<%=cid %>++) {
+				String part_<%=cid %> = mailParts_<%=cid %>[i_<%=cid %>];
+				String sep_<%=cid%>= mailSeparator_<%=cid %>[i_<%=cid %>];
+				if(part_<%=cid %>.equalsIgnoreCase("body")) {
+					boolean multiValueBody_<%=cid%> = ("true").equals(mailChecked_<%=cid %>[i_<%=cid%>]);
 					if(msg_<%=cid %>.isMimeType("multipart/*")) {
 						javax.mail.Multipart mp<%=cid%> = (javax.mail.Multipart) msg_<%=cid %>.getContent();
-	             		for (int i = 0; i < mp<%=cid%>.getCount(); i++) {
-	                 		javax.mail.BodyPart mpart<%=cid%> = mp<%=cid%>.getBodyPart(i);
-	                 		String disposition<%=cid%> = mpart<%=cid%>.getDisposition();
-	                 		if (!((disposition<%=cid%> != null) && ((disposition<%=cid%>
-	                            .equals(javax.mail.Part.ATTACHMENT)) || (disposition<%=cid%>.equals(javax.mail.Part.INLINE))))) {   
-	                    		// the following extract the body part(text/plain + text/html)
-	                    		try{
+						StringBuilder body_<%=cid %> = new StringBuilder();
+						for (int i = 0; i < mp<%=cid%>.getCount(); i++) {
+							javax.mail.BodyPart mpart<%=cid%> = mp<%=cid%>.getBodyPart(i);
+							String disposition<%=cid%> = mpart<%=cid%>.getDisposition();
+							if (!((disposition<%=cid%> != null) && ((disposition<%=cid%>
+								.equals(javax.mail.Part.ATTACHMENT)) || (disposition<%=cid%>.equals(javax.mail.Part.INLINE))))) {
+								// the following extract the body part(text/plain + text/html)
+								try{
 									Object content_<%=cid %> = mpart<%=cid %>.getContent();
 									if (content_<%=cid %> instanceof javax.mail.internet.MimeMultipart) {
 										javax.mail.internet.MimeMultipart mimeMultipart_<%=cid %> = (javax.mail.internet.MimeMultipart) content_<%=cid %>;
@@ -99,130 +102,152 @@
 												for(int j_<%=cid %>_body = 0; j_<%=cid %>_body < mimeMultipart_<%=cid %>_body.getCount(); j_<%=cid %>_body++){
 													javax.mail.BodyPart bodyPart_<%=cid %>_body = mimeMultipart_<%=cid %>_body.getBodyPart(j_<%=cid %>_body);
 													if (bodyPart_<%=cid %>_body.isMimeType("text/*")) {
-														list_<%=cid %>.add(bodyPart_<%=cid %>_body.getContent().toString());
+														if(multiValueBody_<%=cid%>) {
+															body_<%=cid %>.append(bodyPart_<%=cid %>_body.getContent().toString()).append(sep_<%=cid%>);
+														} else {
+															list_<%=cid %>.add(bodyPart_<%=cid %>_body.getContent().toString());
+														}
 													} else {
 														System.out.println("Ignore the part " + bodyPart_<%=cid %>_body.getContentType());
 													}
 												}
 											}else{
 												if (bodyPart_<%=cid %>.isMimeType("text/*")) {
-													list_<%=cid %>.add(bodyPart_<%=cid %>.getContent().toString());
+													if(multiValueBody_<%=cid%>) {
+														body_<%=cid %>.append(bodyPart_<%=cid %>.getContent().toString()).append(sep_<%=cid%>);
+													} else {
+														list_<%=cid %>.add(bodyPart_<%=cid %>.getContent().toString());
+													}
 												} else {
 													System.out.println("Ignore the part " + bodyPart_<%=cid %>.getContentType());
 												}
 											}
 										}
 									} else {    
-										list_<%=cid %>.add(mpart<%=cid%>.getContent().toString());
+										if(multiValueBody_<%=cid%>) {
+											body_<%=cid %>.append(mpart<%=cid%>.getContent().toString()).append(sep_<%=cid%>);
+										} else {
+											list_<%=cid %>.add(mpart<%=cid%>.getContent().toString());
+										}
 									}    
 								}catch (java.io.UnsupportedEncodingException e){
 									java.io.ByteArrayOutputStream bao_<%=cid%> = new java.io.ByteArrayOutputStream();
 									mpart<%=cid %>.writeTo(bao_<%=cid%>);
-									list_<%=cid %>.add(bao_<%=cid%>.toString());
+									if(multiValueBody_<%=cid%>) {
+										body_<%=cid %>.append(bao_<%=cid%>.toString()).append(sep_<%=cid%>);
+									} else {
+										list_<%=cid %>.add(bao_<%=cid%>.toString());
+									}
 									<%if(isLog4jEnabled){%>
 										log.warn("<%=cid%> - " + bao_<%=cid%>.toString());
 									<%}%>
 								}
 								<%
-				 				//both attachment and message context in the email,bug TDI-19065
+								//both attachment and message context in the email,bug TDI-19065
 								%>
 							}else if(disposition<%=cid%> != null && disposition<%=cid%>.equals(javax.mail.Part.INLINE)){
-								list_<%=cid %>.add(mpart<%=cid%>.getContent().toString());
-					 		}
+								if(multiValueBody_<%=cid%>) {
+									body_<%=cid %>.append(mpart<%=cid%>.getContent().toString()).append(sep_<%=cid%>);
+								} else {
+									list_<%=cid %>.add(mpart<%=cid%>.getContent().toString());
+								}
+							}
+						}
+						if(multiValueBody_<%=cid%>) {
+							list_<%=cid %>.add(body_<%=cid %>.toString());
 						}
 					} else {
 						java.io.InputStream in_<%=cid %> = msg_<%=cid %>.getInputStream();
-	     		    	byte[] buffer_<%=cid %> = new byte[1024];
-		     		    int length_<%=cid %> = 0;
-		     		    java.io.ByteArrayOutputStream baos_<%=cid %> =  new java.io.ByteArrayOutputStream();
-		     		    while ((length_<%=cid %> = in_<%=cid %>.read(buffer_<%=cid %>, 0, 1024)) != -1) {
-		     		        baos_<%=cid %>.write(buffer_<%=cid %>, 0, length_<%=cid %>);
-		     		    }
-		     		    String contentType_<%=cid%> = msg_<%=cid%>.getContentType();
-		     		    String charsetName_<%=cid%> = "";
-		     		    if(contentType_<%=cid%>!=null && contentType_<%=cid%>.trim().length()>0){
-		     		    	javax.mail.internet.ContentType cy_<%=cid%>  = new javax.mail.internet.ContentType(contentType_<%=cid%>);
-		     		    	charsetName_<%=cid%> = cy_<%=cid%>.getParameter("charset");
-		     		    }
-		     		    if(charsetName_<%=cid%>!=null && charsetName_<%=cid%>.length()>0){
-		     		    	list_<%=cid %>.add(baos_<%=cid %>.toString(charsetName_<%=cid%>));
-		     		    }else{
-		     		    	list_<%=cid %>.add(baos_<%=cid %>.toString());
-		     		    }
-		     		    in_<%=cid %>.close();
-		     		    baos_<%=cid %>.close();
+						byte[] buffer_<%=cid %> = new byte[1024];
+						int length_<%=cid %> = 0;
+						java.io.ByteArrayOutputStream baos_<%=cid %> =  new java.io.ByteArrayOutputStream();
+						while ((length_<%=cid %> = in_<%=cid %>.read(buffer_<%=cid %>, 0, 1024)) != -1) {
+							baos_<%=cid %>.write(buffer_<%=cid %>, 0, length_<%=cid %>);
+						}
+						String contentType_<%=cid%> = msg_<%=cid%>.getContentType();
+						String charsetName_<%=cid%> = "";
+						if(contentType_<%=cid%>!=null && contentType_<%=cid%>.trim().length()>0){
+							javax.mail.internet.ContentType cy_<%=cid%>  = new javax.mail.internet.ContentType(contentType_<%=cid%>);
+							charsetName_<%=cid%> = cy_<%=cid%>.getParameter("charset");
+						}
+						if(charsetName_<%=cid%>!=null && charsetName_<%=cid%>.length()>0){
+							list_<%=cid %>.add(baos_<%=cid %>.toString(charsetName_<%=cid%>));
+						}else{
+							list_<%=cid %>.add(baos_<%=cid %>.toString());
+						}
+						in_<%=cid %>.close();
+						baos_<%=cid %>.close();
 					}
-	        	}else if(part_<%=cid %>.equalsIgnoreCase("header")){
-		            java.util.Enumeration em = msg_<%=cid %>.getAllHeaderLines();
-		            int em_count=0;
-		            
-		            String tempStr_<%=cid %>="";
-		            
+				}else if(part_<%=cid %>.equalsIgnoreCase("header")){
+					java.util.Enumeration em = msg_<%=cid %>.getAllHeaderLines();
+					int em_count=0;
+
+					String tempStr_<%=cid %>="";
+
 					while (em.hasMoreElements()) {
 						tempStr_<%=cid %> = tempStr_<%=cid %> + (String) em.nextElement() + sep_<%=cid%> ;
 					}
 					list_<%=cid%>.add(decode_<%=cid%>.decode(tempStr_<%=cid%>));
-	        	}else{
-		        	if(("true").equals(mailChecked_<%=cid %>[i_<%=cid%>])){   
+				} else {
+					if(("true").equals(mailChecked_<%=cid %>[i_<%=cid%>])){
 						String[] sa_<%=cid%> = msg_<%=cid %>.getHeader(part_<%=cid%>);
 						String tempStr_<%=cid%>="";
 						for(int i=0;i<sa_<%=cid%>.length;i++){
 							tempStr_<%=cid%>=tempStr_<%=cid%>+sa_<%=cid%>[i] + sep_<%=cid%>;
 						}
 						list_<%=cid%>.add(decode_<%=cid%>.decode(tempStr_<%=cid%>));
-		        	}else{ 
-			           String content_<%=cid %> = msg_<%=cid %>.getHeader(part_<%=cid %>, null);
-			           list_<%=cid %>.add(decode_<%=cid%>.decode(content_<%=cid %>));
-		           	}    
-		        }   
-		 	}   
-	
-
-	
-			//attachment Deal
-			if(msg_<%=cid %>.isMimeType("multipart/*")){
-	 	    	javax.mail.Multipart mp<%=cid%> = (javax.mail.Multipart) msg_<%=cid %>.getContent();
-	 	    	String attachfileName<%=cid%> = "";
-	 	    	String path<%=cid%> = "";
-	 	    	java.io.BufferedOutputStream out<%=cid%> = null;
-	 	    	java.io.BufferedInputStream in<%=cid%> = null;
-	            for (int i = 0; i < mp<%=cid%>.getCount(); i++) {
-	                javax.mail.BodyPart mpart<%=cid%> = mp<%=cid%>.getBodyPart(i);
-	                String disposition<%=cid%> = mpart<%=cid%>.getDisposition();
-					<%
-			    	// fixed bug TDI-8586,to deal with attachments download
-					%>                
-	                if (mpart<%=cid%>.getFileName()!=null
-	                    && ((disposition<%=cid%> != null && (disposition<%=cid%>.equals(javax.mail.Part.ATTACHMENT) || disposition<%=cid%>.equals(javax.mail.Part.INLINE)))
-						||  disposition<%=cid%>==null)) {	<%// TDI-29179 %>
-	        	        attachfileName<%=cid%> = mpart<%=cid%>.getFileName();
-	                    attachfileName<%=cid%> = javax.mail.internet.MimeUtility.decodeText(attachfileName<%=cid%>);
-	                   
-	                    
-						if(!(<%=directory%>).endsWith("/")){
-	           				path<%=cid%> = <%=directory%> + "/";
-	        			}else{
-							path<%=cid%> =<%=directory%>;
-						}
-	                    path<%=cid%> = path<%=cid%> + attachfileName<%=cid%>;
-	                    <% if(isLog4jEnabled){ %>
-		                    	log.info("<%= cid %> - Extracted attachment: '"+attachfileName<%=cid%>+"'.");
-	                    <% } %>
-	                    java.io.File attachFile  = new java.io.File(path<%=cid%>);
-	                    out<%=cid%> = new java.io.BufferedOutputStream(new java.io.FileOutputStream(attachFile));
-	                    in<%=cid%> = new java.io.BufferedInputStream(mpart<%=cid%>.getInputStream());
-	                    int buffer<%=cid%> = 0;
-	                    while ((buffer<%=cid%> = in<%=cid%>.read()) != -1) {
-	                           out<%=cid%>.write(buffer<%=cid%>);
-	                           out<%=cid%>.flush();
-						}   
-	                    out<%=cid%>.close();
-	                    in<%=cid%>.close();    
-	                }
-	            }
+					}else{
+					   String content_<%=cid %> = msg_<%=cid %>.getHeader(part_<%=cid %>, null);
+					   list_<%=cid %>.add(decode_<%=cid%>.decode(content_<%=cid %>));
+					}
+				}
 			}
- 	         
- 	          
+
+            //attachment Deal
+            class MessagePartProcessor {
+                void saveAttachment(javax.mail.Part mpart) throws IOException, javax.mail.MessagingException {
+                    if (mpart.getFileName() != null && (
+                            mpart.getDisposition() == null ||
+                            (mpart.getDisposition().equals(javax.mail.Part.ATTACHMENT) || mpart.getDisposition().equals(javax.mail.Part.INLINE))
+                    )) {
+                        String attachFileName = javax.mail.internet.MimeUtility.decodeText(mpart.getFileName());
+
+                        String path = <%=directory%>;
+                        if(!path.endsWith("/")){
+                            path = path + "/";
+                        }
+                        path = path + attachFileName;
+                        <% if(isLog4jEnabled){ %>
+                                log.info("<%= cid %> - Extracted attachment: '" + attachFileName + "'.");
+                        <% } %>
+
+                        java.io.File attachFile = new java.io.File(path);
+                        java.io.BufferedOutputStream out = new java.io.BufferedOutputStream(new java.io.FileOutputStream(attachFile));
+                        java.io.BufferedInputStream in = new java.io.BufferedInputStream(mpart.getInputStream());
+                        int buffer = 0;
+                        while ((buffer = in.read()) != -1) {
+                            out.write(buffer);
+                            out.flush();
+                        }
+                        out.close();
+                        in.close();
+                    }
+                }
+
+                // recursively process body parts
+                void processPart(javax.mail.Part part) throws javax.mail.MessagingException, IOException {
+                    if (part.isMimeType("multipart/*")) {
+                        javax.mail.Multipart multipartContent = (javax.mail.Multipart) part.getContent();
+                        for (int i = 0; i < multipartContent.getCount(); i++) {
+                            javax.mail.Part mpart = multipartContent.getBodyPart(i);
+                            saveAttachment(mpart);
+                            processPart(mpart);
+                        }
+                    }
+                }
+            }
+            new MessagePartProcessor().processPart(msg_<%=cid %>);
+
 			// for output
 			<%
 			List< ? extends IConnection> conns = node.getOutgoingSortedConnections();
@@ -282,10 +307,10 @@
 						for (int i=1;i<conns.size();i++) {
 							IConnection conn2 = conns.get(i);
 							if ((conn2.getName().compareTo(firstConnName)!=0)&&(conn2.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA))) {
-				    			for (IMetadataColumn column: metadata.getListColumns()) {%>
-	      							<%=conn2.getName() %>.<%=column.getLabel() %> = <%=firstConnName %>.<%=column.getLabel() %>;
-				 				<%
-					 			}
+								for (IMetadataColumn column: metadata.getListColumns()) {%>
+									<%=conn2.getName() %>.<%=column.getLabel() %> = <%=firstConnName %>.<%=column.getLabel() %>;
+								<%
+								}
 							}
 						}
 					}

main/plugins/org.talend.designer.components.localprovider/components/tFileInputMail/tFileInputMail_begin.javajet

@@ -37,6 +37,17 @@ if ((metadatas!=null)&&(metadatas.size()>0)) {
 		}else{
 			globalMap.put("<%=cid %>_EXPORTED_FILE_PATH",<%=directory%>);
 		}
+
+		// create output directory if not exists
+		String path = <%=directory%>;
+		if(!path.endsWith("/")){
+			path = path + "/";
+		}
+		java.io.File outputDir = new java.io.File(path);
+		if (!outputDir.exists()) {
+			outputDir.mkdirs();
+		}
+
 		boolean hasData_<%=cid%> = false;
 		<%
 		if("MIME".equals(mailType)) {

main/plugins/org.talend.designer.components.localprovider/components/tGSConnection/tGSConnection_java.xml

@@ -62,8 +62,8 @@
 	    <IMPORTS>
 	       	<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0"  REQUIRED="true" />
 			<IMPORT NAME="commons-logging-1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED="true" />
-			<IMPORT NAME="httpclient-4.3.6" MODULE="httpclient-4.3.6.jar" MVN="mvn:org.talend.libraries/httpclient-4.3.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.3.6.jar" REQUIRED="true" />
-			<IMPORT NAME="httpcore-4.3.3" MODULE="httpcore-4.3.3.jar" MVN="mvn:org.talend.libraries/httpcore-4.3.3/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.3.3.jar" REQUIRED="true" />
+			<IMPORT NAME="httpclient-4.5.5" MODULE="httpclient-4.5.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.5.5/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpclient-4.5.5.jar" REQUIRED="true" />
+			<IMPORT NAME="httpcore-4.4.9" MODULE="httpcore-4.4.9.jar" MVN="mvn:org.talend.libraries/httpcore-4.4.9/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpcore-4.4.9.jar" REQUIRED="true" />
 			<IMPORT NAME="commons-codec-1.6" MODULE="commons-codec-1.6.jar" MVN="mvn:org.talend.libraries/commons-codec-1.6/6.0.0"  UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-codec-1.6.jar" REQUIRED="true" />
 		</IMPORTS>
 	</CODEGENERATION>

main/plugins/org.talend.designer.components.localprovider/components/tMap/tMap_begin.inc.javajet

@@ -454,8 +454,15 @@ for (int iInputTable = 0; iInputTable < sizeInputTables - 1; iInputTable++) { //
 		        		IMetadataTable table = connection.getMetadataTable();
 						List<IMetadataColumn> listColumns = table.getListColumns();
                     	for (IMetadataColumn column : listColumns) {
-							%>this.<%= connection.getName() %>__<%= column.getLabel() %> = <%= connection.getName() %>.<%= column.getLabel() %>;
+                    		if ("Dynamic".equals(JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable()))) {
+							%>
+								this.<%= connection.getName() %>__<%= column.getLabel() %> = (routines.system.Dynamic) <%= connection.getName() %>.<%= column.getLabel() %>.clone();
 	                		<%
+	                		} else {
+	                		%>
+	                			this.<%= connection.getName() %>__<%= column.getLabel() %> = <%= connection.getName() %>.<%= column.getLabel() %>;
+	                		<%
+	                		}
                 		}
                 	}
                 	%>
@@ -463,10 +470,15 @@ for (int iInputTable = 0; iInputTable < sizeInputTables - 1; iInputTable++) { //
                     sizeNextColumns = nextColumnsKeys.size();
                     for (int iColumn = 0; iColumn < sizeNextColumns; iColumn++) { // T_TM_B_104
                     	IMetadataColumn column = nextColumnsKeys.get(iColumn);
-
-						%>this.exprKey_<%= nextInputTable.getName() %>__<%=column.getLabel() %> = exprKey_<%= nextInputTable.getName() %>__<%=column.getLabel() %>;
+						if ("Dynamic".equals(JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable()))) {
+						%>
+							this.exprKey_<%= nextInputTable.getName() %>__<%=column.getLabel() %> = (routines.system.Dynamic) exprKey_<%= nextInputTable.getName() %>__<%=column.getLabel() %>.clone();
 						<%
-                    
+						} else {
+						%>
+							this.exprKey_<%= nextInputTable.getName() %>__<%=column.getLabel() %> = exprKey_<%= nextInputTable.getName() %>__<%=column.getLabel() %>;
+						<%
+                    	}
                     } // T_TM_B_104
                     %>
                 }
@@ -653,14 +665,25 @@ for (int iInputTable = 0; iInputTable < sizeInputTables - 1; iInputTable++) { //
 					
 					<%
 				} else {
-					typeToGenerate =JavaTypesManager.getTypeToGenerate(column.getTalendType(), false);
-					typeToGenerate=typeToGenerate.substring(0,1).toUpperCase()+typeToGenerate.substring(1);
+					typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), false);
+					typeToGenerate = typeToGenerate.substring(0,1).toUpperCase()+typeToGenerate.substring(1);
 					%>
 			            length = dis.readByte();
            				if (length == -1) {
            	    			this.<%= connection.getName() %>__<%= column.getLabel() %> = null;
            				} else {
-           					this.<%= connection.getName() %>__<%= column.getLabel() %> = dis.read<%= typeToGenerate %>();
+           					<%
+           						if ("Dynamic".equals(typeToGenerate)) {
+           							hasAtLeastOneObjectType = true;
+           					%>
+           							this.<%= connection.getName() %>__<%= column.getLabel() %> = (<%= typeToGenerate %>) dis.readObject();
+           					<%
+           						} else {
+           					%>
+           							this.<%= connection.getName() %>__<%= column.getLabel() %> = dis.read<%= typeToGenerate %>();
+           					<%
+           						}
+           					%>
            				}
 					<%
 				}
@@ -727,14 +750,25 @@ for (int iInputTable = 0; iInputTable < sizeInputTables - 1; iInputTable++) { //
 					
 					<%
 				} else {
-					typeToGenerate =JavaTypesManager.getTypeToGenerate(column.getTalendType(), false);
-					typeToGenerate=typeToGenerate.substring(0,1).toUpperCase()+typeToGenerate.substring(1);
+					typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), false);
+					typeToGenerate = typeToGenerate.substring(0,1).toUpperCase()+typeToGenerate.substring(1);
 					%>
 			            length = dis.readByte();
            				if (length == -1) {
            	    			this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %> = null;
            				} else {
-           					this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %> = dis.read<%= typeToGenerate %>();
+           					<%
+           						if ("Dynamic".equals(typeToGenerate)) {
+           							hasAtLeastOneObjectType = true;
+           					%>
+           							this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %> = (<%= typeToGenerate %>) dis.readObject();
+           					<%
+           						} else {
+           					%>
+           							this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %> = dis.read<%= typeToGenerate %>();
+           					<%
+           						}
+           					%>
            				}
 					<%
 				}
@@ -832,7 +866,17 @@ for (int iInputTable = 0; iInputTable < sizeInputTables - 1; iInputTable++) { //
 			                dos.writeByte(-1);
 						} else {
                				dos.writeByte(0);
-           			    	dos.write<%= typeToGenerate %>(this.<%= connection.getName() %>__<%= column.getLabel() %>);
+           			    	<%
+           						if ("Dynamic".equals(typeToGenerate)) {
+           					%>
+           							dos.writeObject(this.<%= connection.getName() %>__<%= column.getLabel() %>);
+           					<%
+           						} else {
+           					%>
+           							dos.write<%= typeToGenerate %>(this.<%= connection.getName() %>__<%= column.getLabel() %>);
+           					<%
+           						}
+           					%>
 		            	}
 					<%
 				
@@ -893,7 +937,17 @@ for (int iInputTable = 0; iInputTable < sizeInputTables - 1; iInputTable++) { //
 			                dos.writeByte(-1);
 						} else {
                				dos.writeByte(0);
-           			    	dos.write<%= typeToGenerate %>(this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %>);
+           			    	<%
+           						if ("Dynamic".equals(typeToGenerate)) {
+           					%>
+           							dos.writeObject(this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %>);
+           					<%
+           						} else {
+           					%>
+           							dos.write<%= typeToGenerate %>(this.exprKey_<%= nextInputTable.getName() %>__<%= column.getLabel() %>);
+           					<%
+           						}
+           					%>
 		            	}
 					<%
 				

main/plugins/org.talend.designer.components.localprovider/components/tS3Put/tS3Put_begin.javajet

@@ -32,6 +32,13 @@ imports="
 	String key_<%=cid%> = <%=key%>;
 	
 	int partSizeInBytes_<%=cid%> = <%=partSizeMb%> * 1024 * 1024;
+	if(partSizeInBytes_<%=cid%> < 5 << 20 ) {
+		<%if(isLog4jEnabled) {%>
+		log.info("Set part size as 5MB, as it is the recommended minimun value.");
+		<%}%>
+		partSizeInBytes_<%=cid%> = 5 << 20;
+	}
+	
 	
 	Object fileOrStream_<%=cid%> = <%=file%>;
 	

main/plugins/org.talend.designer.components.localprovider/components/tSSH/tSSH_begin.javajet

@@ -189,4 +189,6 @@ if ((metadatas!=null)&&(metadatas.size()>0)) {
 }
 %>
 		StringBuilder stringStdout_<%=cid%> =new StringBuilder();
-        StringBuilder stringStderr_<%=cid%> =new StringBuilder();
+		StringBuilder stringStderr_<%=cid%> =new StringBuilder();
+		String out_content_<%=cid%> = null;
+		String err_content_<%=cid%> = null;

main/plugins/org.talend.designer.components.localprovider/components/tSSH/tSSH_end.javajet

@@ -23,7 +23,7 @@ imports="
 			if("TO_CONSOLE".equals(standardOutput) || "TO_CONSOLE_AND_GLOBAL_VARIABLE".equals(standardOutput)){
 %>
 				if(stringStdout_<%=cid%>.length() > 0) {
-					System.out.println(stringStdout_<%=cid%>.toString());
+					System.out.println(out_content_<%=cid%>);
 				}
 <%
 			}
@@ -31,7 +31,7 @@ imports="
 			if("TO_CONSOLE".equals(errorOutput) || "TO_CONSOLE_AND_GLOBAL_VARIABLE".equals(errorOutput)){
 %>
 				if(stringStderr_<%=cid%>.length() > 0) {
-					System.out.println(stringStderr_<%=cid%>.toString());
+					System.out.println(err_content_<%=cid%>);
 				}
 <%
 			}

main/plugins/org.talend.designer.components.localprovider/components/tSSH/tSSH_main.javajet

@@ -77,7 +77,9 @@ if ((metadatas!=null)&&(metadatas.size()>0)) {
 <%
 		}
 %>
-		java.io.InputStream stdout_<%=cid%> = sess_<%=cid%>.getStdout();
+		java.io.InputStream stdout_<%=cid%> = new ch.ethz.ssh2.StreamGobbler(sess_<%=cid%>.getStdout());
+		java.io.InputStream stderr_<%=cid%> = new ch.ethz.ssh2.StreamGobbler(sess_<%=cid%>.getStderr());
+		
 		java.io.BufferedReader brout_<%=cid%>= new java.io.BufferedReader(new java.io.InputStreamReader(stdout_<%=cid%>));
         String line_<%=cid%> = "";
 
@@ -91,7 +93,6 @@ if ((metadatas!=null)&&(metadatas.size()>0)) {
 	    brout_<%=cid%>.close();
 		stdout_<%=cid%>.close();
 		
-		java.io.InputStream stderr_<%=cid%> = sess_<%=cid%>.getStderr();
         java.io.BufferedReader breer_<%=cid%>= new java.io.BufferedReader(new java.io.InputStreamReader(stderr_<%=cid%>));
         String line1_<%=cid%> = "";
 
@@ -110,37 +111,41 @@ if ((metadatas!=null)&&(metadatas.size()>0)) {
 		 /* Close this session */
 		
 		sess_<%=cid%>.close();
+		
+		out_content_<%=cid%> = stringStdout_<%=cid%>.toString();
+		err_content_<%=cid%> = stringStderr_<%=cid%>.toString();
+		
 		<%if(isLog4jEnabled){%>
-			log.debug("<%=cid%> - Command line standard result: " + stringStdout_<%=cid%>.toString());
-			log.debug("<%=cid%> - Command line error result: " + stringStderr_<%=cid%>.toString());
+			log.debug("<%=cid%> - Command line standard result: " + out_content_<%=cid%>);
+			log.debug("<%=cid%> - Command line error result: " + err_content_<%=cid%>);
 		<%}%>
 <%
 		//-----begin to handle output
 		if(("TO_GLOBAL_VARIABLE").equals(standardOutput)){
 %>
-			globalMap.put("<%=cid %>_STDOUT", stringStdout_<%=cid%>.toString());
+			globalMap.put("<%=cid %>_STDOUT", out_content_<%=cid%>);
 <%
 		}else if(("TO_CONSOLE_AND_GLOBAL_VARIABLE").equals(standardOutput)){
 %>
-			globalMap.put("<%=cid %>_STDOUT", stringStdout_<%=cid%>.toString());
+			globalMap.put("<%=cid %>_STDOUT", out_content_<%=cid%>);
 <%
 		}else if(("NORMAL_OUTPUT").equals(standardOutput)){
 %>
-			output_<%=cid%>.add(stringStdout_<%=cid%>.toString());
+			output_<%=cid%>.add(out_content_<%=cid%>);
 			stringStdout_<%=cid%>.delete(0,stringStdout_<%=cid%>.length());
 <%
 		}
 		if(("TO_GLOBAL_VARIABLE").equals(errorOutput)){
 %>
-			globalMap.put("<%=cid %>_STDERR", stringStderr_<%=cid%>.toString());
+			globalMap.put("<%=cid %>_STDERR", err_content_<%=cid%>);
 <%		
 		}else if(("TO_CONSOLE_AND_GLOBAL_VARIABLE").equals(errorOutput)){
 %>
-			globalMap.put("<%=cid %>_STDERR", stringStderr_<%=cid%>.toString());
+			globalMap.put("<%=cid %>_STDERR", err_content_<%=cid%>);
 <%		
 		}else if(("NORMAL_OUTPUT").equals(errorOutput)){
 %>
-			output_<%=cid%>.add(stringStderr_<%=cid%>.toString());
+			output_<%=cid%>.add(err_content_<%=cid%>);
 			stringStderr_<%=cid%>.delete(0, stringStderr_<%=cid%>.length());
 <%		
 		}

main/plugins/org.talend.designer.components.localprovider/components/tSybaseBulkExec/tSybaseBulkExec_begin.javajet

@@ -105,7 +105,7 @@ if(useExistingConnection) {
 			}
 			else {
 %>
-			String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + host_<%=cid%> + ":" + port_<%=cid%> + ";DatabaseName" + db_<%=cid%> + ";" + <%=dbproperties%>;
+			String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + host_<%=cid%> + ":" + port_<%=cid%> + ";DatabaseName=" + db_<%=cid%> + ";" + <%=dbproperties%>;
 <%
 			}
 		}

main/plugins/org.talend.designer.components.localprovider/components/tSybaseConnection/tSybaseConnection_begin.javajet

@@ -20,7 +20,7 @@ imports="
 				}
 				else{
 %>
-		String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+		String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 				}
 			}

main/plugins/org.talend.designer.components.localprovider/components/tSybaseIQBulkExec/tSybaseIQBulkExec_begin_Load.javajet

@@ -36,7 +36,7 @@ if(useExistingConn) {
                 }
                 else{
 %>
-                        String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=host %> + ":" + <%=port %> + ";DatabaseName" + <%=dbName %> + ";" + <%=dbproperties%>;
+                        String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=host %> + ":" + <%=port %> + ";DatabaseName=" + <%=dbName %> + ";" + <%=dbproperties%>;
 <%
                 }
 %>

main/plugins/org.talend.designer.components.localprovider/components/tSybaseInput/tSybaseInput_begin.javajet

@@ -29,7 +29,7 @@ imports="
 				}
 				else{
 %>
-					String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbServer %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+					String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbServer %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 				}
                         }

main/plugins/org.talend.designer.components.localprovider/components/tSybaseOutput/tSybaseOutput_begin.javajet

@@ -187,7 +187,7 @@ if(useExistingConn){
 		}
 		else{
 %>
-		    String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbServer %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+		    String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbServer %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 		}
 	}

main/plugins/org.talend.designer.components.localprovider/components/tSybaseRow/tSybaseRow_begin.javajet

@@ -26,7 +26,7 @@ imports="
 				}
 				else{
 %>
-				String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbServer %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+				String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbServer %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 				}
 			}

main/plugins/org.talend.designer.components.localprovider/components/tSybaseSCD/tSybaseSCD_begin.javajet

@@ -85,7 +85,7 @@ if(inputSchema != null && inputSchema.size() > 0 && outputSchema != null && outp
 			}
 			else{
 %>
-				String connectionString_<%=cid %> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+				String connectionString_<%=cid %> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 			}
             }

main/plugins/org.talend.designer.components.localprovider/components/tSybaseSCDELT/tSybaseSCDELT_begin.javajet

@@ -60,7 +60,7 @@ if(columnList != null && columnList.size() > 0) {
 			}
 			else{
 %>
-				String url_<%=cid %> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+				String url_<%=cid %> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 			}
             }

main/plugins/org.talend.designer.components.localprovider/components/tSybaseSP/tSybaseSP_begin.javajet

@@ -58,7 +58,7 @@ if(useExistingConn){
 		}
 		else{
 %>
-			String connectionString_<%=cid %> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
+			String connectionString_<%=cid %> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
 <%
 		}
 	}

main/plugins/org.talend.designer.components.localprovider/components/tVerticaBulkExec/tVerticaBulkExec_begin.javajet

@@ -178,7 +178,7 @@ if(("UPDATE").equals(dataAction)) {
 	    	tableName_<%=cid%> = dbschema_<%=cid%> + "." + tableName_<%=cid%>;
 		}
 
-		String sql_create_tmpdb = "<%=manager.getCreateTableSQL(stmtStructure)%>)";
+		String sql_create_tmpdb_<%=cid%> = "<%=manager.getCreateTableSQL(stmtStructure)%>)";
 
 <%
 	}

main/plugins/org.talend.designer.components.localprovider/components/tVerticaBulkExec/tVerticaBulkExec_main.javajet

@@ -66,19 +66,29 @@ java.io.FileInputStream fis_<%=cid %> = (java.io.FileInputStream)<%=connName %>.
     else {
 %>
 java.io.FileInputStream fis_<%=cid %> = null;
+<%
+    }
+    
+    boolean isUpdate = ("UPDATE".equals(dataAction) && columnList != null && columnList.size() > 0);
+    if(isUpdate) {
+%>
+    String tmpTableName_<%=cid%>= "tmp_<%=cid%>" + pid + Thread.currentThread().getId();
+    if(dbschema_<%=cid%>!= null && !dbschema_<%=cid%>.trim().isEmpty()) {
+        tmpTableName_<%=cid%> = dbschema_<%=cid%>+ "." + tmpTableName_<%=cid%>;
+    }
 <%
     }
 %>
 try{
 <%
-if(("UPDATE").equals(dataAction)) {
+if(("UPDATE").equals(dataAction)) {//TODO use isUpdate, now only follow refactor way
 %>
 java.sql.Statement stmtCreateTmp_<%=cid%> = conn_<%=cid%>.createStatement();
 <%if(isLog4jEnabled){%>
 log.info("<%=cid%> - Creating temp table <%=manager.getLProtectedChar()%>" + tableName_<%=cid%> + "<%=manager.getRProtectedChar()%>." );
 <%}%>
 
-stmtCreateTmp_<%=cid%>.execute(sql_create_tmpdb);
+stmtCreateTmp_<%=cid%>.execute(sql_create_tmpdb_<%=cid%>);
 <%if(isLog4jEnabled){%>
     log.info("<%=cid%> - Create temp table <%=manager.getLProtectedChar()%>" + tableName_<%=cid%> + "<%=manager.getRProtectedChar()%> has succeeded.");
 <%}%>
@@ -121,15 +131,12 @@ if(("INSERT").equals(dataAction) || columnList != null && columnList.size() > 0)
 
 }
 
-if(("UPDATE").equals(dataAction) && columnList != null && columnList.size() > 0) {
+if(isUpdate) {
 %>
-    String tmpTableName_<%=cid%>= "tmp_<%=cid%>" + pid + Thread.currentThread().getId();
     if(dbschema_<%=cid%>== null || dbschema_<%=cid%>.trim().length() == 0) {
         tableName_<%=cid%>=<%=table%>;
-    }
-    else {
-        tableName_<%=cid%>= dbschema_<%=cid%>+ "." +<%=table%>;
-        tmpTableName_<%=cid%>=dbschema_<%=cid%>+ "." + tmpTableName_<%=cid%>;
+    } else {
+        tableName_<%=cid%>= dbschema_<%=cid%>+ "." + <%=table%>;
     }
     java.sql.Statement stmtUpdateBulk_<%=cid%>= conn_<%=cid%>.createStatement();
 <%
@@ -147,15 +154,14 @@ if(("UPDATE").equals(dataAction) && columnList != null && columnList.size() > 0)
 %>
 
     stmtUpdateBulk_<%=cid%>.close();
-    tableName_<%=cid%>= tmpTableName_<%=cid%>;
 <%
 }
 %>
 }finally{
-
 <%
-    if(("UPDATE").equals(dataAction) && columnList != null && columnList.size() > 0) {
+    if(isUpdate) {
 %>
+        tableName_<%=cid%>= tmpTableName_<%=cid%>;
         java.sql.Statement stmtTmpDrop_<%=cid%> = conn_<%=cid%>.createStatement();
 <%
         if(isLog4jEnabled){

main/plugins/org.talend.designer.core.generic/src/main/java/org/talend/designer/core/generic/model/Component.java

@@ -1189,7 +1189,7 @@ public class Component extends AbstractBasicComponent {
 
     /*
      * (non-Javadoc)
-     * 
+     *
      * @see org.talend.core.model.components.IComponent#getModulesNeeded()
      */
     @Override
@@ -1470,10 +1470,15 @@ public class Component extends AbstractBasicComponent {
         if (GenericTypeUtils.isStringType(property)
                 && property.getTaggedValue(IGenericConstants.LINE_SEPARATOR_REPLACED_TO) != null) {
             //process for the sql field for jdbc, snowflake, salesforce, LINE_SEPARATOR_REPLACED_TO key can tell us which a sql type field,
-            //as sql type value may have newline and return characters, which make compiler issue in java code, 
+            //as sql type value may have newline and return characters, which make compiler issue in java code,
             //so have to convert the newline characters to visible "\r", "\n" for pass the compiler issue and can't only convert them to white space as TDI-41898
-            //jdbc drivers, salesforce driver can work like that sql : select * \nfrom Account, so it is ok 
-            return NodeUtil.replaceCRLFInMEMO_SQL(value);
+            //jdbc drivers, salesforce driver can work like that sql : select * \nfrom Account, so it is ok
+            String replacedString = NodeUtil.replaceCRLFInMEMO_SQL(value).trim();
+
+            // For the case when sql field ends with extra semicolon, it has to be removed to avoid compilation error.
+            return replacedString.endsWith(";")
+                    ? replacedString.substring(0, replacedString.length() -1)
+                    : replacedString;
         }
         if (GenericTypeUtils.isSchemaType(property)) {
             // Handles embedded escaped quotes which might occur
@@ -1496,8 +1501,8 @@ public class Component extends AbstractBasicComponent {
         }
         return value;
     }
-    
-    
+
+
 
     @Override
     public int hashCode() {
@@ -1510,7 +1515,7 @@ public class Component extends AbstractBasicComponent {
 
     /*
      * (non-Javadoc)
-     * 
+     *
      * @see java.lang.Object#equals(java.lang.Object)
      */
     @Override
@@ -1692,7 +1697,7 @@ public class Component extends AbstractBasicComponent {
 
     /*
      * (non-Javadoc)
-     * 
+     *
      * @see org.talend.core.model.components.IComponent#isVisible()
      */
     @Override
@@ -1724,7 +1729,7 @@ public class Component extends AbstractBasicComponent {
 
     /*
      * (non-Javadoc)
-     * 
+     *
      * @see org.talend.core.model.components.IComponent#isTechnical()
      */
     @Override

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/utils/BigDataJobUtil.java

@@ -151,7 +151,12 @@ public class BigDataJobUtil {
 
             for (IElementParameter pt : parameters) {
                 if (pt.getName().equals("DISTRIBUTION")) { //$NON-NLS-1$
-                    return true;
+                    String value = String.valueOf(pt.getValue());
+                    if ("MICROSOFT_HD_INSIGHT".equals(value) //$NON-NLS-1$
+                            || "GOOGLE_CLOUD_DATAPROC".equals(value) //$NON-NLS-1$
+                            || "CLOUDERA_ALTUS".equals(value) || "DATABRICKS".equals(value) || "QUBOLE".equals(value)) { //$NON-NLS-1$
+                        return true;
+                    }
                 }
             }
             if (isSparkWithYarnClusterMode()) {

main/plugins/org.talend.designer.core/src/main/java/org/talend/designer/core/utils/JavaProcessUtil.java

@@ -15,12 +15,16 @@ package org.talend.designer.core.utils;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.runtime.IPath;
@@ -28,6 +32,7 @@ import org.eclipse.core.runtime.Path;
 import org.talend.core.CorePlugin;
 import org.talend.core.hadoop.IHadoopClusterService;
 import org.talend.core.hadoop.repository.HadoopRepositoryUtil;
+import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.process.EParameterFieldType;
 import org.talend.core.model.process.IContext;
@@ -48,6 +53,7 @@ import org.talend.core.utils.BitwiseOptionUtils;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.core.model.components.EParameterName;
 import org.talend.designer.core.model.components.EmfComponent;
+import org.talend.designer.maven.utils.MavenVersionHelper;
 import org.talend.designer.runprocess.ItemCacheManager;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
 
@@ -180,6 +186,8 @@ public class JavaProcessUtil {
         String hadoopItemId = null;
         List<INode> nodeList = (List<INode>) process.getGeneratingNodes();
 
+        Set<ModuleNeeded> highPriorityLinkedSet = new LinkedHashSet<ModuleNeeded>();
+        Set<ModuleNeeded> jdbcCustomizeModulesSet = new HashSet<ModuleNeeded>();
         for (INode node : nodeList) {
             if (hadoopItemId == null) {
                 String itemId = getHadoopClusterItemId(node);
@@ -205,11 +213,30 @@ public class JavaProcessUtil {
                             highPriorityModuleNeeded.add(moduleNeeded);
                         }
                     }
-                    LastGenerationInfo.getInstance().setHighPriorityModuleNeeded(process.getId(), process.getVersion(),
-                            highPriorityModuleNeeded);
+                    highPriorityLinkedSet.addAll(highPriorityModuleNeeded);
+                }
+
+                // for JDBC, user set the customize driver jars need the high Priority
+                // but after tLibraryLoad
+                if (isSetHighPriorityForJDBCCustomizeDriver()) {
+                    if (node.getComponent().getComponentType() == EComponentType.GENERIC && !isTestcaseProcess) {
+                        List<ModuleNeeded> jdbcNodeModuleNeededList = new ArrayList<ModuleNeeded>();
+                        for (IElementParameter curParam : node.getElementParameters()) {
+                            if (curParam.getFieldType() == EParameterFieldType.TABLE) {
+                                getModulesInTable(process, curParam, jdbcNodeModuleNeededList);
+                            }
+                        }
+                        jdbcCustomizeModulesSet.addAll(jdbcNodeModuleNeededList);
+                    }
                 }
             }
         }
+        // in case of multiple Version jars in customize modules, descendOrder
+        if (isSetHighPriorityForJDBCCustomizeDriver()) {
+            highPriorityLinkedSet.addAll(descendingOrderModuleList(jdbcCustomizeModulesSet));
+        }
+        LastGenerationInfo.getInstance().setHighPriorityModuleNeeded(process.getId(), process.getVersion(),
+                highPriorityLinkedSet);
 
         if (hadoopItemId == null) { // Incase it is a bigdata process.
             IElementParameter propertyParam = process.getElementParameter("MR_PROPERTY"); //$NON-NLS-1$
@@ -228,6 +255,61 @@ public class JavaProcessUtil {
         new BigDataJobUtil(process).setExcludedModules(modulesNeeded);
     }
 
+    private static boolean isSetHighPriorityForJDBCCustomizeDriver() {
+        return Boolean.getBoolean("talend.JDBC.setHighPriorityForDriverJar"); //$NON-NLS-1$
+    }
+
+    public static List<ModuleNeeded> descendingOrderModuleList(Set<ModuleNeeded> moduleList) {
+        List<ModuleNeeded> orderedList = new ArrayList<ModuleNeeded>();
+        Map<String, List<ModuleNeeded>> multipleVersionHM = new HashMap<String, List<ModuleNeeded>>();
+        Set<ModuleNeeded> tmpModuleList = new HashSet<ModuleNeeded>(moduleList);
+        Iterator<ModuleNeeded> it = tmpModuleList.iterator();
+        Pattern pattern = Pattern.compile("(-\\d+\\.\\d.*)+?(.jar)"); //$NON-NLS-1$
+        Pattern versionPattern = Pattern.compile("(?<=-)\\d+\\.\\d.*(?=.jar)"); //$NON-NLS-1$
+        while (it.hasNext()) {
+            ModuleNeeded module = it.next();
+            String moduleName = module.getModuleName();
+            Matcher matcher = pattern.matcher(moduleName);
+            if (matcher.find()) {
+                String matchStr = matcher.group();
+                String key = moduleName.substring(0, moduleName.indexOf(matchStr));
+                if (multipleVersionHM.get(key) == null) {
+                    multipleVersionHM.put(key, new ArrayList<ModuleNeeded>());
+                }
+                multipleVersionHM.get(key).add(module);
+                it.remove();
+            }
+        }
+
+        for (String key : multipleVersionHM.keySet()) {
+            multipleVersionHM.get(key).sort(new Comparator<ModuleNeeded>() {
+
+                @Override
+                public int compare(ModuleNeeded o1, ModuleNeeded o2) {
+                    String o1Version = "";
+                    String o2Version = "";
+                    String o1Name = o1.getModuleName();
+                    String o2Name = o2.getModuleName();
+                    Matcher o1Matcher = versionPattern.matcher(o1Name);
+                    if (o1Matcher.find()) {
+                        o1Version = o1Matcher.group();
+                    }
+                    Matcher o2Matcher = versionPattern.matcher(o2Name);
+                    if (o2Matcher.find()) {
+                        o2Version = o2Matcher.group();
+                    }
+                    return MavenVersionHelper.compareTo(o2Version, o1Version);
+                }
+
+            });
+
+            orderedList.addAll(multipleVersionHM.get(key));
+        }
+        orderedList.addAll(tmpModuleList);
+
+        return orderedList;
+    }
+
     public static String getHadoopClusterItemId(INode node) {
         IHadoopClusterService hadoopClusterService = HadoopRepositoryUtil.getHadoopClusterService();
         if (hadoopClusterService == null) {

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/java/JavaProcessor.java

@@ -150,6 +150,7 @@ import org.talend.designer.runprocess.i18n.Messages;
 import org.talend.designer.runprocess.prefs.RunProcessPrefsConstants;
 import org.talend.designer.runprocess.utils.JobVMArgumentsUtil;
 import org.talend.repository.ProjectManager;
+import org.talend.repository.utils.EmfModelUtils;
 import org.talend.repository.utils.EsbConfigUtils;
 import org.talend.utils.io.FilesUtils;
 
@@ -353,16 +354,51 @@ public class JavaProcessor extends AbstractJavaProcessor implements IJavaBreakpo
             outputFolder = tProcessJavaProject.getTestOutputFolder();
         } else {
             srcFolder = tProcessJavaProject.getSrcFolder();
-            boolean needsToHaveContextInsideJar = true;
+            boolean needContextInJar = false;
 
-            if (property != null && property.getItem() instanceof ProcessItem) {
-                needsToHaveContextInsideJar = !new BigDataJobUtil(process).needsToHaveContextInsideJar();
+            if (process != null) {
+                needContextInJar = new BigDataJobUtil(process).needsToHaveContextInsideJar();
+                if (ProcessorUtilities.getMainJobInfo() != null) {
+                    if (ProcessorUtilities.getMainJobInfo().getProcess() != null) {
+                        if (ProcessorUtilities.isEsbJob(ProcessorUtilities.getMainJobInfo().getProcess(), true)
+                                || "CAMEL".equals(ProcessorUtilities.getMainJobInfo().getProcess().getComponentsType())) {
+                            if (property.getItem() instanceof ProcessItem) {
+                                if (!needContextInJar) {
+                                    if (null != EmfModelUtils.getComponentByName((ProcessItem) property.getItem(), "tRunJob",
+                                            "cTalendJob")) {
+                                        needContextInJar = false;
+                                    } else {
+                                        if (ProcessorUtilities.isEsbJob(process, true)) {
+                                            needContextInJar = false;
+                                        } else {
+                                            needContextInJar = true;
+                                        }
+                                    }
+                                }
+                            } else if (property.getItem().eClass().getClassifierID() == 4) {
+                                // CamelPropertiesPackage Line 516 int ROUTELET_PROCESS_ITEM = 4;
+                                needContextInJar = true;
+                            }
+                        }
+                    }
+                }
             }
 
-            if (ProcessorUtilities.isExportConfig() && property != null && needsToHaveContextInsideJar) {
-                resourcesFolder = tProcessJavaProject.getExternalResourcesFolder();
-            } else {
+            if (needContextInJar) {
                 resourcesFolder = tProcessJavaProject.getResourcesFolder();
+            } else {
+                resourcesFolder = tProcessJavaProject.getExternalResourcesFolder();
+            }
+            if (ProcessorUtilities.isExportConfig() && !needContextInJar) {
+                try {
+                    for (IResource resource : tProcessJavaProject.getResourcesFolder().members()) {
+                        if (resource.exists()) {
+                            resource.delete(true, monitor);
+                        }
+                    }
+                } catch (CoreException e) {
+                    ExceptionHandler.process(e);
+                }
             }
             outputFolder = tProcessJavaProject.getOutputFolder();
         }

main/plugins/org.talend.designer.runprocess/src/main/java/org/talend/designer/runprocess/maven/listener/ProcessChangeListener.java

@@ -29,6 +29,8 @@ import org.eclipse.core.runtime.Path;
 import org.eclipse.ltk.core.refactoring.resource.MoveResourceChange;
 import org.eclipse.ltk.core.refactoring.resource.RenameResourceChange;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.exception.PersistenceException;
+import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.Property;
@@ -143,7 +145,7 @@ public class ProcessChangeListener implements PropertyChangeListener {
                 IFolder targetFolder = processTypeFolder.getFolder(targetPath);
                 try {
                     if (!targetFolder.exists()) {
-                        targetFolder.create(true, true, null);
+                        ResourceUtils.createFolder(targetFolder);
                     }
                     List<IRepositoryViewObject> allVersions = ProxyRepositoryFactory.getInstance().getAllVersion(obj.getId());
                     for (IRepositoryViewObject objs : allVersions) {
@@ -271,13 +273,20 @@ public class ProcessChangeListener implements PropertyChangeListener {
             if (getAllProcessTypes().contains(processType)) {
                 IFolder processTypeFolder = getAggregatorPomsHelper().getProcessFolder(processType);
                 IFolder sourceFolder = processTypeFolder.getFolder(sourcePath);
+                if (!sourceFolder.exists()) {
+                    return;
+                }
                 try {
+                    sourceFolder.refreshLocal(IResource.DEPTH_INFINITE, new NullProgressMonitor());
                     removeFromParentSourceFolder(sourceFolder);
                     IFolder targetFolder = processTypeFolder.getFolder(targetPath);
+                    if (!targetFolder.exists()) {
+                        ResourceUtils.createFolder(targetFolder);
+                    }
                     MoveResourceChange change = new MoveResourceChange(sourceFolder, targetFolder);
                     change.perform(new NullProgressMonitor());
                     updatePomsInNewFolder(targetFolder.getFolder(sourceFolder.getName()));
-                } catch (OperationCanceledException | CoreException e) {
+                } catch (OperationCanceledException | CoreException | PersistenceException e) {
                     ExceptionHandler.process(e);
                 }
             }

main/plugins/org.talend.repository/plugin.xml

@@ -3085,6 +3085,15 @@
               name="RedshiftLogFileMigrationTask"
               version="7.0.2">
         </projecttask>
+        <projecttask
+              beforeLogon="false"
+              breaks="7.0.1"
+              class="org.talend.repository.model.migration.SAPBAPIParameterTypeMigrationTask"
+              description="adjust the tsapbapi ui"
+              id="org.talend.repository.model.migration.SAPBAPIParameterTypeMigrationTask"
+              name="SAPBAPIParameterTypeMigrationTask"
+              version="7.0.2">
+        </projecttask>
    </extension>
 
    <extension

main/plugins/org.talend.repository/src/main/java/org/talend/repository/model/migration/SAPBAPIParameterTypeMigrationTask.java

@@ -0,0 +1,109 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.repository.model.migration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.List;
+
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.model.components.ComponentUtilities;
+import org.talend.core.model.components.ModifyComponentsAction;
+import org.talend.core.model.components.conversions.IComponentConversion;
+import org.talend.core.model.components.filters.IComponentFilter;
+import org.talend.core.model.components.filters.NameComponentFilter;
+import org.talend.core.model.migration.AbstractJobMigrationTask;
+import org.talend.core.model.properties.Item;
+import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
+import org.talend.designer.core.model.utils.emf.talendfile.ElementValueType;
+import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
+import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
+import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
+
+public class SAPBAPIParameterTypeMigrationTask extends AbstractJobMigrationTask {
+
+	@Override
+	public ExecutionResult execute(Item item) {
+		ProcessType processType = getProcessType(item);
+		if (processType == null) {
+			return ExecutionResult.NOTHING_TO_DO;
+		}
+		String[] componentsName = new String[] { "tSAPBapi" };
+
+		try {
+
+			for (String element : componentsName) {
+				IComponentFilter filter = new NameComponentFilter(element);
+				ModifyComponentsAction.searchAndModify(item, processType, filter,
+						Arrays.<IComponentConversion>asList(new IComponentConversion() {
+
+							@SuppressWarnings("unchecked")
+							@Override
+							public void transform(NodeType node) {
+								ElementParameterType mapping_input = ComponentUtilities.getNodeProperty(node,
+										"MAPPING_INPUT");
+								TalendFileFactory fileFact = TalendFileFactory.eINSTANCE;
+								if (mapping_input != null) {
+									List<ElementValueType> newElementValues = new ArrayList<ElementValueType>();
+									List<ElementValueType> elementValues = mapping_input.getElementValue();
+									boolean tableType = false;
+									for (ElementValueType elementValue : elementValues) {
+										if ("TYPE".equals(elementValue.getElementRef())
+												&& "TABLE".equals(elementValue.getValue())) {
+											tableType = true;
+										}
+
+										if ("CHANGING".equals(elementValue.getElementRef())) {
+											ElementValueType value = fileFact.createElementValueType();
+											value.setElementRef("PARAMETER_TYPE");
+											boolean is_changing_parameter = "true".equalsIgnoreCase(elementValue.getValue());
+											if (tableType) {
+												value.setValue(is_changing_parameter ? "CHANGING" : "TABLES");
+											} else {
+												value.setValue(is_changing_parameter ? "CHANGING" : "IMPORT");
+											}
+
+											newElementValues.add(value);
+
+											tableType = false;
+
+											continue;
+										}
+
+										newElementValues.add(elementValue);
+									}
+
+									elementValues.clear();
+									elementValues.addAll(newElementValues);
+								}
+							}
+
+						}));
+			}
+
+			return ExecutionResult.SUCCESS_NO_ALERT;
+		} catch (Exception e) {
+			ExceptionHandler.process(e);
+			return ExecutionResult.FAILURE;
+		}
+	}
+
+	@Override
+	public Date getOrder() {
+		GregorianCalendar gc = new GregorianCalendar(2019, 9, 23, 18, 0, 0);
+		return gc.getTime();
+	}
+
+}

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/ConfigExternalLib/LibraryField.java

@@ -84,7 +84,8 @@ public class LibraryField extends TableField {
                     final TableItem item = contextTable.getSelection()[0];
                     if (item.getBounds(1).contains(event.x, event.y)) {
                         IMPORTType it = (IMPORTType) getList().get(contextTable.getSelectionIndex());
-                        if ("BeanItem".equals(it.eContainer().eClass().getName())) {
+                        if ("BeanItem".equals(it.eContainer().eClass().getName())
+                                || "RoutineItem".equals(it.eContainer().eClass().getName())) {
                             it.setREQUIRED(!it.isREQUIRED());
                             setInput(getList());
                         }

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/action/JobExportAction.java

@@ -43,9 +43,11 @@ import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.repository.job.JobResource;
 import org.talend.core.model.repository.job.JobResourceManager;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
+import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.ui.CoreUIPlugin;
 import org.talend.core.ui.export.ArchiveFileExportOperationFullPath;
 import org.talend.core.ui.services.IDesignerCoreUIService;
+import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.designer.runprocess.ItemCacheManager;
 import org.talend.designer.runprocess.ProcessorException;
 import org.talend.designer.runprocess.ProcessorUtilities;
@@ -60,12 +62,14 @@ import org.talend.repository.ui.wizards.exportjob.scriptsmanager.JobScriptsManag
 import org.talend.repository.ui.wizards.exportjob.util.ExportJobUtil;
 
 /**
- * 
+ *
  * class global comment. Detailled comment
  */
 public class JobExportAction implements IRunnableWithProgress {
 
-    private List<? extends IRepositoryNode> nodes;
+    private static final String LIB = "lib";
+
+	private List<? extends IRepositoryNode> nodes;
 
     private String jobVersion;
 
@@ -204,6 +208,7 @@ public class JobExportAction implements IRunnableWithProgress {
             manager.setTopFolder(resourcesToExport);
         }
 
+        cleanLibFolderBeforeExport();
         doArchiveExport(monitor, resourcesToExport);
 
         clean();
@@ -228,7 +233,7 @@ public class JobExportAction implements IRunnableWithProgress {
 
     /**
      * DOC ggu Comment method "doArchiveExport".
-     * 
+     *
      * @param monitor
      * @param resourcesToExport
      */
@@ -244,7 +249,7 @@ public class JobExportAction implements IRunnableWithProgress {
 
     /**
      * DOC ggu Comment method "generatedCodes".
-     * 
+     *
      * @param version
      * @param monitor
      * @param processes
@@ -283,7 +288,7 @@ public class JobExportAction implements IRunnableWithProgress {
 
     /**
      * DOC zli Comment method "getTempDestinationValue".
-     * 
+     *
      * @return
      */
     protected String getTempDestinationValue() {
@@ -315,9 +320,9 @@ public class JobExportAction implements IRunnableWithProgress {
     }
 
     /**
-     * 
+     *
      * DOC aiming Comment method "reBuildJobZipFile".
-     * 
+     *
      * @param processes
      */
     protected void reBuildJobZipFile(List<ExportFileResource> processes) {
@@ -365,7 +370,7 @@ public class JobExportAction implements IRunnableWithProgress {
     /**
      * Added by Marvin Wang on Feb.1, 2012 for estimating if the file can be created. In win7 or other systems, have not
      * the permission to create a file directly under system disk(like C:\).
-     * 
+     *
      * @param disZipFileStr
      * @return
      */
@@ -419,4 +424,22 @@ public class JobExportAction implements IRunnableWithProgress {
         return nodes.size() > 1;
     }
 
+    public void cleanLibFolderBeforeExport() {
+    	IRunProcessService runProcessService = CorePlugin.getDefault().getRunProcessService();
+        ITalendProcessJavaProject talendProcessJavaProject = runProcessService
+                .getTalendJobJavaProject(nodes.get(0).getObject().getProperty());
+
+        if (talendProcessJavaProject != null) {
+	        File libFolder = new File(talendProcessJavaProject.getBundleResourcesFolder().getLocation().toOSString()
+	                + File.separator
+	                + LIB);
+	
+	        if (libFolder.exists()) {
+		        for(File file: libFolder.listFiles())
+		            if (!file.isDirectory() && file.getName().endsWith(".jar"))
+		                file.delete();
+	        }
+        }
+    }
+
 }

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/handler/AbstractBuildJobHandler.java

@@ -12,11 +12,7 @@
 // ============================================================================
 package org.talend.repository.ui.wizards.exportjob.handler;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang.BooleanUtils;
@@ -30,11 +26,9 @@ import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.CorePlugin;
-import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Project;
-import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.runtime.process.IBuildJobHandler;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.LastGenerationInfo;
@@ -44,6 +38,7 @@ import org.talend.core.runtime.repository.build.IBuildResourceParametes;
 import org.talend.core.runtime.repository.build.IBuildResourcesProvider;
 import org.talend.core.runtime.util.ParametersUtil;
 import org.talend.designer.maven.model.TalendMavenConstants;
+import org.talend.designer.runprocess.IProcessor;
 import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.designer.runprocess.ProcessorUtilities;
 import org.talend.repository.ProjectManager;
@@ -309,19 +304,9 @@ public abstract class AbstractBuildJobHandler implements IBuildJobHandler, IBuil
         parameters.put(VERSION, version);
         parameters.put(OBJ_PROCESS_JAVA_PROJECT, talendProcessJavaProject);
 
-        //
-        List<Item> dependenciesItems = new ArrayList<Item>();
-        Collection<IRepositoryViewObject> allProcessDependencies = ProcessUtils.getAllProcessDependencies(Arrays
-                .asList(processItem));
-        if (!allProcessDependencies.isEmpty()) {
-            for (IRepositoryViewObject repositoryObject : allProcessDependencies) {
-                dependenciesItems.add(repositoryObject.getProperty().getItem());
-            }
-            parameters.put(OBJ_ITEM_DEPENDENCIES, dependenciesItems);
-        }
-
         // generate sources
-        generateJobFiles(monitor);
+        IProcessor processor = generateJobFiles(monitor);
+        parameters.put(OBJ_ITEM_DEPENDENCIES, processor.getBuildChildrenJobs());
 
         // export items
         if (ParametersUtil.hasBoolFlag(parameters, OPTION_ITEMS)) {

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/scriptsmanager/esb/JobJavaScriptOSGIForESBManager.java

@@ -55,6 +55,7 @@ import org.talend.core.PluginChecker;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.process.IProcess;
+import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
@@ -756,9 +757,7 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
             ITalendProcessJavaProject talendProcessJavaProject = service.getTalendJobJavaProject(processItem.getProperty());
             if (talendProcessJavaProject != null) {
                 String optional = ";resolution:=optional";
-                String src = JavaResourcesHelper.getJobClassFilePath(processItem, true);
-                IFile srcFile = talendProcessJavaProject.getSrcFolder().getFile(src);
-                Set<String> imports = importCompiler(srcFile.getLocation().toString());
+                Set<String> imports = importCompiler(service, processItem);
                 String[] defaultPackages = analyzer.getProperty(Analyzer.IMPORT_PACKAGE).split(",");
                 for (String dp : defaultPackages) {
                     if (!imports.contains(dp) && !imports.contains(dp + optional)) {
@@ -856,7 +855,9 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
             Set<URL> resources = libResource.getResourcesByRelativePath(path);
             for (URL url : resources) {
                 // TESB-21804:Fail to deploy cMessagingEndpoint with quartz component in runtime for ClassCastException
-                if (url.getPath().matches("(.*)camel-(.*)-alldep-(.*)$")) {
+                if (url.getPath().matches("(.*)camel-(.*)-alldep-(.*)$") 
+                        || url.getPath().matches("(.*)activemq-all-[\\d\\.]*.jar$")
+                        || url.getPath().matches("(.*)jms[\\d\\.-]*.jar$")) {
                     continue;
                 }
                 File dependencyFile = new File(FilesUtils.getFileRealPath(url.getPath()));
@@ -1158,6 +1159,26 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
         }
     }
     
+    private Set<String> importCompiler(IRunProcessService service, ProcessItem processItem) {
+        Set<String> imports = new HashSet<String>();
+        if (processItem != null && service != null && processItem.getProperty() != null) {
+            ITalendProcessJavaProject talendProcessJavaProject = service.getTalendJobJavaProject(processItem.getProperty());
+            if (talendProcessJavaProject != null) {
+                String src = JavaResourcesHelper.getJobClassFilePath(processItem, true);
+                IFile srcFile = talendProcessJavaProject.getSrcFolder().getFile(src);
+                imports.addAll(importCompiler(srcFile.getLocation().toString()));
+
+                // include imports from child jobs
+                if (ERepositoryObjectType.getType(processItem.getProperty()).equals(ERepositoryObjectType.PROCESS)) {
+                    for (JobInfo subjobInfo : ProcessorUtilities.getChildrenJobInfo(processItem)) {
+                        imports.addAll(importCompiler(service, subjobInfo.getProcessItem()));
+                    }
+                }
+            }
+        }
+        return imports;
+    }
+
     private Set<String> importCompiler(String src) {
         Set<String> imports = new HashSet<String>();
         ByteArrayOutputStream out = new ByteArrayOutputStream();

test/plugins/org.talend.designer.core.test/src/org/talend/designer/core/utils/JavaProcessUtilTest.java

@@ -23,8 +23,10 @@ import static org.mockito.Mockito.when;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.junit.Test;
 import org.talend.core.model.components.ComponentCategory;
@@ -151,4 +153,35 @@ public class JavaProcessUtilTest {
         return param;
     }
 
+    @Test
+    public void testDescendingOrderModuleList() {
+        String[] moduleNames = new String[] { "slf4j-api-1.5.11.jar", "slf4j-api-1.7.5.jar", "slf4j-api-1.8.0-beta1.jar",
+                "slf4j-api-1.7.25.jar" };
+        Set<ModuleNeeded> moduleNeededSet = prepareModuleNeededSet(moduleNames);
+        List<ModuleNeeded> descendingOrderModuleList = JavaProcessUtil.descendingOrderModuleList(moduleNeededSet);
+        assertEquals(4, descendingOrderModuleList.size());
+        assertEquals(moduleNames[2], descendingOrderModuleList.get(0).getModuleName());
+        assertEquals(moduleNames[3], descendingOrderModuleList.get(1).getModuleName());
+        assertEquals(moduleNames[1], descendingOrderModuleList.get(2).getModuleName());
+        assertEquals(moduleNames[0], descendingOrderModuleList.get(3).getModuleName());
+
+        moduleNames = new String[] { "auto-common-0.8.jar", "auto-common-0.3.jar", "auto-common-0.10.jar" };
+        moduleNeededSet = prepareModuleNeededSet(moduleNames);
+        descendingOrderModuleList = JavaProcessUtil.descendingOrderModuleList(moduleNeededSet);
+        assertEquals(3, descendingOrderModuleList.size());
+        assertEquals(moduleNames[2], descendingOrderModuleList.get(0).getModuleName());
+        assertEquals(moduleNames[0], descendingOrderModuleList.get(1).getModuleName());
+        assertEquals(moduleNames[1], descendingOrderModuleList.get(2).getModuleName());
+    }
+
+    private Set<ModuleNeeded> prepareModuleNeededSet(String[] moduleNames) {
+        String fakeNodeName = "tJDBCConnection";
+        Set<ModuleNeeded> moduleNeededSet = new HashSet<ModuleNeeded>();
+        for (String moduleName : moduleNames) {
+            ModuleNeeded module = new ModuleNeeded(fakeNodeName, moduleName, null, false);
+            moduleNeededSet.add(module);
+        }
+        return moduleNeededSet;
+    }
+
 }