fix(TDI-48782):upgrade hsqldb.jar to solve cve. (#8188)

* fix(TDI-48782):upgrade hsqldb.jar to solve cve.

* fix(TDI-48782): add system property.

* fix(TDI-48782):use hsqldb jdk8 version for 7.3.1

i18n/plugins/org.talend.designer.core.nl/src/main/java/messages_en.properties

@@ -584,13 +584,11 @@ EParameterName.jdbcURL=JDBC URL
 EParameterName.driverJar=Driver jar
 EParameterName.className=Class name
 EParameterName.mappingFile=Mapping file
-SetupProcessDependenciesRoutinesAction.title=Setup routine dependencies
+SetupProcessDependenciesRoutinesAction.title=Setup Codes Dependencies
 SetupProcessDependenciesRoutinesDialog.systemRoutineLabel=System routines
-SetupProcessDependenciesRoutinesDialog.userRoutineLabel=User routines
 PerformancePreferencePage.addAllSystemRoutines=Add all system routines to job dependencies, when creating a new job
 PerformancePreferencePage.addAllUserRoutines=Add all user routines to job dependencies, when creating a new job
 ShowRoutineItemsDialog.systemTitle=Select Sytem Routines
-ShowRoutineItemsDialog.title=Select Routines
 AbstractMultiPageTalendEditor_pleaseWait=Saving Please Wait....
 DocumentationPreferencePage.use_css_template=Use CSS file as a template when export to HTML
 DocumentationPreferencePage.css_file=CSS File

i18n/plugins/org.talend.repository.view.di.nl/plugin_en.properties

@@ -1,5 +1,5 @@
 NavigatorContent.contexts=Contexts
-NavigatorContent.routines=Routines
+NavigatorContent.routines=Global Routines
 NavigatorContent.sqlTemplates=SQL Templates
 NavigatorContent.documentation=Documentation
 NavigatorContent.activation=di.fake.for.activation

main/features/org.talend.rcp.branding.tos.test.feature/feature.xml

@@ -6,7 +6,7 @@
   <license url="http://www.example.com/license">[Enter License Description here.]</license>  
   <requires>
     <import feature="org.eclipse.test" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.junit" version="0.0.0" match="greaterOrEqual"/>
+    <import plugin="org.junit" version="4.13.2" match="greaterOrEqual"/>
     <import plugin="org.talend.commons.runtime" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.talend.commons.ui" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.talend.core" version="0.0.0" match="greaterOrEqual"/>
@@ -50,6 +50,7 @@
   <plugin id="org.talend.repository.localprovider.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.repository.metadata.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.repository.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
+  <plugin id="org.talend.presentation.onboarding.resource.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.repository.view.di.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.repository.view.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.testutils" download-size="0" install-size="0" version="0.0.0" unpack="false"/>

main/features/org.talend.tos.libraries.feature/feature.xml

@@ -16,7 +16,6 @@
   </requires>  
   <plugin id="org.talend.libraries.apache" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.apache.axis2" download-size="0" install-size="0" version="0.0.0"/>
-  <plugin id="org.talend.libraries.apache.batik" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.apache.chemistry" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.apache.common" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.apache.cxf" download-size="0" install-size="0" version="0.0.0"/>
@@ -51,5 +50,4 @@
   <plugin id="org.talend.libraries.slf4j" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.xml" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.zmq" download-size="0" install-size="0" version="0.0.0"/>
-  <plugin id="org.talend.libraries.zookeeper" download-size="0" install-size="0" version="0.0.0"/>
 </feature>

main/plugins/org.talend.designer.codegen/jet_stub/component_part_header.javajet

@@ -412,7 +412,7 @@
                     
                     if(ignoredParamsNames.contains(name)) {
                         //do nothing
-                    } else if(org.talend.core.model.process.EParameterFieldType.PASSWORD.equals(ep.getFieldType())){
+                    } else if(org.talend.core.model.process.EParameterFieldType.PASSWORD.equals(ep.getFieldType()) || org.talend.core.model.process.EParameterFieldType.HIDDEN_TEXT.equals(ep.getFieldType())){
                         //not log password
                     }else{
                         String value = org.talend.core.model.utils.NodeUtil.getRuntimeParameterValue(node, ep);

main/plugins/org.talend.designer.codegen/jet_stub/footer.javajet

@@ -126,8 +126,16 @@
 
     boolean exist_tSCP = false;
     List<INode> scpComponentsList = (List<INode>)process.getNodesOfType("tSCPConnection");
-    if (scpComponentsList.size() > 0) {
+	String parameterNames = "";
+	int scpsize = scpComponentsList.size();
+    if (scpsize > 0) {
         exist_tSCP = true;
+        for (int i = 0; i < scpsize; i++) {
+            parameterNames += "\"conn_" + scpComponentsList.get(i).getUniqueName() + "\"";
+			if(i < scpsize-1){
+			    parameterNames += ",";
+			}
+        }
     }
 
     boolean exist_tCassandra = false;
@@ -506,17 +514,35 @@
 				}
 				org.apache.logging.log4j.core.config.Configurator.setLevel(org.apache.logging.log4j.LogManager.getRootLogger().getName(), log.getLevel());
 				<%}%>
-    	    }
-        	log.info("TalendJob: '<%=codeGenArgument.getJobName()%>' - Start.");
-    	<%}%>
-    	
-    	<%
-    	INode jobCatcherNode = null;
+			}
+			log.info("TalendJob: '<%=codeGenArgument.getJobName()%>' - Start.");
+		<%}%>
+		
+		<%
+		INode jobCatcherNode = null;
+		
+		int threadPoolSize = 0;
+		boolean tRESTRequestLoopExists = false;
 		for (INode nodeInProcess : process.getGeneratingNodes()) {
 			String componentName = nodeInProcess.getComponent().getName();
-			if("tJobStructureCatcher".equals(componentName)) {
+
+			if(jobCatcherNode==null && "tJobStructureCatcher".equals(componentName)) {
 				jobCatcherNode = nodeInProcess;
-				break;
+				continue;
+			}
+
+			if(!nodeInProcess.isActivate()) continue;
+
+			if("tRESTRequestLoop".equals(componentName)) {
+				tRESTRequestLoopExists = true;
+				continue;
+			}
+			
+			if("tWriteXMLFieldOut".equals(componentName)) {
+				IConnection nextMergeConn = NodeUtil.getNextMergeConnection(nodeInProcess);
+				if(nextMergeConn == null || nextMergeConn.getInputId()==1){
+					threadPoolSize++;
+				}
 			}
 		}
 		
@@ -644,15 +670,15 @@
 
         boolean inOSGi = routines.system.BundleUtils.inOSGi();
 
-        if (inOSGi) {
-            java.util.Dictionary<String, Object> jobProperties = routines.system.BundleUtils.getJobProperties(jobName);
-
-            if (jobProperties != null) {
-                contextStr = (String)jobProperties.get("context");
-            }
-        }
-
         try {
+            java.util.Dictionary<String, Object> jobProperties = null;
+            if (inOSGi) {
+                jobProperties = routines.system.BundleUtils.getJobProperties(jobName);
+    
+                if (jobProperties != null && jobProperties.get("context") != null) {
+                    contextStr = (String)jobProperties.get("context");
+                }
+            }
             //call job/subjob with an existing context, like: --context=production. if without this parameter, there will use the default context instead.
             java.io.InputStream inContext = <%=className%>.class.getClassLoader().getResourceAsStream("<%=jobClassPackageFolder%>/contexts/" + contextStr + ".properties");
             if (inContext == null) {
@@ -662,8 +688,17 @@
                 try {
                     //defaultProps is in order to keep the original context value
                     if(context != null && context.isEmpty()) {
-	                defaultProps.load(inContext);
-	                context = new ContextProperties(defaultProps);
+    	                defaultProps.load(inContext);
+    	                if (inOSGi && jobProperties != null) {
+                             java.util.Enumeration<String> keys = jobProperties.keys();
+                             while (keys.hasMoreElements()) {
+                                 String propKey = keys.nextElement();
+                                 if (defaultProps.containsKey(propKey)) {
+                                     defaultProps.put(propKey, (String) jobProperties.get(propKey));
+                                 }
+                             }
+    	                }
+    	                context = new ContextProperties(defaultProps);
                     }
                 } finally {
                     inContext.close();
@@ -733,34 +768,39 @@
 <%
                         } else if(typeToGenerate.equals("java.util.Date")) {
 %>
-                        try{
                             String context_<%=ctxParam.getName()%>_value = context.getProperty("<%=ctxParam.getName()%>");
-                            if (context_<%=ctxParam.getName()%>_value == null){
-                                context_<%=ctxParam.getName()%>_value = "";
-                            }
-                            int context_<%=ctxParam.getName()%>_pos = context_<%=ctxParam.getName()%>_value.indexOf(";");
-                            String context_<%=ctxParam.getName()%>_pattern =  "yyyy-MM-dd HH:mm:ss";
-                            if(context_<%=ctxParam.getName()%>_pos > -1){
-                                context_<%=ctxParam.getName()%>_pattern = context_<%=ctxParam.getName()%>_value.substring(0, context_<%=ctxParam.getName()%>_pos);
-                                context_<%=ctxParam.getName()%>_value = context_<%=ctxParam.getName()%>_value.substring(context_<%=ctxParam.getName()%>_pos + 1);
-                            }
+                            try{
+                                if (context_<%=ctxParam.getName()%>_value == null){
+                                    context_<%=ctxParam.getName()%>_value = "";
+                                }
+                                int context_<%=ctxParam.getName()%>_pos = context_<%=ctxParam.getName()%>_value.indexOf(";");
+                                String context_<%=ctxParam.getName()%>_pattern =  "yyyy-MM-dd HH:mm:ss";
+                                if(context_<%=ctxParam.getName()%>_pos > -1){
+                                    context_<%=ctxParam.getName()%>_pattern = context_<%=ctxParam.getName()%>_value.substring(0, context_<%=ctxParam.getName()%>_pos);
+                                    context_<%=ctxParam.getName()%>_value = context_<%=ctxParam.getName()%>_value.substring(context_<%=ctxParam.getName()%>_pos + 1);
+                                }
 
-                            context.<%=ctxParam.getName()%>=(java.util.Date)(new java.text.SimpleDateFormat(context_<%=ctxParam.getName()%>_pattern).parse(context_<%=ctxParam.getName()%>_value));
+                                context.<%=ctxParam.getName()%>=(java.util.Date)(new java.text.SimpleDateFormat(context_<%=ctxParam.getName()%>_pattern).parse(context_<%=ctxParam.getName()%>_value));
 
-                        } catch(ParseException e) {
+                            } catch(ParseException e) {
+                                try { <% /*try to check if date passed as long also*/ %>
+                                    long context_<%=ctxParam.getName()%>_longValue = Long.valueOf(context_<%=ctxParam.getName()%>_value);
+                                    context.<%=ctxParam.getName()%> = new java.util.Date(context_<%=ctxParam.getName()%>_longValue);
+                                } catch (NumberFormatException cantParseToLongException) {
 <%
-                            if (isLog4jEnabled) {
+                                    if (isLog4jEnabled) {
 %>
-                                log.warn(String.format("<%=warningMessageFormat %>", "<%=ctxParam.getName() %>", e.getMessage()));
+                                        log.warn(String.format("<%=warningMessageFormat %>", "<%=ctxParam.getName() %>", "Can't parse date string: " + e.getMessage() + " and long: " + cantParseToLongException.getMessage()));
 <%
-                            } else {
+                                    } else {
 %>
-                                System.err.println(String.format("<%=warningMessageFormat %>", "<%=ctxParam.getName() %>", e.getMessage()));
+                                        System.err.println(String.format("<%=warningMessageFormat %>", "<%=ctxParam.getName() %>", "Can't parse date string: " + e.getMessage() + " and long: " + cantParseToLongException.getMessage()));
 <%
+                                    }
+%>
+                                    context.<%=ctxParam.getName()%>=null;
+                                }
                             }
-%>
-                            context.<%=ctxParam.getName()%>=null;
-                        }
 <%
                         } else if(typeToGenerate.equals("Object")||typeToGenerate.equals("String")||typeToGenerate.equals("java.lang.String")) {
 %>
@@ -844,7 +884,7 @@
         }
 %>
         //Resume: jobStart
-        resumeUtil.addLog("JOB_STARTED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "","","","",resumeUtil.convertToJsonText(context,parametersToEncrypt));
+        resumeUtil.addLog("JOB_STARTED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "","","","",resumeUtil.convertToJsonText(context,ContextProperties.class,parametersToEncrypt));
 
 <%
         if (stats) {
@@ -1117,9 +1157,15 @@ this.globalResumeTicket = false;//to run others jobs
             if (!componentName.equals("tJobStructureCatcher") && !componentName.equals("tLogCatcher") && !componentName.equals("tFlowMeterCatcher") && !componentName.equals("tAssertCatcher") && !componentName.equals("tStatCatcher") && !componentName.equals("tAsyncIn")) {
 %>
 <%=createCallProcess(rootNode, className, false) %>
-<%    }
-    }
-}// end if(isRunInMultiThread)
+<%
+                if (process.getNodesOfType("tStatCatcher").size() > 0) {
+%>
+<%=statsErrorHandlingAfterMainCall(rootNode, process.getNodesOfType("tStatCatcher"))%>
+<%
+                }
+            }
+        }
+    }// end if(isRunInMultiThread)
 %>
 
 this.globalResumeTicket = true;//to run tPostJob
@@ -1177,6 +1223,26 @@ this.globalResumeTicket = true;//to run tPostJob
 	            e.printStackTrace();
 	        }
         }
+<%
+    }
+
+    //tRESTRequest may appear in microservice, the code may be called before call submit(task) method, so can't shutdown it here
+    if(!tRESTRequestLoopExists && threadPoolSize>0) {
+%>
+        es.shutdown();
+        <%//shutdownNow should never be executed, only for safe%>
+        try {
+            if(!es.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) {
+                es.shutdownNow();
+                if(!es.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) {
+
+                }
+            }
+        } catch (java.lang.InterruptedException ie) {
+            es.shutdownNow();
+        } catch (java.lang.Exception e) {
+
+        }
 <%
     }
 %>
@@ -1185,9 +1251,12 @@ this.globalResumeTicket = true;//to run tPostJob
     closeJmsConnections();
 <% } %>
 
-<% if (exist_tSCP) { %>
-    closeScpConnections();
-<% } %>
+<% if (exist_tSCP) { 
+%>
+    closeCloseableConnections(<%=parameterNames%>);
+<% 
+} 
+%>
 
 <%
     if (stats) {
@@ -1230,7 +1299,7 @@ if (execStat) {
          returnCode = errorCode.intValue();
     }
     resumeUtil.addLog("JOB_ENDED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "","" + returnCode,"","","");
-
+    resumeUtil.flush();
     return returnCode;
 
   }
@@ -1241,7 +1310,7 @@ if (execStat) {
     closeJmsConnections();
 <% } %>
 <% if(exist_tSCP) { %>
-    closeScpConnections();
+    closeCloseableConnections(<%=parameterNames%>);
 <% } %>
 <% if (exist_tSQLDB) { %>
     closeSqlDbConnections();
@@ -1309,22 +1378,17 @@ if (execStat) {
 <%
     if(exist_tSCP) {
 %>
-    private void closeScpConnections() {
-        try {
-            Object obj_conn;
-<%
-            for (INode scpNode : scpComponentsList) {
-%>
-            obj_conn = globalMap.remove("conn_<%=scpNode.getUniqueName() %>");
-            if (null != obj_conn) {
-                ((ch.ethz.ssh2.Connection) obj_conn).close();
+    private void closeCloseableConnections(String... names) {
+	        java.util.Arrays.stream(names).forEach(name-> {
+            try {
+			    Object obj_conn = globalMap.remove(name);
+				if(obj_conn != null){
+				    ((java.io.Closeable)obj_conn).close();
+				}
+            } catch (IOException ioException) {
             }
-<%
-            }
-%>
-        } catch (java.lang.Exception e) {
-        }
-    }
+        });
+	}
 <%
     }
 %>
@@ -1434,6 +1498,7 @@ if (execStat) {
             if ("sftp".equals(type)) { %>
                 ((com.jcraft.jsch.ChannelSftp) obj_conn).quit();
             <%} else { %>
+                ((org.apache.commons.net.ftp.FTPClient) obj_conn).logout();
                 ((org.apache.commons.net.ftp.FTPClient) obj_conn).disconnect();
             <%}%>
             }

main/plugins/org.talend.designer.codegen/jet_stub/footer_java.skeleton

@@ -42,10 +42,10 @@ public class CLASS
       
        //List< ? extends IConnection> onSubJobErrorConns = rootNode.getOutgoingConnections(EConnectionType.ON_SUBJOB_ERROR);
        //if(onSubJobErrorConns!=null){
-       //    for(IConnection conn : onSubJobErrorConns) {               
+       //    for(IConnection conn : onSubJobErrorConns) {
        //        toReturn += createCallProcess(conn.getTarget(),  className, isMultiThread);
        //    }
-       //}            
+       //}
         if(isMultiThread){
             toReturn += "\n}catch (java.lang.Error e_" + rootNode.getUniqueName() + ") {\n";
             toReturn += "globalMap.put(\""+rootNode.getUniqueName()+ "_SUBPROCESS_STATE\", -1);\n";
@@ -55,7 +55,20 @@ public class CLASS
        toReturn += "\n}"; 
        return toReturn;
     }
-		 
+
+    public String statsErrorHandlingAfterMainCall(INode rootNode, List<? extends INode> statsNodes) {
+        String catchErrorReturn = "catch (Error error_" + rootNode.getUniqueName() + " ) {\n";
+        catchErrorReturn+="end = System.currentTimeMillis();\n";
+        for (INode statCatcherNode : statsNodes) {
+            catchErrorReturn += statCatcherNode.getUniqueName() + ".addMessage(\"failure\", (end-startTime));\n";
+            catchErrorReturn += "try {\n " + statCatcherNode.getDesignSubjobStartNode().getUniqueName() + "Process(globalMap);\n";
+            catchErrorReturn += "} catch (Exception e_" + statCatcherNode.getUniqueName() + ") {\n";
+            catchErrorReturn += "e_" + statCatcherNode.getUniqueName() + ".printStackTrace();\n}\n";
+        }
+        catchErrorReturn+= "throw error_" + rootNode.getUniqueName() + ";\n}\n";
+        return catchErrorReturn;
+    }
+
     public String generate(Object argument) {
         return "";
     }

main/plugins/org.talend.designer.codegen/jet_stub/generic/component_begin.javajet

@@ -56,11 +56,25 @@ if ((metadatas != null) && (metadatas.size() > 0)) { // metadata
 // Set up the component definition, and the properties for all types of
 // components.
 
+List<? extends IConnection> allInLineJobConns = NodeUtil.getFirstIncomingLineConnectionsOfType(node, "tRESTRequestIn");
+%>
+boolean doesNodeBelongToRequest_<%=cid%> = <%= allInLineJobConns.size() %> == 0;
+@SuppressWarnings("unchecked")
+java.util.Map<String, Object> restRequest_<%=cid%> = (java.util.Map<String, Object>)globalMap.get("restRequest");
+String currentTRestRequestOperation_<%=cid%> = (String)(restRequest_<%=cid%> != null ? restRequest_<%=cid%>.get("OPERATION") : null);
+<%
+for (IConnection inLineConn : allInLineJobConns) {
+    %>
+        if("<%= inLineConn.getName() %>".equals(currentTRestRequestOperation_<%=cid%>)) {
+            doesNodeBelongToRequest_<%=cid%> = true;
+        }
+	<%
+    }
 %>
 
 org.talend.components.api.component.ComponentDefinition def_<%=cid %> =
         new <%= def.getClass().getName()%>();
-        
+
 org.talend.components.api.component.runtime.Writer writer_<%=cid%> = null;
 org.talend.components.api.component.runtime.Reader reader_<%=cid%> = null;
 
@@ -149,7 +163,7 @@ globalMap.put("TALEND_COMPONENTS_VERSION", "<%=component.getVersion()%>");
 boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
 if (isParallelize) {
 %>
-final String buffersSizeKey_<%=cid%> = "buffersSizeKey_<%=cid%>_" + Thread.currentThread().getId(); 
+final String buffersSizeKey_<%=cid%> = "buffersSizeKey_<%=cid%>_" + Thread.currentThread().getId();
 <%
 }
 %>
@@ -219,9 +233,11 @@ if(componentRuntime_<%=cid%> instanceof org.talend.components.api.component.runt
 org.talend.components.api.component.runtime.SourceOrSink sourceOrSink_<%=cid%> = null;
 if(componentRuntime_<%=cid%> instanceof org.talend.components.api.component.runtime.SourceOrSink) {
 	sourceOrSink_<%=cid%> = (org.talend.components.api.component.runtime.SourceOrSink)componentRuntime_<%=cid%>;
-	org.talend.daikon.properties.ValidationResult vr_<%=cid%> = sourceOrSink_<%=cid%>.validate(container_<%=cid%>);
-	if (vr_<%=cid%>.getStatus() == org.talend.daikon.properties.ValidationResult.Result.ERROR ) {
-    	throw new RuntimeException(vr_<%=cid%>.getMessage());
+	if (doesNodeBelongToRequest_<%=cid%>) {
+        org.talend.daikon.properties.ValidationResult vr_<%=cid%> = sourceOrSink_<%=cid%>.validate(container_<%=cid%>);
+        if (vr_<%=cid%>.getStatus() == org.talend.daikon.properties.ValidationResult.Result.ERROR ) {
+            throw new RuntimeException(vr_<%=cid%>.getMessage());
+        }
 	}
 }
 
@@ -297,13 +313,13 @@ if (hasOutputOnly || asInputComponent) {
 
         for (; available_<%=cid%>; available_<%=cid%> = reader_<%=cid%>.advance()) {
 			nb_line_<%=cid %>++;
-	
+
 			<%if(hasDataOutput) {%>
 			if (multi_output_is_allowed_<%=cid%>) {
 				<%if(main!=null){%>
 					<%=main.getName()%> = null;
 				<%}%>
-	
+
 				<%if(reject!=null){%>
 					<%=reject.getName()%> = null;
 				<%}%>
@@ -315,11 +331,11 @@ if (hasOutputOnly || asInputComponent) {
 				<%
 				if (main != null) {
 					%>
-	
+
 					if(multi_output_is_allowed_<%=cid%>) {
 						<%=main.getName()%> = new <%=main.getName() %>Struct();
 					}
-	
+
 					<%
 					irToRow.generateConvertRecord("data_" + cid, main.getName(), main.getMetadataTable().getListColumns());
 				}
@@ -330,7 +346,7 @@ if (hasOutputOnly || asInputComponent) {
 				if (reject!=null) {
 					%>
 					Object data_<%=cid%> = info_<%=cid%>.get("talend_record");
-	
+
 					if (multi_output_is_allowed_<%=cid%>) {
 						<%=reject.getName()%> = new <%=reject.getName() %>Struct();
 					}
@@ -343,19 +359,19 @@ if (hasOutputOnly || asInputComponent) {
 					}
 					<%
 					Set<String> commonColumns = new HashSet<String>();
-	
+
 					for (IMetadataColumn column : columnList) {
 						commonColumns.add(column.getLabel());
 					}
-	
+
 					//pass error columns
 					List<IMetadataColumn> rejectColumns = reject.getMetadataTable().getListColumns();
 					for(IMetadataColumn column : rejectColumns) {
 						String columnName = column.getLabel();
-	
+
 						// JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
 						String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
-	
+
 						//error columns
 						if(!commonColumns.contains(columnName)) {
 							%>
@@ -385,7 +401,7 @@ if (hasOutputOnly || asInputComponent) {
 				<%
 				}
 			%>
-			} // end of catch 
+			} // end of catch
     <%
     // The for loop around the incoming records from the reader is left open.
 
@@ -397,9 +413,13 @@ if (hasOutputOnly || asInputComponent) {
     	org.talend.components.api.component.runtime.Sink sink_<%=cid%> =
             	(org.talend.components.api.component.runtime.Sink)sourceOrSink_<%=cid%>;
         org.talend.components.api.component.runtime.WriteOperation writeOperation_<%=cid%> = sink_<%=cid%>.createWriteOperation();
-        writeOperation_<%=cid%>.initialize(container_<%=cid%>);
+        if (doesNodeBelongToRequest_<%=cid%>) {
+            writeOperation_<%=cid%>.initialize(container_<%=cid%>);
+        }
         writer_<%=cid%> = writeOperation_<%=cid%>.createWriter(container_<%=cid%>);
-        writer_<%=cid%>.open("<%=cid%>");
+        if (doesNodeBelongToRequest_<%=cid%>) {
+            writer_<%=cid%>.open("<%=cid%>");
+        }
 
         resourceMap.put("writer_<%=cid%>", writer_<%=cid%>);
     } // end of "sourceOrSink_<%=cid%> instanceof ...Sink"
@@ -448,7 +468,7 @@ if (hasOutputOnly || asInputComponent) {
     }
 }
  %>
-    
+
                 java.lang.Iterable<?> outgoingMainRecordsList_<%=cid%> = new java.util.ArrayList<Object>();
                 java.util.Iterator outgoingMainRecordsIt_<%=cid%> = null;
 

main/plugins/org.talend.designer.codegen/jet_stub/generic/component_main.javajet

@@ -120,7 +120,8 @@ if(hasInput){
                             dm_<%=cid%>.getLogicalType(),
                             dm_<%=cid%>.getFormat(),
                             dm_<%=cid%>.getDescription(),
-                            dm_<%=cid%>.isNullable());
+                            dm_<%=cid%>.isNullable(),
+                            dm_<%=cid%>.isKey());
                 }
                 incomingEnforcer_<%=cid%>.createRuntimeSchema();
             }

main/plugins/org.talend.designer.codegen/jet_stub/generic/component_util_indexedrecord_to_rowstruct.javajet

@@ -170,16 +170,8 @@ class IndexedRecordToRowStructGenerator {
             if (columnName.equals(dynamicColName)) {
             %>
                 java.util.Map<String, Object> dynamicValue_<%=cid%> = (java.util.Map<String, Object>) <%=codeVarSchemaEnforcer%>.get(<%=i%>);
-				org.apache.avro.Schema dynSchema_<%=cid%> = ((org.talend.codegen.enforcer.OutgoingDynamicSchemaEnforcer) <%=codeVarSchemaEnforcer%>).getDynamicFieldsSchema();
-                for (org.apache.avro.Schema.Field dynamicField_<%=cid%> : dynSchema_<%=cid%>.getFields()){
-					String name = dynamicField_<%=cid%>.name();
-					if("true".equals(dynamicField_<%=cid%>.getProp("ENABLE_SPECIAL_TABLENAME"))){
-							dynamicValue_<%=cid%>.put(dynamicField_<%=cid%>.getProp("talend.field.dbColumnName"), dynamicValue_<%=cid%>.get(name));
-							dynamicValue_<%=cid%>.remove(name);
-					}
-				}
                 for (java.util.Map.Entry<String, Object> dynamicValueEntry_<%=cid%> : dynamicValue_<%=cid%>.entrySet()) {
-                    <%=codeVarDynamic%>.setColumnValue(<%=codeVarDynamic%>.getIndex(dynamicValueEntry_<%=cid%>.getKey()), dynamicValueEntry_<%=cid%>.getValue());
+                    <%=codeVarDynamic%>.addColumnValue(dynamicValueEntry_<%=cid%>.getValue());
                 }
                 <%=codeVarRowStruct%>.<%=dynamicColName%> = <%=codeVarDynamic%>;
             <%

main/plugins/org.talend.designer.codegen/jet_stub/header.javajet

@@ -73,6 +73,9 @@ import pigudf.<%=routine%>;
 import routines.<%=routine%>;
 <%  }
 }%>
+<%for (String codesJar : CodeGeneratorRoutine.getRequiredCodesJarName(process)) {%>
+import <%=codesJar%>;
+<%}%>
 import routines.system.*;
 import routines.system.api.*;
 import java.text.ParseException;
@@ -382,11 +385,101 @@ public <%=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true)%> get<%=Ch
 
 <%
 	INode jobCatcherNode = null;
+
+	//one matched component or part, one thread
+	//why not computed by cpu or resource : please image this case : 
+	//loop==>(input==>(twritexmlfield A)==>(twritexmlfield B)==>(twritexmlfield C)==>output), dead lock as cycle dependency and only one thead in thread pool
+	//maybe newCachedThreadPool is a better idea,  but that have risk for creating more threads, then more memory for TDI-47230
+	//why not generate thread pool object in subprocess scope : 
+	// 1: major reason : difficult to control the var scope, somewhere can't access it, then compiler issue
+	// 2: we may need this thread pool for bigger scope, not only for twritexmlfield/twritejsonfield in future
+	// 3: we don't suppose this thread pool cost big resource after all tasks done, so we can shutdown it more later,
+	//		for example, most time, user will use less than 3 twritexmlfield in one job, then 3 threads thread pool, we can close them in job finish code part, 
+	//		not a big cost to keep that. And of course, we best to start&clean it in subprocess finish, but that's risk of 1 above.
+	int threadPoolSize = 0;
+	boolean tHMapExists = false;
+	boolean tHMapOutExists = false;
+	boolean tRESTRequestLoopExists = false;
 	for (INode nodeInProcess : processNodes) {
 		String componentName = nodeInProcess.getComponent().getName();
-		if("tJobStructureCatcher".equals(componentName)) {
+
+		if(jobCatcherNode==null && "tJobStructureCatcher".equals(componentName)) {
 			jobCatcherNode = nodeInProcess;
-			break;
+			continue;
+		}
+
+		if(!nodeInProcess.isActivate()) continue;
+
+		if("tHMap".equals(componentName)) {
+			tHMapExists = true;
+			continue;
+		}
+
+		if("tHMapOut".equals(componentName)) {
+			tHMapOutExists = true;
+			continue;
+		}
+
+		if("tRESTRequestLoop".equals(componentName)) {
+			tRESTRequestLoopExists = true;
+			continue;
+		}
+		
+		if("tWriteXMLFieldOut".equals(componentName)) {
+			IConnection nextMergeConn = NodeUtil.getNextMergeConnection(nodeInProcess);
+			if(nextMergeConn == null || nextMergeConn.getInputId()==1){
+				threadPoolSize++;
+			}
+		}
+	}
+
+	if(threadPoolSize>0) {
+		if(tRESTRequestLoopExists) {//miscroservice
+%>
+			private class DaemonThreadFactory implements java.util.concurrent.ThreadFactory {
+				java.util.concurrent.ThreadFactory factory = java.util.concurrent.Executors.defaultThreadFactory();
+				
+				public java.lang.Thread newThread(java.lang.Runnable r) {
+					java.lang.Thread t = factory.newThread(r);
+					t.setDaemon(true);
+					return t;
+				}
+			}
+<%
+		}
+
+		if(tHMapExists || tHMapOutExists) {
+%>
+private final java.util.concurrent.ExecutorService es = java.util.concurrent.Executors.newFixedThreadPool(<%=threadPoolSize%> <%if(tRESTRequestLoopExists) {%>,new DaemonThreadFactory()<%}%>);
+<%
+		} else {
+%>
+private final java.util.concurrent.ExecutorService es = java.util.concurrent.Executors.newCachedThreadPool(<%if(tRESTRequestLoopExists) {%>new DaemonThreadFactory()<%}%>);
+<%
+		}
+
+		if(tRESTRequestLoopExists) {//miscroservice
+%>
+		{
+			java.lang.Runtime.getRuntime().addShutdownHook(new java.lang.Thread() {
+				public void run() {
+					es.shutdown();
+					try {
+						if(!es.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) {
+							es.shutdownNow();
+							if(!es.awaitTermination(60, java.util.concurrent.TimeUnit.SECONDS)) {
+
+							}
+						}
+					} catch (java.lang.InterruptedException ie) {
+						es.shutdownNow();
+					} catch (java.lang.Exception e) {
+
+					}
+				}
+			});
+		}
+<%
 		}
 	}
 	

main/plugins/org.talend.designer.codegen/jet_stub/subprocess_header.javajet

@@ -20,6 +20,7 @@
 		org.talend.core.model.process.IHashConfiguration
 		org.talend.core.model.process.IHashableColumn
 		org.talend.core.model.utils.NodeUtil
+		org.talend.core.model.utils.TalendTextUtils
 	"
 	class="SubProcessHeader"
 	skeleton="subprocess_header_java.skeleton"
@@ -183,6 +184,48 @@ public static class <%=conn.getName() %>Struct<%=templateOrigin %> implements ro
 				public <%= typeToGenerate %> get<%=column.getLabel().substring(0, 1).toUpperCase()%><%=column.getLabel().substring(1)%> () {
 					return this.<%=column.getLabel()%>;
 				}
+
+				public Boolean <%=column.getLabel()%>IsNullable(){
+				    return <%=column.isNullable()%>;
+				}
+				public Boolean <%=column.getLabel()%>IsKey(){
+				    return <%=column.isKey()%>;
+				}
+				public Integer <%=column.getLabel()%>Length(){
+				    return <%=column.getLength()%>;
+				}
+				public Integer <%=column.getLabel()%>Precision(){
+				    return <%=column.getPrecision()%>;
+				}
+				public String <%=column.getLabel()%>Default(){
+				<% if (column.getDefault() == null) { %>
+					return null;
+				<% } else { %>
+					return "<%=TalendTextUtils.escapeJavaText(TalendTextUtils.removeQuotes(column.getDefault()))%>";
+				<% } %>
+				}
+				public String <%=column.getLabel()%>Comment(){
+				<% if (column.getComment() == null) { %>
+				    return null;
+				<% } else { %>
+				    return "<%=TalendTextUtils.escapeJavaText(TalendTextUtils.removeQuotes(column.getComment()))%>";
+				<% } %>
+				}
+				public String <%=column.getLabel()%>Pattern(){
+				<% if (column.getPattern() == null) { %>
+				    return null;
+				<% } else { %>
+					return "<%=TalendTextUtils.escapeJavaText(TalendTextUtils.removeQuotes(column.getPattern()))%>";
+				<% } %>
+				}
+				public String <%=column.getLabel()%>OriginalDbColumnName(){
+				<% if (column.getOriginalDbColumnName() == null) { %>
+				    return null;
+				<% } else { %>
+					return "<%=TalendTextUtils.escapeJavaText(TalendTextUtils.removeQuotes(column.getOriginalDbColumnName()))%>";
+				<% } %>
+				}
+
 				<%
 				if((conn.getLineStyle() == EConnectionType.FLOW_REF) && conn.getTarget().getUniqueName().startsWith("tXMLMap") && "id_Document".equals(javaType.getId())) {
 				%>

main/plugins/org.talend.designer.codegen/src/main/java/messages.properties

@@ -10,6 +10,7 @@ CodeGenerator.getGraphicalNode2=------process.getGeneratingNodes()------
 CodeGenerator.JET.TimeOut=JET initialisation Time Out
 CodeGenerator.newLine=\n\n\n\n
 CodeGenerator.Node.NotFound=Node not found in current process
+CodeGenerator.Components.NotFound={0}: Component is missing: {1}; use -D{2}=false in your studio or commandline to skip this check, and a warning message will be logged.
 JavaRoutineSynchronizer.UnsupportedOperation.Exception1=method not implemented: org.talend.designer.codegen.JavaRoutineSynchronizer line:49
 JavaRoutineSynchronizer.UnsupportedOperation.Exception2=method not implemented: org.talend.designer.codegen.JavaRoutineSynchronizer line:58
 JetSkeletonManager.unableLoad=unable to load skeleton update cache file

main/plugins/org.talend.designer.codegen/src/main/java/org/talend/designer/codegen/CodeGenerator.java

@@ -66,6 +66,7 @@ import org.talend.designer.codegen.model.CodeGeneratorEmittersPoolFactory;
 import org.talend.designer.codegen.model.CodeGeneratorInternalTemplatesFactoryProvider;
 import org.talend.designer.codegen.proxy.JetProxy;
 import org.talend.designer.core.generic.model.Component;
+import org.talend.designer.core.model.components.DummyComponent;
 import org.talend.designer.runprocess.ProcessorUtilities;
 
 /**
@@ -845,6 +846,25 @@ public class CodeGenerator implements ICodeGenerator {
             IComponentFileNaming componentFileNaming = ComponentsFactoryProvider.getFileNamingInstance();
 
             IComponent component = node.getComponent();
+            if (component instanceof DummyComponent) {
+                if (((DummyComponent) component).isMissingComponent()) {
+                    String processName = "";
+                    try {
+                        IProcess proc = node.getProcess();
+                        processName = proc.getName() + " " + proc.getVersion();
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                    if (IProcess.ERR_ON_COMPONENT_MISSING) {
+                        throw new CodeGeneratorException(Messages.getString("CodeGenerator.Components.NotFound", processName,
+                                component.getName(), IProcess.PROP_ERR_ON_COMPONENT_MISSING));
+                    }
+                    if (ECodePart.BEGIN.equals(part)) {
+                        log.warn(Messages.getString("CodeGenerator.Components.NotFound", processName, component.getName(),
+                                IProcess.PROP_ERR_ON_COMPONENT_MISSING));
+                    }
+                }
+            }
             // some code unification to handle all component types the same way.
             String templateURI = component.getTemplateFolder() + TemplateUtil.DIR_SEP
                     + componentFileNaming.getJetFileName(component.getTemplateNamePrefix(), language.getExtension(), part);

main/plugins/org.talend.designer.codegen/src/main/java/org/talend/designer/codegen/JavaRoutineSynchronizer.java

@@ -69,6 +69,15 @@ public class JavaRoutineSynchronizer extends AbstractRoutineSynchronizer {
         syncRoutineItems(getRoutines(true), true);
     }
 
+    @Override
+    public void syncAllInnerCodes() throws SystemException {
+        syncInnerCodeItems(false);
+    }
+
+    @Override
+    public void syncAllInnerCodesForLogOn() throws SystemException {
+        syncInnerCodeItems(true);
+    }
 
     private void syncRoutineItems(Collection<RoutineItem> routineObjects, boolean forceUpdate) throws SystemException {
         for (RoutineItem routineItem : routineObjects) {

main/plugins/org.talend.designer.components.exchange/META-INF/MANIFEST.MF

@@ -8,6 +8,7 @@ Require-Bundle: org.eclipse.core.runtime,
  org.eclipse.ui,
  org.apache.log4j,
  org.apache.commons.collections,
+ org.apache.commons.discovery,
  org.apache.commons.logging,
  org.apache.commons.beanutils,
  org.apache.commons.io,
@@ -25,7 +26,6 @@ Require-Bundle: org.eclipse.core.runtime,
  org.talend.repository,
  org.talend.core.repository,
  org.talend.updates.runtime,
- org.apache.axis,
  org.eclipse.ui.intro,
  org.eclipse.ui.forms,
  org.eclipse.jface.text

main/plugins/org.talend.designer.components.exchange/src/main/java/org/talend/designer/components/exchange/proxy/DefaultHTTPSTransportClientProperties.java

@@ -1,66 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.designer.components.exchange.proxy;
-
-import org.apache.commons.lang.StringUtils;
-
-/**
- *
- * DOC hcyi class global comment. Detailled comment
- */
-public class DefaultHTTPSTransportClientProperties extends DefaultHTTPTransportClientProperties {
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getProxyHost()
-     */
-    @Override
-    public String getProxyHost() {
-        return StringUtils.trimToEmpty(System.getProperty("https.proxyHost")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getNonProxyHosts()
-     */
-    @Override
-    public String getNonProxyHosts() {
-
-        return StringUtils.trimToEmpty(System.getProperty("https.nonProxyHosts")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getPort()
-     */
-    @Override
-    public String getProxyPort() {
-
-        return StringUtils.trimToEmpty(System.getProperty("https.proxyPort")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getUser()
-     */
-    @Override
-    public String getProxyUser() {
-
-        return StringUtils.trimToEmpty(System.getProperty("https.proxyUser")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getPassword()
-     */
-    @Override
-    public String getProxyPassword() {
-
-        return StringUtils.trimToEmpty(System.getProperty("https.proxyPassword")); //$NON-NLS-1$
-    }
-}

main/plugins/org.talend.designer.components.exchange/src/main/java/org/talend/designer/components/exchange/proxy/DefaultHTTPTransportClientProperties.java

@@ -1,58 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.designer.components.exchange.proxy;
-
-import org.apache.axis.components.net.TransportClientProperties;
-import org.apache.commons.lang.StringUtils;
-
-/**
- *
- * DOC hcyi class global comment. Detailled comment
- */
-public class DefaultHTTPTransportClientProperties implements TransportClientProperties {
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getProxyHost()
-     */
-    public String getProxyHost() {
-        return StringUtils.trimToEmpty(System.getProperty("http.proxyHost")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getNonProxyHosts()
-     */
-    public String getNonProxyHosts() {
-        return StringUtils.trimToEmpty(System.getProperty("http.nonProxyHosts")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getPort()
-     */
-    public String getProxyPort() {
-        return StringUtils.trimToEmpty(System.getProperty("http.proxyPort")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getProxyUser()
-     */
-    public String getProxyUser() {
-        return StringUtils.trimToEmpty(System.getProperty("http.proxyUser")); //$NON-NLS-1$
-    }
-
-    /**
-     * @see org.apache.axis.components.net.TransportClientProperties#getProxyPassword()
-     */
-    public String getProxyPassword() {
-        return StringUtils.trimToEmpty(System.getProperty("http.proxyPassword")); //$NON-NLS-1$
-    }
-}

main/plugins/org.talend.designer.components.exchange/src/main/java/org/talend/designer/components/exchange/util/ExchangeUtils.java

@@ -25,11 +25,9 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-
-import org.apache.axis.components.net.TransportClientProperties;
-import org.apache.axis.components.net.TransportClientPropertiesFactory;
 import org.apache.commons.beanutils.BeanUtils;
 import org.apache.commons.collections.map.MultiValueMap;
+import org.apache.commons.discovery.tools.ManagedProperties;
 import org.apache.commons.httpclient.HostConfiguration;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.NameValuePair;
@@ -151,14 +149,17 @@ public class ExchangeUtils {
     public static String sendGetRequest(String urlAddress) throws Exception {
         HttpClient httpclient = new HttpClient();
         GetMethod getMethod = new GetMethod(urlAddress);
-        TransportClientProperties tcp = TransportClientPropertiesFactory.create("http");
-        if (tcp.getProxyHost().length() != 0) {
+        String proxyUser = ManagedProperties.getProperty("http.proxyUser");
+        String proxyPassword = ManagedProperties.getProperty("http.proxyPassword");
+        String proxyHost = ManagedProperties.getProperty("http.proxyHost");
+        proxyHost = proxyHost != null ? proxyHost : "";
+        String proxyPort = ManagedProperties.getProperty("http.proxyPort");
+        if (proxyHost.length() != 0) {
             UsernamePasswordCredentials creds = new UsernamePasswordCredentials(
-                    tcp.getProxyUser() != null ? tcp.getProxyUser() : "",
-                    tcp.getProxyPassword() != null ? tcp.getProxyUser() : "");
+                    proxyUser != null ? proxyUser : "", proxyPassword != null ? proxyPassword : "");
             httpclient.getState().setProxyCredentials(AuthScope.ANY, creds);
             HostConfiguration hcf = new HostConfiguration();
-            hcf.setProxy(tcp.getProxyHost(), Integer.parseInt(tcp.getProxyPort()));
+            hcf.setProxy(proxyHost, Integer.parseInt(proxyPort));
             httpclient.executeMethod(hcf, getMethod);
         } else {
             httpclient.executeMethod(getMethod);

main/plugins/org.talend.designer.components.libs/libs_src/adal4j/pom.xml

@@ -100,7 +100,7 @@
 		<dependency>
 			<groupId>com.google.code.gson</groupId>
 			<artifactId>gson</artifactId>
-			<version>2.2.4</version>
+			<version>2.8.9</version>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>

main/plugins/org.talend.designer.components.libs/libs_src/adal4j/src/samples/web-app-samples-for-adal4j/pom.xml

@@ -42,7 +42,18 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-log4j12</artifactId>
 			<version>1.7.25</version>
+			<exclusions>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+            </exclusions>
 		</dependency>
+		<dependency>
+            <groupId>ch.qos.reload4j</groupId>
+            <artifactId>reload4j</artifactId>
+            <version>1.2.19</version>
+        </dependency>
 		<!-- Spring 3 dependencies -->
 		<dependency>
 			<groupId>org.springframework</groupId>

main/plugins/org.talend.designer.components.libs/libs_src/checkArchive/pom.xml

@@ -1,69 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>org.talend.libraries</groupId>
-  <artifactId>checkArchive-1.1-20190917</artifactId>
-  <version>6.0.0</version>
-  <name>checkArchive</name>
-  <description>Dependence for tFileArchive and tFileUnAchive</description>
-<url>http://maven.apache.org</url>
-
-	<properties>
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
-		<java.source.version>1.7</java.source.version>
-	</properties>
-
-	<distributionManagement>
-		<snapshotRepository>
-			<id>talend_nexus_deployment</id>
-			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
-			<snapshots>
-				<enabled>true</enabled>
-			</snapshots>
-			<releases>
-				<enabled>false</enabled>
-			</releases>
-		</snapshotRepository>
-		<repository>
-			<id>talend_nexus_deployment</id>
-			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
-			<snapshots>
-				<enabled>false</enabled>
-			</snapshots>
-			<releases>
-				<enabled>true</enabled>
-			</releases>
-		</repository>
-	</distributionManagement>
-
-	<dependencies>
-  <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
-<dependency>
-    <groupId>org.apache.commons</groupId>
-    <artifactId>commons-compress</artifactId>
-    <version>1.19</version>
-</dependency>
-	</dependencies>
-	<build>
-		<resources>
-			<resource>
-				<directory>src/main/java</directory>
-			</resource>
-		</resources>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-compiler-plugin</artifactId>
-				<version>2.3.2</version>
-				<configuration>
-					<source>${java.source.version}</source>
-					<target>${java.source.version}</target>
-					<showDeprecation>true</showDeprecation>
-					<showWarnings>true</showWarnings>
-					<compilerArgument>-XDignore.symbol.file</compilerArgument>
-					<fork>true</fork>
-				</configuration>
-			</plugin>
-		</plugins>
-	</build>
-</project>

main/plugins/org.talend.designer.components.libs/libs_src/filecopy/pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.talend.components</groupId>
 	<artifactId>filecopy</artifactId>
-	<version>2.0.1</version>
+	<version>2.0.3</version>
 	<packaging>jar</packaging>
 
 	<name>talend-copy</name>
@@ -14,6 +14,7 @@
 		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
 		<java.source.version>1.8</java.source.version>
 		<junit5.version>5.4.2</junit5.version>
+		<slf4j.version>1.7.28</slf4j.version>
 	</properties>
 
 	<distributionManagement>
@@ -52,7 +53,12 @@
 			<version>${junit5.version}</version>
 			<scope>test</scope>
 		</dependency>
-
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>${slf4j.version}</version>
+			<scope>provided</scope>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

main/plugins/org.talend.designer.components.libs/libs_src/filecopy/src/main/java/org/talend/FileCopy.java

@@ -20,11 +20,16 @@ import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.FileTime;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * DOC Administrator class global comment. Detailled comment
  */
 public class FileCopy {
 
+    static Logger  logger = LoggerFactory.getLogger(Object.class);
+
     /** Private constructor, only static methods */
     private FileCopy() {
     }
@@ -37,19 +42,57 @@ public class FileCopy {
      * @param delSrc : true if delete source.
      * @throws IOException : if IO pb.
      */
-    public static void copyFile(String srcFileName, String desFileName, boolean delSrc) throws IOException {
+    public static void copyFile(String srcFileName, String desFileName, boolean delSrc, boolean keepModified)
+            throws IOException {
         final Path source = Paths.get(srcFileName);
         final Path destination =  Paths.get(desFileName);
-
+        FileTime lastModifiedTime = null;
+        try {
+            lastModifiedTime = Files.getLastModifiedTime(source);
+        } catch (IOException e) {
+            logger.warn(e.getLocalizedMessage());
+        }
         if (delSrc) {
             // move : more efficient if in same FS and mustr delete existing file.
-            FileTime lastModifiedTime = Files.getLastModifiedTime(source);
             Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING);
-            Files.setLastModifiedTime(destination,lastModifiedTime);
         } else {
             Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
-            Files.setLastModifiedTime(destination,Files.getLastModifiedTime(source));
+        }
+        if(keepModified){
+            try {
+                Files.setLastModifiedTime(destination,lastModifiedTime);
+            } catch (IOException e) {
+                logger.warn(e.getLocalizedMessage());
+            }
         }
     }
 
+    public static void copyFile(String srcFileName, String desFileName, boolean delSrc ) throws IOException {
+        copyFile(srcFileName,desFileName,delSrc,true);
+    }
+
+    /**
+     * Force Copy and Delete files.
+     *
+     * @param srcFileName : file name for source file.
+     * @param desFileName : file name for destination file.
+     * @throws IOException : if IO pb.
+     */
+    public static void forceCopyAndDelete(String srcFileName, String desFileName, boolean keepModified) throws IOException {
+        final Path source = Paths.get(srcFileName);
+        final Path destination =  Paths.get(desFileName);
+        final long lastModifiedTime = new File(srcFileName).lastModified();
+
+        Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
+        Files.delete(source);
+        if(keepModified){
+            destination.toFile().setLastModified(lastModifiedTime);
+        }
+
+    }
+
+    public static void forceCopyAndDelete(String srcFileName, String desFileName) throws IOException {
+        forceCopyAndDelete(srcFileName,desFileName,true);
+    }
+
 }

main/plugins/org.talend.designer.components.libs/libs_src/filecopy/src/test/java/org/talend/FileCopyTest.java

@@ -100,6 +100,26 @@ class FileCopyTest {
         Assertions.assertEquals(referenceSize, copy.length(), "Size error");
     }
 
+    @Test
+    void testForceCopyWithDelete() throws Exception {
+        final URL repCopy = Thread.currentThread().getContextClassLoader().getResource("copy");
+
+        File file = this.buildFile("fileToDelete.txt", 10L * 1024L);
+        file.deleteOnExit();
+        File copy = new File(repCopy.getPath(), "fileToDelete.txt");
+        long referenceSize = file.length();
+        if (!copy.exists()) {
+            copy.createNewFile();
+        }
+        copy.deleteOnExit();
+
+        FileCopy.forceCopyAndDelete(file.getPath(), copy.getPath());
+
+        Assertions.assertFalse(file.exists(), "file not delete");
+        Assertions.assertTrue(copy.exists(), "small file : original file deleted");
+        Assertions.assertEquals(referenceSize, copy.length(), "Size error");
+    }
+
     @Test
     void testLastModifiedTime() throws Exception {
         final URL repCopy = Thread.currentThread().getContextClassLoader().getResource("copy");
@@ -143,4 +163,22 @@ class FileCopyTest {
 
         return generatedFile;
     }
+
+    @Test
+    void testKeepLastModifiedTime() throws Exception {
+        final URL repCopy = Thread.currentThread().getContextClassLoader().getResource("copy");
+
+        File file = this.buildFile("fileLMT.txt", 10L * 1024L);
+        file.deleteOnExit();
+        long referencceTime = 324723894L;
+        file.setLastModified(referencceTime);
+
+        File copy = new File(repCopy.getPath(), "fileLMTDestination.txt");
+        if (copy.exists()) {
+            copy.delete();
+        }
+        copy.deleteOnExit();
+        FileCopy.copyFile(file.getPath(), copy.getPath(), true,true);
+        Assertions.assertEquals(referencceTime, copy.lastModified(), "modified time is not idential");
+    }
 }

main/plugins/org.talend.designer.components.libs/libs_src/json-lib-talend/pom.xml

@@ -230,9 +230,9 @@
          <scope>compile</scope>
       </dependency>
       <dependency>
-         <groupId>log4j</groupId>
-         <artifactId>log4j</artifactId>
-         <version>1.2.14</version>
+         <groupId>ch.qos.reload4j</groupId>
+         <artifactId>reload4j</artifactId>
+         <version>1.2.19</version>
          <scope>runtime</scope>
          <optional>true</optional>
       </dependency>

main/plugins/org.talend.designer.components.libs/libs_src/mdm_services_tomcat/pom.xml

@@ -11,7 +11,7 @@
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<cxf.version>3.1.2</cxf.version>
+		<cxf.version>3.4.7</cxf.version>
 	</properties>
 
 	<build>
@@ -77,8 +77,8 @@
                     <artifactId>maven-compiler-plugin</artifactId>
                     <version>2.5.1</version>
                     <configuration>
-                        <source>1.7</source>
-                        <target>1.7</target>
+                        <source>1.8</source>
+                        <target>1.8</target>
                         <encoding>UTF-8</encoding>
                     </configuration>
                 </plugin>

main/plugins/org.talend.designer.components.libs/libs_src/mdm_services_tomcat/src/main/java/org/talend/mdm/webservice/WSTransformerV2.java

@@ -22,6 +22,7 @@ import javax.xml.bind.annotation.XmlType;
  *         <element name="description" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/>
  *         <element name="name" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/>
  *         <element name="processSteps" type="{http://www.talend.com/mdm}WSTransformerProcessStep" maxOccurs="unbounded" minOccurs="0"/>
+ *         <element name="withAdminPermissions" type="{http://www.w3.org/2001/XMLSchema}boolean" minOccurs="0"/>
  *       </sequence>
  *     </restriction>
  *   </complexContent>
@@ -34,7 +35,8 @@ import javax.xml.bind.annotation.XmlType;
 @XmlType(name = "WSTransformerV2", propOrder = {
     "description",
     "name",
-    "processSteps"
+    "processSteps",
+    "withAdminPermissions"
 })
 public class WSTransformerV2 {
 
@@ -42,6 +44,7 @@ public class WSTransformerV2 {
     protected String name;
     @XmlElement(nillable = true)
     protected List<WSTransformerProcessStep> processSteps;
+    protected Boolean withAdminPermissions;
 
     /**
      * Default no-arg constructor
@@ -55,10 +58,11 @@ public class WSTransformerV2 {
      * Fully-initialising value constructor
      * 
      */
-    public WSTransformerV2(final String description, final String name, final List<WSTransformerProcessStep> processSteps) {
+    public WSTransformerV2(final String description, final String name, final List<WSTransformerProcessStep> processSteps, final Boolean withAdminPermissions) {
         this.description = description;
         this.name = name;
         this.processSteps = processSteps;
+        this.withAdminPermissions = withAdminPermissions;
     }
 
     /**
@@ -138,4 +142,28 @@ public class WSTransformerV2 {
         return this.processSteps;
     }
 
+    /**
+     * Gets the value of the withAdminPermissions property.
+     * 
+     * @return
+     *     possible object is
+     *     {@link Boolean }
+     *     
+     */
+    public Boolean isWithAdminPermissions() {
+        return withAdminPermissions;
+    }
+
+    /**
+     * Sets the value of the withAdminPermissions property.
+     * 
+     * @param value
+     *     allowed object is
+     *     {@link Boolean }
+     *     
+     */
+    public void setWithAdminPermissions(Boolean value) {
+        this.withAdminPermissions = value;
+    }
+    
 }

main/plugins/org.talend.designer.components.libs/libs_src/mqrfh2header/pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.talend</groupId>
 	<artifactId>talendMQRFH2</artifactId>
-	<version>1.0.1-20190206</version>
+	<version>1.1.0-20220307</version>
 	<packaging>jar</packaging>
 
 	<properties>
@@ -36,20 +36,10 @@
 	</distributionManagement>
 
 	<dependencies>
-		<dependency>
-			<groupId>com.ibm.mq</groupId>
-			<artifactId>com.ibm.mq</artifactId>
-			<version>8.0.0.9</version>
-		</dependency>
-		<dependency>
-			<groupId>com.ibm.mq</groupId>
-			<artifactId>com.ibm.mqjms</artifactId>
-			<version>8.0.0.9</version>
-		</dependency>
 		<dependency>
 			<groupId>com.ibm.mq</groupId>
 			<artifactId>com.ibm.mq.allclient</artifactId>
-			<version>8.0.0.9</version>
+			<version>9.2.4.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.talend.libraries</groupId>
@@ -67,9 +57,9 @@
 			<version>6.0.0</version>
 		</dependency>
 		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>1.2.17</version>
+			<groupId>ch.qos.reload4j</groupId>
+			<artifactId>reload4j</artifactId>
+			<version>1.2.19</version>
 		</dependency>
 	</dependencies>
 

main/plugins/org.talend.designer.components.libs/libs_src/msmq/build.xml

@@ -6,7 +6,7 @@
 	<!-- #################################################### -->
 
 	<!-- modification 1: config -->
-	<property name="jar.name" value="MsmqTalend-1.0.0.jar" />
+	<property name="jar.name" value="MsmqTalend-1.0.5.jar" />
 	<property name="component.name" value="tMicrosoftMQInput" />
 	<property name="author.name" value="ytao" />
 

main/plugins/org.talend.designer.components.libs/libs_src/neo4j-talend-component/pom.xml

@@ -113,9 +113,9 @@
             <scope>test</scope>
         </dependency>
 		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>1.2.17</version>
+			<groupId>ch.qos.reload4j</groupId>
+			<artifactId>reload4j</artifactId>
+			<version>1.2.19</version>
 			<scope>test</scope>
 		</dependency>
 		<!-- JUnit -->

main/plugins/org.talend.designer.components.libs/libs_src/simpleexcel/pom.xml

@@ -63,9 +63,9 @@
 			<version>4.1.2</version>
 		</dependency>
 		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>1.2.17</version>
+			<groupId>ch.qos.reload4j</groupId>
+			<artifactId>reload4j</artifactId>
+			<version>1.2.19</version>
 		</dependency>
 	</dependencies>
 	<build>
@@ -90,4 +90,4 @@
 			</plugin>
 		</plugins>
 	</build>
-</project>
+</project>

main/plugins/org.talend.designer.components.libs/libs_src/simpleexcel/src/main/java/com/talend/excel/xssf/event/DefaultTalendSheetContentsHandler.java

@@ -108,4 +108,4 @@ public class DefaultTalendSheetContentsHandler implements TalendXSSFSheetXMLHand
         }
         return columnIndex;
     }
-}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-codegen-utils/pom.xml

@@ -7,21 +7,21 @@
 	<groupId>org.talend.libraries</groupId>
 	<artifactId>talend-codegen-utils</artifactId>
 	<!-- release for revert version of library -->
-	<version>0.28.0</version>
+	<version>0.31.0</version>
 	<packaging>jar</packaging>
 	
 	<properties>
 		<avro.version>1.8.0</avro.version>
-		<components.version>0.25.0-SNAPSHOT</components.version>
-		<daikon.version>0.26.0-SNAPSHOT</daikon.version>
+		<components.version>0.30.0</components.version>
+		<daikon.version>0.31.11</daikon.version>
 		<hamcrest.version>1.3</hamcrest.version>
 		<junit.version>4.12</junit.version>
 		<java-formatter.plugin.version>0.1.0</java-formatter.plugin.version>
 		<formatter.plugin.version>1.6.0-SNAPSHOT</formatter.plugin.version>
 		<mockito.version>2.2.15</mockito.version>
 		<jacoco.plugin.version>0.7.8</jacoco.plugin.version>
-		<maven.compiler.source>1.7</maven.compiler.source>
-		<maven.compiler.target>1.7</maven.compiler.target>
+		<maven.compiler.source>1.8</maven.compiler.source>
+		<maven.compiler.target>1.8</maven.compiler.target>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
 	</properties>

main/plugins/org.talend.designer.components.libs/libs_src/talend-codegen-utils/src/main/java/org/talend/codegen/enforcer/IncomingSchemaEnforcer.java

@@ -24,8 +24,10 @@ import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TimeZone;
 
 import org.apache.avro.Schema;
@@ -33,9 +35,11 @@ import org.apache.avro.Schema.Field;
 import org.apache.avro.SchemaBuilder;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.IndexedRecord;
+import org.apache.avro.SchemaParseException;
 import org.talend.codegen.DiSchemaConstants;
 import org.talend.daikon.avro.AvroUtils;
 import org.talend.daikon.avro.LogicalTypeUtils;
+import org.talend.daikon.avro.NameUtil;
 import org.talend.daikon.avro.SchemaConstants;
 
 /**
@@ -133,6 +137,7 @@ public class IncomingSchemaEnforcer {
         }
     }
 
+    //TODO remove this method as no place use it now in javajet
     /**
      * Take all of the parameters from the dynamic metadata and adapt it to a field for the runtime Schema.
      *
@@ -144,6 +149,62 @@ public class IncomingSchemaEnforcer {
         addDynamicField(name, type, null, format, description, isNullable);
     }
 
+    private Set<String> existNames;
+
+    private Map<String, String> unvalidName2ValidName;
+
+    private int index = 0;
+
+    /**
+     * Recreates dynamic field from parameters retrieved from DI dynamic metadata
+     *
+     * @param name dynamic field name
+     * @param diType di column type
+     * @param logicalType dynamic field logical type; could be null
+     * @param fieldPattern dynamic field date format
+     * @param description dynamic field description
+     * @param isNullable defines whether dynamic field may contain <code>null</code> value
+     * @param isKey defines whether dynamic field is key field
+     */
+    public void addDynamicField(String name, String diType, String logicalType, String fieldPattern, String description,
+            boolean isNullable, boolean isKey) {
+        if (!needsInitDynamicColumns())
+            return;
+        Schema fieldSchema = diToAvro(diType, logicalType);
+
+        if (isNullable) {
+            fieldSchema = SchemaBuilder.nullable().type(fieldSchema);
+        }
+
+        Schema.Field field;
+        try {
+            field = new Schema.Field(name, fieldSchema, description, (Object) null);
+        } catch (SchemaParseException e) {
+            //if the name contains special char which can't pass avro name check like $ and #, 
+            //but uniode like Japanese which can pass too though that is not expected
+            if (existNames == null) {
+                existNames = new HashSet<>();
+                unvalidName2ValidName = new HashMap<>();
+            }
+
+            String validName = NameUtil.correct(name, index++, existNames);
+            existNames.add(validName);
+            unvalidName2ValidName.put(name, validName);
+
+            field = new Schema.Field(validName, fieldSchema, description, (Object) null);
+            field.addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, name);
+        }
+
+        // Set pattern for date type
+        if ("id_Date".equals(diType) && fieldPattern != null) {
+            field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, fieldPattern);
+        }
+        if (isKey) {
+            field.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true");
+        }
+        dynamicFields.add(field);
+    }
+
     /**
      * Recreates dynamic field from parameters retrieved from DI dynamic metadata
      *
@@ -154,21 +215,10 @@ public class IncomingSchemaEnforcer {
      * @param description dynamic field description
      * @param isNullable defines whether dynamic field may contain <code>null</code> value
      */
+    @Deprecated
     public void addDynamicField(String name, String diType, String logicalType, String fieldPattern, String description,
             boolean isNullable) {
-        if (!needsInitDynamicColumns())
-            return;
-        Schema fieldSchema = diToAvro(diType, logicalType);
-
-        if (isNullable) {
-            fieldSchema = SchemaBuilder.nullable().type(fieldSchema);
-        }
-        Schema.Field field = new Schema.Field(name, fieldSchema, description, (Object) null);
-        // Set pattern for date type
-        if ("id_Date".equals(diType) && fieldPattern != null) {
-            field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, fieldPattern);
-        }
-        dynamicFields.add(field);
+        addDynamicField(name, diType, logicalType, fieldPattern, description, isNullable, false);
     }
 
     public void addIncomingNodeField(String name, String className) {
@@ -250,6 +300,8 @@ public class IncomingSchemaEnforcer {
             fieldSchema = AvroUtils._decimal();
         } else if ("id_Date".equals(diType)) {
             fieldSchema = AvroUtils._date();
+        } else if ("id_byte[]".equals(diType)) {
+            fieldSchema = AvroUtils._bytes();
         } else {
             throw new UnsupportedOperationException("Unrecognized type " + diType);
         }
@@ -369,6 +421,9 @@ public class IncomingSchemaEnforcer {
         return designSchema;
     }
 
+    //here we do special process for dynamic input name, but in fact, 
+    //we have issue which support Japanese char or special char as label for basic talend column too,
+    //so not only dynamic columns may have special name, but also basic may have, but here, we don't consider that, that's TODO
     /**
      * Converts DI data value to Avro format and put it into record by field name
      *
@@ -376,9 +431,16 @@ public class IncomingSchemaEnforcer {
      * @param diValue data value
      */
     public void put(String name, Object diValue) {
+        if (unvalidName2ValidName != null) {
+            String validName = unvalidName2ValidName.get(name);
+            if (validName != null) {
+                name = validName;
+            }
+        }
         put(columnToFieldIndex.get(name), diValue);
     }
 
+    //TODO make it private, no place to call it except current class?
     /**
      * Converts DI data value to Avro format and put it into record by field index
      *

main/plugins/org.talend.designer.components.libs/libs_src/talend-codegen-utils/src/test/java/org/talend/codegen/enforcer/IncomingSchemaEnforcerTest.java

@@ -52,6 +52,8 @@ public class IncomingSchemaEnforcerTest {
      */
     private IndexedRecord componentRecord;
 
+    private IndexedRecord componentRecordWithSpecialName;
+
     @Rule
     public ExpectedException thrown = ExpectedException.none();
 
@@ -72,9 +74,29 @@ public class IncomingSchemaEnforcerTest {
         componentRecord.put(3, true);
         componentRecord.put(4, "Main Street");
         componentRecord.put(5, "This is a record with six columns.");
+
+        Schema componentSchemaWithSpecialName = SchemaBuilder.builder().record("Record").fields() //
+                .name("id").type().intType().noDefault() //
+                .name("name").type().stringType().noDefault() //
+                .name("age").type().intType().noDefault() //
+                .name("性别").type().booleanType().noDefault() //why this don't store the origin name, as it can pass the avro name check, it's a avro bug
+                .name("address_").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "address#").type().stringType().noDefault() //
+                .name("comment_").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "comment$").type().stringType().noDefault() //
+                .endRecord();
+        componentRecordWithSpecialName = new GenericData.Record(componentSchemaWithSpecialName);
+        componentRecordWithSpecialName.put(0, 1);
+        componentRecordWithSpecialName.put(1, "User");
+        componentRecordWithSpecialName.put(2, 100);
+        componentRecordWithSpecialName.put(3, true);
+        componentRecordWithSpecialName.put(4, "Main Street");
+        componentRecordWithSpecialName.put(5, "This is a record with six columns.");
     }
 
     private void checkEnforcerWithComponentRecordData(IncomingSchemaEnforcer enforcer) {
+        checkEnforcerWithComponentRecordData(enforcer, false);
+    }
+
+    private void checkEnforcerWithComponentRecordData(IncomingSchemaEnforcer enforcer, boolean specialName) {
         // The enforcer must be ready to receive values.
         assertThat(enforcer.needsInitDynamicColumns(), is(false));
 
@@ -88,15 +110,25 @@ public class IncomingSchemaEnforcerTest {
         IndexedRecord adapted = enforcer.createIndexedRecord();
 
         // Ensure that the result is the same as the expected component record.
-        assertThat(adapted, is(componentRecord));
+        if (specialName) {
+            assertThat(adapted, is(componentRecordWithSpecialName));
+        } else {
+            assertThat(adapted, is(componentRecord));
+        }
 
         // Ensure that we create a new instance when we give it another value.
         enforcer.put("id", 2);
         enforcer.put("name", "User2");
         enforcer.put("age", 200);
-        enforcer.put("valid", false);
-        enforcer.put("address", "2 Main Street");
-        enforcer.put("comment", "2 This is a record with six columns.");
+        if (specialName) {
+            enforcer.put("性别", false);
+            enforcer.put("address#", "2 Main Street");
+            enforcer.put("comment$", "2 This is a record with six columns.");
+        } else {
+            enforcer.put("valid", false);
+            enforcer.put("address", "2 Main Street");
+            enforcer.put("comment", "2 This is a record with six columns.");
+        }
         IndexedRecord adapted2 = enforcer.createIndexedRecord();
 
         // It should have the same schema, but not be the same instance.
@@ -392,6 +424,39 @@ public class IncomingSchemaEnforcerTest {
         checkEnforcerWithComponentRecordData(enforcer);
     }
 
+    @Test
+    public void testDynamicColumnWithSpecialName() {
+        Schema designSchema = SchemaBuilder.builder().record("Record") //
+                .prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "3") //
+                .prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
+                .fields() //
+                .name("id").type().intType().noDefault() //
+                .name("name").type().stringType().noDefault() //
+                .name("age").type().intType().noDefault() //
+                .endRecord();
+
+        IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
+
+        // The enforcer isn't usable yet.
+        assertThat(enforcer.getDesignSchema(), is(designSchema));
+        assertFalse(enforcer.areDynamicFieldsInitialized());
+        assertThat(enforcer.getRuntimeSchema(), nullValue());
+
+        enforcer.addDynamicField("性别", "id_Boolean", null, null, null, false, false);
+        enforcer.addDynamicField("address#", "id_String", null, null, null, false, false);
+        enforcer.addDynamicField("comment$", "id_String", null, null, null, false, false);
+        assertFalse(enforcer.areDynamicFieldsInitialized());
+        enforcer.createRuntimeSchema();
+        assertTrue(enforcer.areDynamicFieldsInitialized());
+
+        // Check the run-time schema was created.
+        assertThat(enforcer.getDesignSchema(), is(designSchema));
+        assertThat(enforcer.getRuntimeSchema(), not(nullValue()));
+
+        // Put values into the enforcer and get them as an IndexedRecord.
+        checkEnforcerWithComponentRecordData(enforcer, true);
+    }
+
     @Test
     public void testTypeConversion_toDate() {
         // The expected schema after enforcement.
@@ -699,6 +764,28 @@ public class IncomingSchemaEnforcerTest {
         assertThat(record.get(1), is((Object) new Date(1234567891011L)));
     }
 
+    /**
+     * Checks key field setting
+     */
+    @Test
+    public void testAddDynamicFieldKey() {
+        Schema expectedRuntimeSchema = SchemaBuilder.builder().record("Record").fields().name("id")
+                .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().intType().noDefault().endRecord();
+
+        Schema designSchema = SchemaBuilder.builder().record("Record").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true")
+                .prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0").fields().endRecord();
+
+        IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
+
+        enforcer.addDynamicField("id", "id_Integer", null, null, null, false, true);
+
+        enforcer.createRuntimeSchema();
+        assertTrue(enforcer.areDynamicFieldsInitialized());
+
+        Schema actualRuntimeSchema = enforcer.getRuntimeSchema();
+        assertEquals(expectedRuntimeSchema, actualRuntimeSchema);
+    }
+
     /**
      * Checks {@link IncomingSchemaEnforcer#put()} converts string value to date according pattern specified in dynamic field
      * TODO (iv.gonchar): this is incorrect behavior, because avro record should not contain java.util.Date value. It should store

main/plugins/org.talend.designer.components.libs/libs_src/talend-db-exasol-util/pom.xml

@@ -45,9 +45,9 @@
             <scope>test</scope>
         </dependency>
 		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>1.2.17</version>
+			<groupId>ch.qos.reload4j</groupId>
+			<artifactId>reload4j</artifactId>
+			<version>1.2.19</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>

main/plugins/org.talend.designer.components.libs/libs_src/talend-httputil/pom.xml

@@ -4,7 +4,7 @@
 	<groupId>org.talend</groupId>
 	<artifactId>talend-httputil</artifactId>
 	<name>talend-httputil</name>
-	<version>1.0.5</version>
+	<version>1.0.6</version>
 
 	<properties>
 		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
@@ -20,7 +20,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.10.1</version>
+            <version>2.11.4</version>
         </dependency>
         
         <dependency>

main/plugins/org.talend.designer.components.libs/libs_src/talend-job-monitoring/pom.xml

@@ -69,13 +69,13 @@
 		<dependency>
 			<groupId>org.apache.logging.log4j</groupId>
 			<artifactId>log4j-api</artifactId>
-			<version>2.12.1</version>
+			<version>2.17.1</version>
 			<optional>true</optional>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.logging.log4j</groupId>
 			<artifactId>log4j-core</artifactId>
-			<version>2.12.1</version>
+			<version>2.17.1</version>
 			<optional>true</optional>
 		</dependency>
 	</dependencies>
@@ -97,4 +97,4 @@
 		</pluginManagement>
 	</build>
 
-</project>
+</project>

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/pom.xml

@@ -1,72 +1,110 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<groupId>org.talend.libraries</groupId>
-	<artifactId>talend-mscrm</artifactId>
-	<version>3.4-20191012</version>
-	<packaging>jar</packaging>
-	
-	<name>talend-mscrm</name>
-	
-	<properties>
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<cxf.version>3.1.1</cxf.version>
-		<odata.version>4.3.0</odata.version>
-        <slf4j.version>1.7.12</slf4j.version>
-		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
-	</properties>
-	
-	<distributionManagement>
-		<snapshotRepository>
-			<id>talend_nexus_deployment</id>
-			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
-			<snapshots>
-				<enabled>true</enabled>
-			</snapshots>
-			<releases>
-				<enabled>false</enabled>
-			</releases>
-		</snapshotRepository>
-		<repository>
-			<id>talend_nexus_deployment</id>
-			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
-			<snapshots>
-				<enabled>false</enabled>
-			</snapshots>
-			<releases>
-				<enabled>true</enabled>
-			</releases>
-		</repository>
-	</distributionManagement>
-	
-	<dependencies>
-		<dependency>
-			<groupId>org.apache.axis2</groupId>
-			<artifactId>axis2-xmlbeans</artifactId>
-			<version>1.7.4</version>
-		</dependency>
-		<dependency>
-    		<groupId>log4j</groupId>
-    		<artifactId>log4j</artifactId>
-    		<version>1.2.5</version>
-		</dependency>
-		<dependency>
-    		<groupId>org.apache.httpcomponents</groupId>
-    		<artifactId>httpclient</artifactId>
-    		<version>4.5.2</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.httpcomponents</groupId>
-			<artifactId>httpcore</artifactId>
-			<version>4.4.9</version>
-		</dependency>
-		<dependency>
-    		<groupId>org.apache.axis2</groupId>
-    		<artifactId>axis2-transport-http</artifactId>
-    		<version>1.7.4</version>
-		</dependency>
-		
-		<dependency>
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.talend.components</groupId>
+    <artifactId>talend-mscrm</artifactId>
+    <version>3.10.2-20220831</version>
+    <packaging>jar</packaging>
+    <description>A forked Talend-MSCRM library, developed to use in Studio 7.3.1 only. It's main intention is to contain CVE fixes.</description>
+
+    <name>talend-mscrm</name>
+
+    <licenses>
+        <license>
+            <name>Apache License, Version 2.0</name>
+            <url>https://www.talendforge.org/modules/licenses/APACHE_v2.txt</url>
+            <distribution>may be downloaded from the Maven repository</distribution>
+        </license>
+    </licenses>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
+
+        <odata.version>4.7.1</odata.version>
+        <slf4j.version>1.7.28</slf4j.version>
+        <axis2.version>1.7.9</axis2.version>
+        <httpclient.version>4.5.13</httpclient.version>
+        <httpcore.version>4.4.13</httpcore.version>
+        <junit.version>4.13.2</junit.version>
+        <adal4j.version>1.6.5-20220701</adal4j.version>
+        <jackson.version>2.13.4</jackson.version>
+        <jackson-databind.version>2.13.4.2</jackson-databind.version>
+
+        <!-- plugin's versions -->
+        <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>
+        <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+    </properties>
+    
+    <repositories>
+        <repository>
+            <id>talend_nexus_deployment</id>
+            <url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+    </repositories>
+    <distributionManagement>
+        <snapshotRepository>
+            <id>talend_nexus_deployment</id>
+            <url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+        </snapshotRepository>
+        <repository>
+            <id>talend_nexus_deployment</id>
+            <url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+    </distributionManagement>
+    
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-xmlbeans</artifactId>
+            <version>${axis2.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.reload4j</groupId>
+            <artifactId>reload4j</artifactId>
+            <version>1.2.19</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>${httpclient.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+            <version>${httpcore.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-transport-http</artifactId>
+            <version>${axis2.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-httpclient</groupId>
+                    <artifactId>commons-httpclient</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        
+        <dependency>
             <groupId>org.apache.olingo</groupId>
             <artifactId>odata-client-core</artifactId>
             <version>${odata.version}</version>
@@ -98,53 +136,55 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
-            <version>1.7.11</version>
+            <version>${slf4j.version}</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.12</version>
+            <version>${junit.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
-	    <groupId>com.microsoft.azure</groupId>
+            <groupId>com.microsoft.azure</groupId>
             <artifactId>adal4j</artifactId>
-            <version>1.1.1-20191012</version>
+            <version>${adal4j.version}</version>
         </dependency>
-	</dependencies>
-	<build>
-		<resources>
-			<resource>
-				<directory>src/main/java</directory>
-			</resource>
-			<resource>
-				<directory>src/main/resources</directory>
-			</resource>
-		</resources>
-		<plugins>
-			<plugin>
-				<artifactId>maven-jar-plugin</artifactId>
-				<executions>
-					<execution>
-						<id>default-jar</id>
-						<phase>package</phase>
-						<goals>
-							<goal>jar</goal>
-						</goals>
-						<configuration>
-							<outputDirectory>../../../org.talend.libraries.crm/lib</outputDirectory>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>${jackson-databind.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/java</directory>
+            </resource>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
+        <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
+                <version>${maven-compiler-plugin.version}</version>
                 <configuration>
                     <source>1.8</source>
                     <target>1.8</target>
                 </configuration>
             </plugin>
         </plugins>
-	</build>
+    </build>
 </project>

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/MSCRMClient.java

@@ -1,8 +1,11 @@
 package org.talend.ms.crm;
 
 import java.rmi.RemoteException;
+import java.util.Iterator;
 import java.util.UUID;
 
+import javax.naming.AuthenticationException;
+import javax.naming.ServiceUnavailableException;
 import javax.xml.stream.XMLStreamException;
 
 import org.apache.axiom.om.OMAbstractFactory;
@@ -19,33 +22,37 @@ import org.apache.axis2.client.ServiceClient;
 import org.apache.axis2.context.ConfigurationContext;
 import org.apache.axis2.context.ConfigurationContextFactory;
 import org.apache.axis2.transport.http.HTTPConstants;
-import org.apache.axis2.transport.http.HTTPTransportConstants;
 import org.apache.axis2.transport.http.HttpTransportProperties;
 import org.apache.axis2.transport.http.HttpTransportProperties.ProxyProperties;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.olingo.client.api.http.HttpClientException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.datacontract.schemas._2004._07.system_collections_generic.KeyValuePairOfEndpointTypestringztYlk6OT;
+import org.talend.ms.crm.odata.ClientConfiguration;
+import org.talend.ms.crm.odata.authentication.AuthStrategyFactory;
+import org.talend.ms.crm.odata.authentication.IAuthStrategy;
+import org.talend.ms.crm.sdk.Instance;
 import org.talend.ms.crm.sdk.OnlineAuthenticationPolicy;
 import org.talend.ms.crm.sdk.OrganizationServiceStubWrapper;
 import org.talend.ms.crm.sdk.RequestDateTimeData;
 import org.talend.ms.crm.sdk.SecurityData;
 import org.talend.ms.crm.sdk.WsdlTokenManager;
 
-import com.microsoft.schemas.xrm._2011.contracts.DiscoveryServiceStub;
-import com.microsoft.schemas.xrm._2011.contracts.IDiscoveryService_Execute_DiscoveryServiceFaultFault_FaultMessage;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.microsoft.schemas.xrm._2011.contracts.OrganizationServiceStub;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.EndpointType;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.ExecuteDocument;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.ExecuteDocument.Execute;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.ExecuteResponseDocument;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.ExecuteResponseDocument.ExecuteResponse;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.OrganizationDetail;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.RetrieveOrganizationRequest;
-import com.microsoft.schemas.xrm._2011.contracts.discovery.RetrieveOrganizationResponse;
 
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -64,21 +71,15 @@ public class MSCRMClient {
 
     static Logger logger = LoggerFactory.getLogger(MSCRMClient.class.getName());
 
-    /**
-     * Microsoft account (e.g. youremail@live.com) or Microsoft Office 365 (Org ID e.g.
-     * youremail@yourorg.onmicrosoft.com) User Name.
-     */
-    private String username;
-
-    /**
-     * Microsoft account or Microsoft Office 365 (Org ID) Password.
-     */
-    private String password;
-
     /**
      * Unique Name of the organization
      */
     private String orgName;
+    
+    /**
+     * Unique Name of the organization
+     */
+    private ClientConfiguration clientConfiguration;
 
     /**
      * Suffix for the Flat WSDL
@@ -87,88 +88,84 @@ public class MSCRMClient {
 
     private OrganizationServiceStub serviceStub;
 
-    private Integer timeout;
+	private String discoveryServiceURL = "https://globaldisco.crm.dynamics.com/api/discovery/v2.0/Instances";
+	
+	/**
+     * This sample application registration values are available for all online instances
+     * This is suggested to use for development and prototyping purposes.For production use, you should create an AppId or ClientId that is specific 
+     * to your tenant in the Azure Management portal.
+     * 
+     */
+    private static final String DEFAULT_CLIENT_ID = "51f81489-12ee-4a9e-aaae-a2591f45987d";
+	
+	private IAuthStrategy authStrategy;
 
-    private Boolean reuseHttpClient;
+    public MSCRMClient(ClientConfiguration clientConfiguration, String orgName, String discoveryServiceURL) throws AuthenticationException {
+    	this.clientConfiguration = clientConfiguration;
+    	this.orgName = orgName;
+        this.discoveryServiceURL  = discoveryServiceURL;
+        init();
 
-    private int maxConnectionRetries = 5;
+    }
+    
+    private void init() throws AuthenticationException {
+        if (clientConfiguration != null && discoveryServiceURL != null && discoveryServiceURL.indexOf("/api/discovery/") > 0) {
+            clientConfiguration.setResource(discoveryServiceURL.substring(0, discoveryServiceURL.indexOf("/api/discovery/")));
+        }
+        
+        if(clientConfiguration != null) {
+        	if (discoveryServiceURL != null && discoveryServiceURL.indexOf("/api/discovery/") > 0) {
+                clientConfiguration.setResource(discoveryServiceURL.substring(0, discoveryServiceURL.indexOf("/api/discovery/")));
+            }
+        	if (clientConfiguration.getClientId()==null || clientConfiguration.getClientId().isEmpty()) {
+                clientConfiguration.setClientId(DEFAULT_CLIENT_ID);
+            }
+        }
 
-    private int attemptsInterval = 1000;
-
-    public MSCRMClient(String username, String password, String orgName) {
-        this.username = username;
-        this.password = password;
-        this.orgName = orgName;
+        authStrategy = AuthStrategyFactory.createAuthStrategy(this.clientConfiguration);
+        authStrategy.init();
 
     }
 
-    public void setTimeout(Integer timeout) {
-        this.timeout = timeout;
-    }
 
-    public void setReuseHttpClient(Boolean reuseHttpClient) {
-        this.reuseHttpClient = reuseHttpClient;
-    }
-
-    public void setMaxConnectionRetries(int maxConnectionRetries) {
-        this.maxConnectionRetries = maxConnectionRetries;
-    }
-
-    public void setAttemptsInterval(int attemptsInterval) {
-        this.attemptsInterval = attemptsInterval;
-    }
-
-    public OrganizationServiceStub getOnlineConnection(String discoveryServiceURL) throws Exception {
-        return new OrganizationServiceStubWrapper(doGetOnlineConnection(discoveryServiceURL), this, discoveryServiceURL,
-                maxConnectionRetries, attemptsInterval);
+    public OrganizationServiceStub getOnlineConnection() throws Exception {
+        return new OrganizationServiceStubWrapper(doGetOnlineConnection(), this, discoveryServiceURL,
+        		clientConfiguration.getMaxRetryTimes(), clientConfiguration.getIntervalTime());
     }
 
     /**
-     * URL for the Discovery Service For North America Microsoft account, discovery service url is
-     * https://dev.crm.dynamics.com/XRMServices/2011/Discovery.svc Microsoft office 365, discovery service url is
-     * https://disco.crm.dynamics.com/XRMServices/2011/Discovery.svc To use appropriate discovery service url for other
+     * Organization information is stored in the Instance table of the Discovery Service. To see the kind of information contained in that table, 
+     * send an HTTP GET request to the service for one of your instances.
      * environments refer http://technet.microsoft.com/en-us/library/gg309401.aspx
      *
      * @throws Exception
      */
-    public OrganizationServiceStub doGetOnlineConnection(String discoveryServiceURL) throws Exception {
+    public OrganizationServiceStub doGetOnlineConnection() throws Exception {
 
-        try {
-            // Retrieve the authentication policy for the discovery service.
-            OnlineAuthenticationPolicy discoveryPolicy = new OnlineAuthenticationPolicy(discoveryServiceURL + FlatWSDLSuffix);
-            WsdlTokenManager discoeryTokenManager = new WsdlTokenManager();
-            // Authenticate the user using the discovery authentication policy.
-            SecurityData discoverySecurityData = discoeryTokenManager.authenticate(discoveryServiceURL, username, password,
-                    discoveryPolicy.getAppliesTo(), discoveryPolicy.getPolicy(), discoveryPolicy.getIssuerUri());
-            // Retrieve discovery stub using organization URL with the security data.
-            DiscoveryServiceStub discoveryServiceStub = createDiscoveryServiceStub(discoveryServiceURL, discoverySecurityData);
-            // Retrieve organization service url using discovery stub.
-            String orgUrl = discoverOrganizationUrl(discoveryServiceStub, orgName);
-            // The discovery service stub cannot be reused against the organization service
-            // as the Issuer and AppliesTo may differ between the discovery and organization services.
-            // Retrieve the authentication policy for the organization service.
-            OnlineAuthenticationPolicy organizationPolicy = new OnlineAuthenticationPolicy(orgUrl + FlatWSDLSuffix);
-            WsdlTokenManager orgTokenManager = new WsdlTokenManager();
-            // Authenticate the user using the organization authentication policy.
-            SecurityData securityData = orgTokenManager.authenticate(orgUrl, username, password,
-                    organizationPolicy.getAppliesTo(), organizationPolicy.getPolicy(), organizationPolicy.getIssuerUri());
+    	String serviceURL = discoveryServiceURL;
+    	if(!discoveryServiceURL.contains("(")) {
+    		serviceURL =  discoveryServiceURL+"(UniqueName='"+orgName+"')";
+    	}
+		
+		String orgUrl = getOrgURL( serviceURL);
+		// The discovery service stub cannot be reused against the organization service
+		// as the Issuer and AppliesTo may differ between the discovery and organization services.
+		// Retrieve the authentication policy for the organization service.
+		OnlineAuthenticationPolicy organizationPolicy = new OnlineAuthenticationPolicy(orgUrl + FlatWSDLSuffix);
+		WsdlTokenManager orgTokenManager = new WsdlTokenManager();
+		// Authenticate the user using the organization authentication policy.
+		SecurityData securityData = orgTokenManager.authenticate(orgUrl, clientConfiguration.getUserName(), clientConfiguration.getPassword(),
+		        organizationPolicy.getAppliesTo(), organizationPolicy.getPolicy(), organizationPolicy.getIssuerUri());
 
-            // Retrieve organization stub using organization URL with the security data.
-            serviceStub = createOrganizationServiceStub(orgUrl, securityData);
+		// Retrieve organization stub using organization URL with the security data.
+		serviceStub = createOrganizationServiceStub(orgUrl, securityData);
 
-            Options options = serviceStub._getServiceClient().getOptions();
-            if (reuseHttpClient != null) {
-                options.setProperty(org.apache.axis2.transport.http.HTTPConstants.REUSE_HTTP_CLIENT, reuseHttpClient);
-            }
-            if (timeout != null) {
-                options.setTimeOutInMilliSeconds(Long.valueOf(timeout));
-                options.setProperty(org.apache.axis2.transport.http.HTTPConstants.SO_TIMEOUT, timeout);
-                options.setProperty(org.apache.axis2.transport.http.HTTPConstants.CONNECTION_TIMEOUT, timeout);
-            }
-
-        } catch (IDiscoveryService_Execute_DiscoveryServiceFaultFault_FaultMessage e) {
-            throw new Exception(e.getFaultMessage().getDiscoveryServiceFault().getMessage());
-        }
+		Options options = serviceStub._getServiceClient().getOptions();
+		
+	    options.setProperty(org.apache.axis2.transport.http.HTTPConstants.REUSE_HTTP_CLIENT, clientConfiguration.isReuseHttpClient());
+	    options.setTimeOutInMilliSeconds(Long.valueOf(clientConfiguration.getTimeout()));
+	    options.setProperty(org.apache.axis2.transport.http.HTTPConstants.SO_TIMEOUT, clientConfiguration.getTimeout());
+	    options.setProperty(org.apache.axis2.transport.http.HTTPConstants.CONNECTION_TIMEOUT, clientConfiguration.getTimeout());
         return serviceStub;
 
     }
@@ -185,18 +182,6 @@ public class MSCRMClient {
         }
     }
 
-    private static DiscoveryServiceStub createDiscoveryServiceStub(String discoveryServiceURL, SecurityData securityData)
-            throws RemoteException, XMLStreamException {
-        try {
-            DiscoveryServiceStub stub = new DiscoveryServiceStub(getConfigurationContext(), discoveryServiceURL);
-            setServiceClientOptions(stub._getServiceClient(), securityData);
-            return stub;
-        } catch (RemoteException e) {
-            logger.error(e.getMessage());
-            throw e;
-        }
-    }
-
     private static void setServiceClientOptions(ServiceClient sc, SecurityData securityData) throws AxisFault, XMLStreamException {
         Options options = sc.getOptions();
 
@@ -306,45 +291,6 @@ public class MSCRMClient {
         return ctx;
     }
 
-    private static String discoverOrganizationUrl(DiscoveryServiceStub serviceStub, String organizationUniqueName)
-            throws RemoteException, IDiscoveryService_Execute_DiscoveryServiceFaultFault_FaultMessage {
-        try {
-            RetrieveOrganizationRequest request = RetrieveOrganizationRequest.Factory.newInstance();
-
-            request.setUniqueName(organizationUniqueName);
-
-            Execute exe = Execute.Factory.newInstance();
-            exe.setRequest(request);
-
-            ExecuteDocument exeDoc = ExecuteDocument.Factory.newInstance();
-            exeDoc.setExecute(exe);
-
-            ExecuteResponseDocument executeRespDoc = serviceStub.execute(exeDoc);
-            ExecuteResponse executeResp = executeRespDoc.getExecuteResponse();
-
-            RetrieveOrganizationResponse result = (RetrieveOrganizationResponse) executeResp.getExecuteResult();
-
-            OrganizationDetail orgDetail = result.getDetail();
-
-            KeyValuePairOfEndpointTypestringztYlk6OT[] keyValuePairs = orgDetail.getEndpoints()
-                    .getKeyValuePairOfEndpointTypestringztYlk6OTArray();
-
-            for (KeyValuePairOfEndpointTypestringztYlk6OT keyValuePair : keyValuePairs) {
-
-                if (keyValuePair.getKey() == EndpointType.ORGANIZATION_SERVICE) {
-                    return keyValuePair.getValue();
-                }
-            }
-        } catch (RemoteException e) {
-            logger.error(e.getMessage());
-            throw e;
-        } catch (IDiscoveryService_Execute_DiscoveryServiceFaultFault_FaultMessage e) {
-            logger.error(e.getMessage());
-            throw e;
-        }
-        return null;
-    }
-
     private static HttpTransportProperties.ProxyProperties getProxyProperties() {
         String proxyHost = null;
         String proxyPort = null;
@@ -370,5 +316,55 @@ public class MSCRMClient {
         }
         return proxyProps;
     }
+    
+    private String getOrgURL(String discoveryServiceURL) throws ServiceUnavailableException {
+        try {
+            HttpGet request = new HttpGet(discoveryServiceURL);
+    		
+            // https://docs.microsoft.com/en-us/powerapps/developer/data-platform/webapi/discover-url-organization-web-api#authentication
+            authStrategy.configureRequest(request);
 
+            try (CloseableHttpClient httpClient = HttpClients.createDefault();
+                 CloseableHttpResponse response = httpClient.execute(request)) {
+            	
+            	int statusCode = response.getStatusLine().getStatusCode();
+            	if(statusCode != HttpStatus.SC_NO_CONTENT && statusCode != HttpStatus.SC_CREATED && statusCode != HttpStatus.SC_OK) {
+                    String message = null;
+                    if (statusCode == HttpStatus.SC_NOT_FOUND ) {
+                        message = "The organization '"+orgName+"' does not exist.";
+                    } else {
+                        message = response.getStatusLine().getReasonPhrase();
+                    }
+                    throw new HttpClientException(message);
+            	}
+
+                HttpEntity entity = response.getEntity();
+                String orgUrl = null;
+                if (entity != null) {
+                	String result = EntityUtils.toString(entity);
+                    ObjectMapper mapper = new ObjectMapper();
+                    mapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true);
+                    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+                    JsonNode tree = mapper.readTree(result);
+                    Iterator<JsonNode> iter = tree.path("value").elements();
+                    while (iter.hasNext()){
+                        JsonNode node = iter.next();
+                        Instance instance = mapper.readValue(node.toString(), Instance.class);
+                        // Should only return one instance.
+                        if(orgName.equals(instance.getUniqueName())) {
+                        	orgUrl = instance.getApiUrl() + "/XRMServices/2011/Organization.svc";
+                        	break;
+                        }
+                    }
+                }
+                if(orgUrl == null) {
+                	throw new HttpClientException("No organization available.");
+                }
+                return orgUrl;
+            }
+        } catch (Exception e) {
+            throw new ServiceUnavailableException(e.getMessage());
+        } 
+	}
+    
 }

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/ClientConfiguration.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -12,12 +12,14 @@
 // ============================================================================
 package org.talend.ms.crm.odata;
 
+import java.util.Map;
+
 public final class ClientConfiguration {
 
     /*
      * Implemented authentication strategies for OData/MS CRM.
      */
-    public static enum AuthStrategyEnum {NTLM, OAUTH};
+    public static enum AuthStrategyEnum {NTLM, OAUTH, OAUTH_PREMISE};
 
     /*
      * Kind of registered app on azure
@@ -69,6 +71,16 @@ public final class ClientConfiguration {
      */
     private String authoryEndpoint;
 
+    /*
+     * The redirect URL
+     */
+    private String redirectURL;
+
+    /*
+     * The service API to retrieve the resource we ask for with oauth on-premise
+     */
+    private String serviceAPI;
+
     private int maxRetryTimes = 5;
 
     /*
@@ -87,6 +99,14 @@ public final class ClientConfiguration {
     private AppRegisteredType appRegisteredType = AppRegisteredType.NATIVE_APP;
     private WebAppPermission webAppPermission = WebAppPermission.DELEGATED;
 
+    /**
+     * The oauth resource is deduced from the "Service root URL" if this value is null.
+     */
+    private String forceResource;
+
+    private Map<String, String> headers;
+    private boolean returnRepresentation = false;
+
 
     ClientConfiguration(AuthStrategyEnum authStrategy) {
         this.authStrategy = authStrategy;
@@ -217,4 +237,44 @@ public final class ClientConfiguration {
         this.domain = domain;
     }
 
-}
+    public String getRedirectURL() {
+        return redirectURL;
+    }
+
+    public void setRedirectURL(String redirectURL) {
+        this.redirectURL = redirectURL;
+    }
+
+    public String getServiceAPI() {
+        return serviceAPI;
+    }
+
+    public void setServiceAPI(String serviceAPI) {
+        this.serviceAPI = serviceAPI;
+    }
+
+    public String getForceResource() {
+        return forceResource;
+    }
+
+    public void setForceResource(String forceResource) {
+        this.forceResource = forceResource;
+    }
+
+    public void setCustomHeaders(Map<String, String> headers){
+        this.headers = headers;
+    }
+
+    public Map<String, String> getCustomHeaders(){
+        return this.headers;
+    }
+
+    public boolean isReturnRepresentation() {
+        return returnRepresentation;
+    }
+
+    public void setReturnRepresentation(boolean returnRepresentation) {
+        this.returnRepresentation = returnRepresentation;
+    }
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/ClientConfigurationFactory.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -16,13 +16,13 @@ import org.talend.ms.crm.odata.ClientConfiguration.AuthStrategyEnum;
 
 /**
  * Generate the ClientConfiguration according to wanted authentication.
- *
+ * <p>
  * Different authentications need different information.
  */
 public class ClientConfigurationFactory {
 
     public final static ClientConfiguration buildOAuthNativeClientConfiguration(String clientId, String userName, String password,
-            String authoryEndpoint) {
+                                                                                String authoryEndpoint) {
         ClientConfiguration clientConfiguration = new ClientConfiguration(AuthStrategyEnum.OAUTH);
         clientConfiguration.setAppRegisteredType(ClientConfiguration.AppRegisteredType.NATIVE_APP);
         clientConfiguration.setClientId(clientId);
@@ -34,7 +34,7 @@ public class ClientConfigurationFactory {
     }
 
     public final static ClientConfiguration buildOAuthWebClientConfiguration(String clientId, String clientSecret, String userName, String password,
-                                                                                String authoryEndpoint, ClientConfiguration.WebAppPermission permission) {
+                                                                             String authoryEndpoint, ClientConfiguration.WebAppPermission permission) {
         ClientConfiguration clientConfiguration = new ClientConfiguration(AuthStrategyEnum.OAUTH);
         clientConfiguration.setAppRegisteredType(ClientConfiguration.AppRegisteredType.WEB_APP);
         clientConfiguration.setWebAppPermission(permission);
@@ -49,7 +49,7 @@ public class ClientConfigurationFactory {
     }
 
     public final static ClientConfiguration buildNtlmClientConfiguration(String userName, String password, String workstation,
-            String domain) {
+                                                                         String domain) {
         ClientConfiguration clientConfiguration = new ClientConfiguration(AuthStrategyEnum.NTLM);
         clientConfiguration.setUserName(userName);
         clientConfiguration.setPassword(password);
@@ -59,4 +59,19 @@ public class ClientConfigurationFactory {
         return clientConfiguration;
     }
 
+    public final static ClientConfiguration buildOAuthPremiseClientConfiguration(String userName, String password, String authoryEndpoint,
+                                                                                 String serviceAPI, String clientId, String clientSecret, String redirectUrl, String forcedResource) {
+        ClientConfiguration clientConfiguration = new ClientConfiguration(AuthStrategyEnum.OAUTH_PREMISE);
+        clientConfiguration.setUserName(userName);
+        clientConfiguration.setPassword(password);
+        clientConfiguration.setAuthoryEndpoint(authoryEndpoint);
+        clientConfiguration.setClientId(clientId);
+        clientConfiguration.setClientSecret(clientSecret);
+        clientConfiguration.setRedirectURL(redirectUrl);
+        clientConfiguration.setServiceAPI(serviceAPI);
+        clientConfiguration.setForceResource(forcedResource);
+
+        return clientConfiguration;
+    }
+
 }

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/ConditionOperator.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/DynamicsCRMClient.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -12,21 +12,9 @@
 // ============================================================================
 package org.talend.ms.crm.odata;
 
-import java.io.ByteArrayOutputStream;
-import java.io.OutputStreamWriter;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import javax.naming.AuthenticationException;
-import javax.naming.ServiceUnavailableException;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -44,6 +32,7 @@ import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.util.EntityUtils;
 import org.apache.olingo.client.api.ODataClient;
 import org.apache.olingo.client.api.communication.ODataClientErrorException;
+import org.apache.olingo.client.api.communication.request.retrieve.EdmMetadataRequest;
 import org.apache.olingo.client.api.communication.request.retrieve.ODataEntitySetIteratorRequest;
 import org.apache.olingo.client.api.communication.request.retrieve.ODataEntitySetRequest;
 import org.apache.olingo.client.api.communication.response.ODataRetrieveResponse;
@@ -69,16 +58,33 @@ import org.talend.ms.crm.odata.httpclientfactory.DefaultHttpClientState;
 import org.talend.ms.crm.odata.httpclientfactory.IHttpClientFactoryObserver;
 import org.talend.ms.crm.odata.httpclientfactory.IHttpclientFactoryObservable;
 
+import javax.naming.AuthenticationException;
+import javax.naming.ServiceUnavailableException;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
 /**
  * Client for accessing Dynamics CRM Online using the Web API
- *
  */
 public class DynamicsCRMClient implements IHttpClientFactoryObserver {
 
     private static final String NAMESPAVE = "Microsoft.Dynamics.CRM";
-    private  static final String LOOKUP_NAME_PREFIX = "_";
-    private  static final String LOOKUP_NAME_SUFFIX = "_value";
-    private static final int LOOKUP_NAME_MINSIZE = LOOKUP_NAME_PREFIX.length()+LOOKUP_NAME_SUFFIX.length();
+    private static final String LOOKUP_NAME_PREFIX = "_";
+    private static final String LOOKUP_NAME_SUFFIX = "_value";
+    private static final int LOOKUP_NAME_MINSIZE = LOOKUP_NAME_PREFIX.length() + LOOKUP_NAME_SUFFIX.length();
 
     private ClientConfiguration clientConfiguration;
 
@@ -101,7 +107,7 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
 
     /**
      * A list which contains all navigation links to set to null.
-     *
+     * <p>
      * A distinct delete command must be sent to set a navigation property to null. It can't be done in the same
      * time that the entity update.
      */
@@ -152,10 +158,26 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
         return odataClient;
     }
 
+    private void addCustomHeaders(ODataEntitySetRequest<ClientEntitySet> request) {
+        if (this.clientConfiguration.getCustomHeaders() == null) {
+            return;
+        }
+        this.clientConfiguration.getCustomHeaders().entrySet().stream().forEach(e -> request.addCustomHeader(e.getKey(), e.getValue()));
+    }
+
+    private void addCustomHeader(HttpRequestBase request) {
+        if (this.clientConfiguration.getCustomHeaders() != null) {
+            this.clientConfiguration.getCustomHeaders().entrySet().stream().forEach(e -> request.addHeader(e.getKey(), e.getValue()));
+        }
+
+        if (this.clientConfiguration.isReturnRepresentation()) {
+            request.addHeader("Prefer", "return=representation");
+        }
+    }
+
     /**
      * Create EntitySet Iterator request
      *
-     * @param entirySet entirySet the EntitySet name which you want to retrieve records
      * @param queryOption
      * @return EntitySet iterator request
      */
@@ -164,6 +186,7 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
         if (queryOption.getReturnEntityProperties() != null) {
             uriBuilder.select(queryOption.getReturnEntityProperties());
         }
+
         if (queryOption.getTop() > 0) {
             uriBuilder.top(queryOption.getTop());
         }
@@ -173,21 +196,68 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
         if (!StringUtils.isEmpty(queryOption.getFilter())) {
             uriBuilder.filter(queryOption.getFilter());
         }
+        final List<String> expands = queryOption.getExpands();
+        if (expands.size() > 0) {
+            String[] expandArray = new String[expands.size()];
+            uriBuilder.expand(expands.toArray(expandArray));
+        }
+
         ODataEntitySetRequest<ClientEntitySet> request = odataClient.getRetrieveRequestFactory()
                 .getEntitySetRequest(uriBuilder.build());
 
         this.authStrategy.configureRequest(request);
 
+        if (expands.size() > 0) {
+            // odata-metadata=full set by olingo generates issue when expand entities
+            request.addCustomHeader("Accept", "application/json;odata-metadata=minimal");
+        }
+
+        addCustomHeaders(request);
+
         return request;
     }
 
+    public EdmMetadataRequest createMetadataRetrieveRequest() {
+        EdmMetadataRequest request = odataClient.getRetrieveRequestFactory()
+                .getMetadataRequest(serviceRootURL);
+        this.authStrategy.configureRequest(request);
+        return request;
+    }
+
+    public ODataEntitySetRequest<ClientEntitySet> createEntitySetRetrieveRequest() {
+        URIBuilder uriBuilder = odataClient.newURIBuilder(serviceRootURL).appendEntitySetSegment("EntityDefinitions").select("EntitySetName,LogicalName");
+        ODataEntitySetRequest<ClientEntitySet> entitySetRequest = odataClient.getRetrieveRequestFactory().getEntitySetRequest(uriBuilder.build());
+        this.authStrategy.configureRequest(entitySetRequest);
+
+        addCustomHeaders(entitySetRequest);
+
+        return entitySetRequest;
+    }
+
+    public ODataEntitySetRequest<ClientEntitySet> createEndpointsNamesRequest() {
+        URIBuilder uriBuilder = odataClient.newURIBuilder(serviceRootURL);
+        ODataEntitySetRequest<ClientEntitySet> entitySetRequest = odataClient.getRetrieveRequestFactory().getEntitySetRequest(uriBuilder.build());
+        this.authStrategy.configureRequest(entitySetRequest);
+
+        addCustomHeaders(entitySetRequest);
+
+        return entitySetRequest;
+    }
+
+    public ODataEntitySetRequest<ClientEntitySet> createRequest(URIBuilder uriBuilder) {
+        ODataEntitySetRequest<ClientEntitySet> entitySetRequest = odataClient.getRetrieveRequestFactory().getEntitySetRequest(uriBuilder.build());
+        this.authStrategy.configureRequest(entitySetRequest);
+
+        addCustomHeaders(entitySetRequest);
+
+        return entitySetRequest;
+    }
+
     /**
      * Retrieve records from EntitySet
      *
-     * @param entitySet the EntitySet name which you want to retrieve records
      * @param queryOption
      * @return the entity set iterator object
-     *
      * @throws ServiceUnavailableException
      */
     public ClientEntitySet retrieveEntities(QueryOptionConfig queryOption) throws AuthenticationException {
@@ -218,12 +288,10 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
     /**
      * Create entity
      *
-     * @param entitySet entitySet the EntitySet name which you want to create record
      * @param entity provided content for create
-     *
      * @throws ServiceUnavailableException
      */
-    public HttpResponse insertEntity(ClientEntity entity) throws ServiceUnavailableException {
+    public CreateUpdateResult insertEntity(ClientEntity entity) throws ServiceUnavailableException {
         URIBuilder insertURIBuilder = odataClient.newURIBuilder(serviceRootURL).appendEntitySetSegment(entitySet);
         HttpEntity httpEntity = convertToHttpEntity(entity);
         return createAndExecuteRequest(insertURIBuilder.build(), httpEntity, HttpMethod.POST);
@@ -233,27 +301,49 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
      * Update entity with provided content.
      * The PATCH method is used, so only given properties are updated.
      * Navigation link properties that must be set to null are updated by another DELETE calls.
+     * Navigation link properties are saved during {@link #addEntityNavigationLink(ClientEntity, String, String, String, boolean, boolean)}
+     * method call
      *
-     * @param entity The payload containing properties to update
+     * @param entity     The payload containing properties to update
      * @param keySegment The id of the entity to update
+     * @throws ServiceUnavailableException
+     * @deprecated use {@link #updateEntity(ClientEntity, String, List)} instead with a list of navigation links list instead;
+     */
+    @Deprecated
+    public CreateUpdateResult updateEntity(ClientEntity entity, String keySegment) throws ServiceUnavailableException {
+        CreateUpdateResult result = updateEntity(entity, keySegment, this.navigationLinksToNull);
+        this.navigationLinksToNull.clear();
+        return result;
+    }
+
+    /**
+     * Update entity with provided content.
+     * The PATCH method is used, so only given properties are updated.
+     * Navigation link properties that must be set to null are updated by another DELETE calls.
      *
+     * @param entity                  The payload containing properties to update
+     * @param keySegment              The id of the entity to update
+     * @param navigationLinksToDelete list of navigation link names to delete
      * @throws ServiceUnavailableException
      */
-    public HttpResponse updateEntity(ClientEntity entity, String keySegment) throws ServiceUnavailableException {
+    public CreateUpdateResult updateEntity(ClientEntity entity, String keySegment, List<String> navigationLinksToDelete) throws ServiceUnavailableException {
         URIBuilder updateURIBuilder = odataClient.newURIBuilder(serviceRootURL).appendEntitySetSegment(entitySet)
                 .appendKeySegment(UUID.fromString(keySegment));
         HttpEntity httpEntity = convertToHttpEntity(entity);
-        HttpResponse updateHttpResponse =  createAndExecuteRequest(updateURIBuilder.build(), httpEntity, HttpMethod.PATCH);
+        CreateUpdateResult result = createAndExecuteRequest(updateURIBuilder.build(), httpEntity, HttpMethod.PATCH);
 
         // No need to test the updateHttpResponse code since it is returned only if it's a success.
         // The deletion of navigation links will be done only if the previous update doesn't throw an exception.
-        this.deleteNavigationLinksToNull(keySegment);
+        this.deleteNavigationLinksToNull(keySegment, navigationLinksToDelete);
 
-        return updateHttpResponse;
+        return result;
     }
 
-    protected void deleteNavigationLinksToNull(String keySegment) throws ServiceUnavailableException {
-        for(String navigationLinkName : this.navigationLinksToNull){
+    protected void deleteNavigationLinksToNull(String keySegment, List<String> navigationLinksToNull) throws ServiceUnavailableException {
+        if (navigationLinksToNull == null || navigationLinksToNull.isEmpty()) {
+            return;
+        }
+        for (String navigationLinkName : navigationLinksToNull) {
             this.deleteNavigationLink(navigationLinkName, keySegment);
         }
     }
@@ -261,16 +351,14 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
     /**
      * Deleted entity by key
      *
-     * @param entitySet entitySet the EntitySet name which you want to delete records
      * @param keySegment Entity key segment
-     *
      * @throws ServiceUnavailableException
      */
     public HttpResponse deleteEntity(String keySegment) throws ServiceUnavailableException {
         URIBuilder deleteURIBuilder = odataClient.newURIBuilder(serviceRootURL).appendEntitySetSegment(entitySet)
                 .appendKeySegment(UUID.fromString(keySegment));
 
-        return createAndExecuteRequest(deleteURIBuilder.build(), null, HttpMethod.DELETE);
+        return createAndExecuteRequest(deleteURIBuilder.build(), null, HttpMethod.DELETE).getResponse();
     }
 
 
@@ -279,17 +367,17 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
      * Jira : TDI-39571
      *
      * @param navigationLinkName The navigation link name (not the _name_value generated lookup property)
-     * @param keySegment The keysegment(id) of the main property
+     * @param keySegment         The keysegment(id) of the main property
      * @return The Http response.
      * @throws ServiceUnavailableException
      */
     public HttpResponse deleteNavigationLink(String navigationLinkName, String keySegment) throws ServiceUnavailableException {
         URIBuilder deleteNavLinkURIBuilder = odataClient.newURIBuilder(serviceRootURL)
-                                                    .appendEntitySetSegment(entitySet)
-                                                    .appendKeySegment(UUID.fromString(keySegment))
-                                                    .appendNavigationSegment(navigationLinkName).appendRefSegment();
+                .appendEntitySetSegment(entitySet)
+                .appendKeySegment(UUID.fromString(keySegment))
+                .appendNavigationSegment(navigationLinkName).appendRefSegment();
 
-        return createAndExecuteRequest(deleteNavLinkURIBuilder.build(), null, HttpMethod.DELETE);
+        return createAndExecuteRequest(deleteNavLinkURIBuilder.build(), null, HttpMethod.DELETE).getResponse();
     }
 
     /**
@@ -349,24 +437,24 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
      *
      * @param entityType
      */
-    public void setEntityType(String entityType){
+    public void setEntityType(String entityType) {
         this.entityType = entityType;
     }
 
     public void addEntityNavigationLink(ClientEntity entity, String lookupEntitySet, String lookupName,
-            String linkedEntityId, boolean emptyLookupIntoNull, boolean ignoreNull) {
+                                        String linkedEntityId, boolean emptyLookupIntoNull, boolean ignoreNull) {
 
         // To help the final user since lookup names '_name_value' are available in the schema
         // But it's the navigation link that can be updated.
         String navigationLinkName = this.extractNavigationLinkName(lookupName);
 
         // If value is empty and emptyLookupIntoNull, then set the value to null to unlink the navigation (set to null)
-        if(emptyLookupIntoNull && linkedEntityId != null && linkedEntityId.isEmpty()){
+        if (emptyLookupIntoNull && linkedEntityId != null && linkedEntityId.isEmpty()) {
             linkedEntityId = null;
         }
 
         // If ignore null is set and the value is null, then don't update/delete this navigation link
-        if(ignoreNull && linkedEntityId == null){
+        if (ignoreNull && linkedEntityId == null) {
             return;
         }
 
@@ -377,14 +465,50 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
             } catch (URISyntaxException e) {
                 throw new HttpClientException(e);
             }
-        }
-        else{
+        } else {
             // Retains all navigation links to delete (set to null)
             navigationLinksToNull.add(navigationLinkName);
         }
     }
 
-    public int getNbNavigationLinkToRemove(){
+    /**
+     * Try to add navigation link to entity.
+     *
+     * @param navigationLinkName  extracted navigation link(see {@link #extractNavigationLinkName(String)})
+     * @param entity              to be updated
+     * @param lookupEntitySet     entity set referred by navigation link
+     * @param linkedEntityId      id of the referred entity
+     * @param emptyLookupIntoNull if empty lookup values should be converted to null values
+     * @param ignoreNull          if null values should be skipped
+     * @return true if the link was added or skipped, false if it should be deleted with a separate call.
+     */
+    public boolean addOrSkipEntityNavigationLink(ClientEntity entity, String lookupEntitySet, String navigationLinkName,
+                                                 String linkedEntityId, boolean emptyLookupIntoNull, boolean ignoreNull) {
+        // If value is empty and emptyLookupIntoNull, then set the value to null to unlink the navigation (set to null)
+        if (emptyLookupIntoNull && linkedEntityId != null && linkedEntityId.isEmpty()) {
+            linkedEntityId = null;
+        }
+
+        // If ignore null is set and the value is null, then don't update/delete this navigation link
+        if (ignoreNull && linkedEntityId == null) {
+            return true;
+        }
+
+        if (linkedEntityId != null) {
+            try {
+                entity.getNavigationLinks().add(odataClient.getObjectFactory().newEntityNavigationLink(navigationLinkName,
+                        new URI(lookupEntitySet + "(" + linkedEntityId + ")")));
+            } catch (URISyntaxException e) {
+                throw new HttpClientException(e);
+            }
+        } else {
+            // Retains all navigation links to delete (set to null)
+            return false;
+        }
+        return true;
+    }
+
+    public int getNbNavigationLinkToRemove() {
         return navigationLinksToNull.size();
     }
 
@@ -397,20 +521,20 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
      * @param lookupName The auto-generated lookup name
      * @return The extracted navigation link name or the lookup name if it can't be extracted.
      */
-    public String extractNavigationLinkName(String lookupName){
+    public String extractNavigationLinkName(String lookupName) {
         final int nameSize = lookupName.length();
-        if(nameSize <= LOOKUP_NAME_MINSIZE){
+        if (nameSize <= LOOKUP_NAME_MINSIZE) {
             return lookupName;
         }
 
         String pref = lookupName.substring(0, LOOKUP_NAME_PREFIX.length());
-        if(!pref.equals(LOOKUP_NAME_PREFIX)){
+        if (!pref.equals(LOOKUP_NAME_PREFIX)) {
             return lookupName;
         }
 
         final int endName = nameSize - LOOKUP_NAME_SUFFIX.length();
         String suff = lookupName.substring(endName, nameSize);
-        if(!suff.equals(LOOKUP_NAME_SUFFIX)){
+        if (!suff.equals(LOOKUP_NAME_SUFFIX)) {
             return lookupName;
         }
 
@@ -421,7 +545,6 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
      * Convert OData entity to HttpEntity type
      *
      * @param entity OData entity.
-     *
      * @return An entity that can be sent or received with an HTTP message.
      * @throws
      */
@@ -446,15 +569,14 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
     /**
      * Created and executes a request
      *
-     * @param uri the request URI
+     * @param uri        the request URI
      * @param httpEntity the entity to send.
-     * @param method HTTP method
-     *
+     * @param method     HTTP method
      * @return the response to the request.
      * @throws ServiceUnavailableException
      */
 
-    protected HttpResponse createAndExecuteRequest(URI uri, HttpEntity httpEntity, HttpMethod method)
+    protected CreateUpdateResult createAndExecuteRequest(URI uri, HttpEntity httpEntity, HttpMethod method)
             throws ServiceUnavailableException {
 
         boolean hasRetried = false;
@@ -476,11 +598,31 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
                 if (request instanceof HttpEntityEnclosingRequestBase) {
                     ((HttpEntityEnclosingRequestBase) request).setEntity(httpEntity);
                 }
+
+                addCustomHeader(request);
+
+                CreateUpdateResult result = new CreateUpdateResult();
                 HttpResponse response = httpClientState.getHttpClient().execute(request);
+                result.setResponse(response);
+
                 if (isResponseSuccess(response.getStatusLine().getStatusCode())) {
+                    if (this.clientConfiguration.isReturnRepresentation()) {
+                        final HttpEntity entity = response.getEntity();
+                        if (entity != null && entity.isStreaming()) {
+                            try(InputStream instream = entity.getContent()) {
+                                String e = new BufferedReader(new InputStreamReader(instream))
+                                        .lines().collect(Collectors.joining("\n"));
+                                result.setEntity(e);
+                                if (instream != null) {
+                                    instream.close();
+                                }
+                            }
+                        }
+                    } else {
+                        EntityUtils.consume(response.getEntity());
+                    }
                     request.releaseConnection();
-                    EntityUtils.consume(response.getEntity());
-                    return response;
+                    return result;
                 } else {
                     if (response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED && !hasRetried) {
                         this.authStrategy.refreshAuth();
@@ -509,7 +651,6 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
      * 201 create/update/delete entity success with return (not used in our component at present)
      *
      * @param statusCode HTTP status code
-     *
      * @return success or not(true or false)
      */
     public boolean isResponseSuccess(int statusCode) {
@@ -542,4 +683,30 @@ public class DynamicsCRMClient implements IHttpClientFactoryObserver {
 
     }
 
+    public final static class CreateUpdateResult {
+
+        private HttpResponse response;
+        private String entity;
+
+        public HttpResponse getResponse() {
+            return response;
+        }
+
+        public void setResponse(HttpResponse response) {
+            this.response = response;
+        }
+
+        public String getEntity() {
+            return entity;
+        }
+
+        public void setEntity(String entity) {
+            this.entity = entity;
+        }
+
+        public boolean hasEntity() {
+            return this.entity != null;
+        }
+    }
+
 }

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/LogicalOperator.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/ProxyProvider.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/QueryExpandUtil.java

@@ -0,0 +1,54 @@
+package org.talend.ms.crm.odata;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class QueryExpandUtil {
+
+    public static List<String> getExpandConfig(List<String> fields, List<String> params) {
+        if (fields == null || params == null || fields.size() != params.size()) {
+            throw new IllegalArgumentException(
+                    "Please make sure list of expands fields and params are not null and their size are same.");
+        }
+
+
+
+        if(fields.size() <= 0){
+            return Collections.emptyList();
+        }
+
+        List<String> expands = new ArrayList<>();
+        for(int i=0; i< fields.size(); i++){
+            String name = fields.get(i);
+            if("".equals(name.trim())){
+                continue;
+            }
+
+            name = getSanitizedName(name);
+            StringBuilder sb = new StringBuilder(name);
+            final String param = params.get(i);
+            if(!"".equals(param.trim())){
+                sb.append('(');
+                sb.append(param);
+                sb.append(')');
+            }
+            expands.add(sb.toString());
+        }
+
+        return expands;
+    }
+
+    /**
+     * expandable attrribute name are _entityid_value, should be transformed to entityid.
+     * @param name
+     * @return
+     */
+    public static String getSanitizedName(final String name){
+        String sanitized = ('_' == name.charAt(0)) ? name.substring(1) : name;
+        sanitized = (sanitized.endsWith("_value")) ? sanitized.substring(0, sanitized.length() - 6): sanitized;
+
+        return sanitized;
+    }
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/QueryOptionConfig.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -12,6 +12,9 @@
 // ============================================================================
 package org.talend.ms.crm.odata;
 
+import java.util.List;
+import java.util.ArrayList;
+
 public class QueryOptionConfig {
 
     private int top = -1;
@@ -24,6 +27,9 @@ public class QueryOptionConfig {
 
     private String filter;
 
+    private List<String> expand = new ArrayList<>();
+
+
     public int getTop() {
         return top;
     }
@@ -64,4 +70,8 @@ public class QueryOptionConfig {
         this.filter = filter;
     }
 
+    public void setExpands(List<String> expand) { this.expand = expand;}
+
+    public List<String> getExpands() { return this.expand; }
+
 }

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/AuthStrategyFactory.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -12,13 +12,11 @@
 // ============================================================================
 package org.talend.ms.crm.odata.authentication;
 
-import javax.naming.AuthenticationException;
-
 import org.talend.ms.crm.odata.ClientConfiguration;
 
 /**
  * Factory for OData authentication strategies.
- *
+ * <p>
  * No need to use singleton pattern since no implementation/extends, static is enough.
  */
 public final class AuthStrategyFactory {
@@ -26,15 +24,18 @@ public final class AuthStrategyFactory {
     private AuthStrategyFactory() {
     }
 
-    public final static IAuthStrategy createAuthStrategy(ClientConfiguration conf) {
+    public static IAuthStrategy createAuthStrategy(ClientConfiguration conf) {
         IAuthStrategy authStrategy = null;
         switch (conf.getAuthStrategy()) {
-        case OAUTH:
-            authStrategy = new OAuthStrategyImpl(conf);
-            break;
-        case NTLM:
-            authStrategy = new NTLMStrategyImpl(conf);
-            break;
+            case OAUTH:
+                authStrategy = new OAuthStrategyImpl(conf);
+                break;
+            case NTLM:
+                authStrategy = new NTLMStrategyImpl(conf);
+                break;
+            case OAUTH_PREMISE:
+                authStrategy = new OAuthPremiseStrategyImpl(conf);
+                break;
         }
 
         return authStrategy;

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/IAuthStrategy.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -31,7 +31,7 @@ public interface IAuthStrategy {
      * Called when DynamicsCRMClient is instanciated.
      *
      */
-    public void init() throws AuthenticationException;
+    void init() throws AuthenticationException;
 
     /**
      * Retrieve the HttpClientFactory.
@@ -41,21 +41,21 @@ public interface IAuthStrategy {
      * Called when DynamicsCRMClient is instanciated.
      *
      */
-    public IHttpclientFactoryObservable getHttpClientFactory() throws AuthenticationException;
+    IHttpclientFactoryObservable getHttpClientFactory() throws AuthenticationException;
 
     /**
      * Refresh Authentication if needed.
      */
-    public void refreshAuth() throws AuthenticationException;
+    void refreshAuth() throws AuthenticationException;
 
     /**
      * Configure request generated by odataClient.getRetrieveRequestFactory().
      */
-    public void configureRequest(ODataRequest request);
+    void configureRequest(ODataRequest request);
 
     /**
      * Configure request HttpRequestBase with POST/PATCH/DELETE method.
      */
-    public void configureRequest(HttpRequestBase request);
+    void configureRequest(HttpRequestBase request);
 
 }

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/NTLMStrategyImpl.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/OAuthPremiseStrategyImpl.java

@@ -0,0 +1,208 @@
+package org.talend.ms.crm.odata.authentication;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonIOException;
+import com.google.gson.JsonSyntaxException;
+import com.google.gson.stream.JsonReader;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.olingo.client.api.communication.request.ODataRequest;
+import org.apache.olingo.commons.api.http.HttpHeader;
+import org.talend.ms.crm.odata.ClientConfiguration;
+import org.talend.ms.crm.odata.authentication.httpclienthelper.HttpClient;
+import org.talend.ms.crm.odata.authentication.httpclienthelper.HttpResponse;
+import org.talend.ms.crm.odata.authentication.httpclienthelper.RequestHttpContext;
+import org.talend.ms.crm.odata.authentication.httpclienthelper.Token;
+import org.talend.ms.crm.odata.httpclientfactory.IHttpclientFactoryObservable;
+import org.talend.ms.crm.odata.httpclientfactory.OAuthHttpClientFactory;
+
+import javax.naming.AuthenticationException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class OAuthPremiseStrategyImpl implements IAuthStrategy {
+
+    private final static int MAX_REDIRECT_AUTH_CODE = 3;
+
+    private final ClientConfiguration conf;
+
+    private Token token;
+    private OAuthHttpClientFactory httpClientFactory;
+
+
+    OAuthPremiseStrategyImpl(ClientConfiguration conf) {
+        this.conf = conf;
+    }
+
+    @Override
+    public void init() throws AuthenticationException {
+        oauthFlow();
+    }
+
+    @Override
+    public IHttpclientFactoryObservable getHttpClientFactory() throws AuthenticationException {
+        oauthFlow();
+
+        if (httpClientFactory == null) {
+            httpClientFactory = new OAuthHttpClientFactory(this.conf);
+        }
+
+        return httpClientFactory;
+    }
+
+    @Override
+    public void refreshAuth() throws AuthenticationException {
+        oauthFlow();
+    }
+
+    @Override
+    public void configureRequest(ODataRequest request) {
+        request.addCustomHeader(HttpHeader.AUTHORIZATION, "Bearer " + token.getAccess_token());
+    }
+
+    @Override
+    public void configureRequest(HttpRequestBase request) {
+        request.addHeader(HttpHeader.AUTHORIZATION, "Bearer " + token.getAccess_token());
+    }
+
+    private void oauthFlow() throws AuthenticationException {
+        int retry = 0;
+
+        while (true) {
+            retry++;
+            try {
+
+                final Optional<String> code = retrieveAuthorizationCode();
+                if (code.isPresent()) {
+                    final String sCode = code.get();
+                    final Optional<Token> token = retrieveToken(sCode);
+                    if (token.isPresent()) {
+                        this.token = token.get();
+                        break;
+                    } else {
+                        throw new IllegalArgumentException("Can't retrieve oauth token, but no exception has been raised.");
+                    }
+                } else {
+                    throw new IllegalArgumentException("Can't retrieve authorization code, but no exception has been raised.");
+                }
+
+            } catch (IOException | InterruptedException | IllegalArgumentException e) {
+                if (retry < conf.getMaxRetryTimes()) {
+                    try {
+                        Thread.sleep(conf.getIntervalTime());
+                    } catch (InterruptedException e1) {
+                        // ignore
+                    }
+                    continue;
+                }
+
+                throw new AuthenticationException("Can't retrieve ms crm oauth token after '" + retry + "' retries : " + e.getMessage());
+            }
+        }
+    }
+
+    private Token json2Token(String json) throws JsonIOException, JsonSyntaxException {
+        Gson gson = new Gson();
+        JsonReader jsr = new JsonReader(new StringReader(json));
+        return gson.fromJson(jsr, Token.class);
+    }
+
+    private String getResource() throws MalformedURLException {
+        if(conf.getForceResource() != null && !"".equals(conf.getForceResource().trim())){
+            return conf.getForceResource();
+        }
+
+        URL url = new URL(conf.getServiceAPI());
+        final int port = url.getPort();
+        StringBuilder sp = new StringBuilder();
+        if (port > -1) {
+            sp.append(":").append(port);
+        }
+        return url.getProtocol() + "://" + url.getHost() + sp.toString();
+    }
+
+    /**
+     * To compute token url, change last segment by token.
+     * @return
+     */
+    private String computeTokenUrl(String url){
+        while(url.endsWith("/")) {
+            url = url.substring(0, url.length() - 1);
+        }
+        String token = url.substring(0, url.lastIndexOf('/')) + "/token";
+        return token;
+    }
+
+    private Optional<Token> retrieveToken(String code) throws IOException, InterruptedException, IllegalArgumentException {
+        Map<String, String> body = new HashMap<>();
+        body.put("grant_type", "authorization_code");
+        body.put("code", code);
+        body.put("redirect_uri", conf.getRedirectURL());
+        body.put("client_id", conf.getClientId());
+
+        if(conf.getClientSecret() != null && !"".equals(conf.getClientSecret().trim())){
+            body.put("client_secret", conf.getClientSecret());
+        }
+
+        RequestHttpContext queryContext = new RequestHttpContext("POST",
+                computeTokenUrl(conf.getAuthoryEndpoint()),
+                Collections.emptyMap(),
+                new HashMap<>(),
+                true,
+                conf.getTimeout() * 1000,
+                conf.getTimeout() * 1000);
+
+        queryContext.setBodyContent(body);
+
+        HttpClient client = new HttpClient(queryContext);
+
+        // Redirect are followed by the java http client
+        final HttpResponse call = client.call(new AtomicInteger(-1), e -> true);
+
+        Token token = null;
+        try {
+            token = json2Token(call.getBody());
+        }
+        catch (JsonIOException | JsonSyntaxException e){
+            throw new IllegalArgumentException("Can't parse retrieve ms crm oauth token : " + e.getMessage(), e);
+        }
+        return Optional.ofNullable(token);
+    }
+
+    private Optional<String> retrieveAuthorizationCode() throws IOException, InterruptedException {
+        Map<String, String> params = new HashMap();
+        params.put("resource", getResource());
+        params.put("response_type", "code");
+        params.put("state", "");
+        params.put("client_id", conf.getClientId());
+        params.put("scope", "");
+        params.put("redirect_uri", conf.getRedirectURL());
+
+        Map<String, String> form = new HashMap<>();
+        form.put("UserName", conf.getUserName());
+        form.put("Password", conf.getPassword());
+        form.put("AuthMethod", "FormsAuthentication");
+
+        RequestHttpContext queryContext = new RequestHttpContext("POST",
+                conf.getAuthoryEndpoint(),
+                params,
+                new HashMap<>(),
+                false,
+                conf.getTimeout() * 1000,
+                conf.getTimeout() * 1000);
+
+        queryContext.setBodyContent(form);
+        HttpClient client = new HttpClient(queryContext);
+
+        // We stop to follow redirect url once we have the code
+        final HttpResponse call = client.call(new AtomicInteger(MAX_REDIRECT_AUTH_CODE), e -> !e.extractCode().isPresent());
+
+        return call.extractCode();
+    }
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/OAuthStrategyImpl.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/httpclienthelper/HttpClient.java

@@ -0,0 +1,127 @@
+package org.talend.ms.crm.odata.authentication.httpclienthelper;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public final class HttpClient {
+
+    private final static int TEMPORARY_REDIRECT = 307;
+    private final static int PERMANENT_REDIRECT = 308;
+
+    private final RequestHttpContext requestHttpContext;
+
+    public HttpClient(RequestHttpContext requestHttpContext) {
+        this.requestHttpContext = requestHttpContext;
+    }
+
+    public static String encodeValue(String value) throws UnsupportedEncodingException {
+        return URLEncoder.encode(value, StandardCharsets.UTF_8.toString());
+    }
+
+    public HttpResponse call(AtomicInteger atomicInteger, Function<HttpResponse, Boolean> acceptRedirect) throws IOException, InterruptedException {
+        final HttpURLConnection conn = buildUrl(requestHttpContext);
+
+        if ("POST".equals(conn.getRequestMethod())) {
+            conn.setDoOutput(true);
+            try (DataOutputStream dos = new DataOutputStream(conn.getOutputStream())) {
+                final byte[] form = requestHttpContext.getBodyContent().getBytes(StandardCharsets.UTF_8);
+                dos.write(form);
+                dos.flush();
+            }
+        } else {
+            conn.connect();
+        }
+
+        final HttpResponse respContext = HttpResponse.fromHttpUrlConnection(conn);
+        return followRedirect(requestHttpContext, respContext, atomicInteger, acceptRedirect);
+    }
+
+    private HttpResponse followRedirect(RequestHttpContext queryContext, HttpResponse response, AtomicInteger nbRedirect, Function<HttpResponse, Boolean> acceptRedirect) throws IOException, InterruptedException {
+        final int nbR = nbRedirect.decrementAndGet();
+        final int status = response.getStatus();
+
+        boolean redirect = false;
+        if (status != HttpURLConnection.HTTP_OK && ((status == HttpURLConnection.HTTP_MOVED_TEMP
+                || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER
+                || status == TEMPORARY_REDIRECT || status == PERMANENT_REDIRECT))) {
+            redirect = true;
+        }
+
+        final Optional<String> location = response.getFirstValueHeader("location");
+
+        if (!redirect || !location.isPresent() || nbR <= 0) {
+            return response;
+        }
+
+        queryContext.setBase(location.get());
+        queryContext.setParams(Collections.emptyMap());
+
+        final List<String> cookie = response.getAllValuesHeader("Set-Cookie").orElse(Collections.emptyList());
+        if (cookie.size() > 0) {
+            final String collect = String.join("; ", cookie); //cookie.stream().collect(Collectors.joining("; "));
+            queryContext.getHeaders().put("Cookie", collect);
+        }
+
+        Boolean continueRedirect = acceptRedirect.apply(response);
+        if (!continueRedirect) {
+            return response;
+        }
+
+        HttpClient manager = new HttpClient(queryContext);
+        final HttpResponse redirectResponseContext = manager.call(nbRedirect, acceptRedirect);
+
+
+        return redirectResponseContext;
+    }
+
+    private HttpURLConnection buildUrl(RequestHttpContext context) throws IOException {
+        StringBuilder sb = new StringBuilder(context.getBase());
+
+        if (context.getParams().size() > 0) {
+            sb.append("?");
+
+            boolean first = true;
+            for (Map.Entry<String, String> e : context.getParams().entrySet()) {
+                if (!first) {
+                    sb.append("&");
+                }
+                first = false;
+                sb.append(e.getKey());
+                sb.append("=");
+                sb.append(encodeValue(e.getValue()));
+            }
+        }
+
+        URL url = new URL(sb.toString());
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+
+        if (context.getHeaders().size() > 0) {
+            for (Map.Entry<String, String> e : context.getHeaders().entrySet()) {
+                conn.setRequestProperty(e.getKey(), e.getValue());
+            }
+        }
+
+        conn.setRequestMethod(context.getMethod());
+        conn.setInstanceFollowRedirects(context.isFollowRedirects());
+        conn.setConnectTimeout(context.getConnectionTimeout());
+        conn.setReadTimeout(context.getReadTimeout());
+
+        return conn;
+    }
+
+    public RequestHttpContext getRequestHttpContext() {
+        return requestHttpContext;
+    }
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/httpclienthelper/HttpResponse.java

@@ -0,0 +1,131 @@
+package org.talend.ms.crm.odata.authentication.httpclienthelper;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public final class HttpResponse {
+    private int status;
+    private Map<String, List<String>> headers;
+    private String body;
+
+    private Optional<String> code = null;
+
+    public static HttpResponse fromHttpUrlConnection(HttpURLConnection conn) throws IOException {
+        final int status = conn.getResponseCode();
+        final Map<String, List<String>> respHeaders = conn.getHeaderFields();
+
+        HttpResponse context = new HttpResponse(status);
+        context.setHeaders(respHeaders);
+
+        try(BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"))){
+            final String body = in.lines().collect(Collectors.joining("\n"));
+            context.setBody(body);
+        }
+
+        return context;
+    }
+
+    private HttpResponse(int status) {
+        this.status = status;
+    }
+
+    public int getStatus() {
+        return status;
+    }
+
+    public void setStatus(int status) {
+        this.status = status;
+    }
+
+    public Map<String, List<String>> getHeaders() {
+        return headers;
+    }
+
+    public void setHeaders(Map<String, List<String>> headers) {
+        this.headers = headers;
+    }
+
+    private Optional<List<String>> _getAllValuesHeader(String key) {
+        if (this.getHeaders() == null || this.getHeaders().size() <= 0) {
+            return Optional.empty();
+        }
+
+        if (!this.getHeaders().containsKey(key)) {
+            return Optional.empty();
+        }
+
+        return Optional.ofNullable(this.getHeaders().get(key));
+    }
+
+    public Optional<List<String>> getAllValuesHeader(String key) {
+        String keyUpper = ("" + key.charAt(0)).toUpperCase() + key.substring(1);
+        String keyLower = ("" + key.charAt(0)).toLowerCase() + key.substring(1);
+
+        Optional<List<String>> values = _getAllValuesHeader(keyUpper);
+        if(!values.isPresent()){
+            values = _getAllValuesHeader(keyLower);
+        }
+
+        return values;
+    }
+
+    public Optional<String> getFirstValueHeader(String key) {
+        final Optional<List<String>> values = getAllValuesHeader(key);
+        if(!values.isPresent()){
+            return Optional.empty();
+        }
+
+        final List<String> ss = values.get();
+        if(ss.size() <= 0){
+            return Optional.empty();
+        }
+
+        return Optional.ofNullable(ss.get(0));
+    }
+
+    public String getBody() {
+        return body;
+    }
+
+    public void setBody(String body) {
+        this.body = body;
+    }
+
+    /**
+     * Extract 'Code' parameter from the redirect url if exist.
+     * @return An Optional with the code if any.
+     */
+    private Optional<String> _extractCode(){
+        final Optional<String> optLocation = this.getFirstValueHeader("Location");
+        if (!optLocation.isPresent()) {
+            return Optional.empty();
+        }
+        final String optLocationValue = optLocation.get();
+        final String[] split = optLocationValue.split("&|\\?");
+
+        final Optional<String> optCode = Arrays.stream(split).filter(e -> e.startsWith("code=")).findFirst();
+        if (optCode.isPresent()) {
+            final String optCodeValue = optCode.get();
+            String code = optCodeValue.substring(5);
+            return Optional.ofNullable(code);
+        }
+
+        return Optional.empty();
+    }
+
+    public Optional<String> extractCode(){
+        if(code != null){
+            return code;
+        }
+
+        code = _extractCode();
+        return code;
+    }
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/httpclienthelper/RequestHttpContext.java

@@ -0,0 +1,106 @@
+package org.talend.ms.crm.odata.authentication.httpclienthelper;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+
+public final class RequestHttpContext {
+
+    private String method;
+    private String base;
+    private Map<String, String> params;
+    private Map<String, String> headers;
+    private boolean followRedirects;
+    private int connectionTimeout;
+    private int readTimeout;
+    private String bodyContent = "";
+
+    public RequestHttpContext(String method, String base, Map<String, String> params, Map<String, String> headers, boolean followRedirects, int connectionTimeout, int readTimeout) {
+        this.method = method;
+        this.base = base;
+        this.params = params;
+        this.headers = headers;
+        this.followRedirects = followRedirects;
+        this.connectionTimeout = connectionTimeout;
+        this.readTimeout = readTimeout;
+    }
+
+    public String getBodyContent() {
+        return bodyContent;
+    }
+
+    public void setBodyContent(Map<String, String> form) throws UnsupportedEncodingException {
+        StringBuilder sb = new StringBuilder();
+
+        boolean first = true;
+        for (Map.Entry<String, String> e : form.entrySet()) {
+            if(!first){
+                sb.append("&");
+            }
+            first = false;
+
+            sb.append(e.getKey()).append("=").append(HttpClient.encodeValue(e.getValue()));
+        }
+
+        this.setBodyContent(sb.toString());
+    }
+
+    public void setBodyContent(String bodyContent) {
+        this.bodyContent = bodyContent;
+    }
+
+    public String getMethod() {
+        return method;
+    }
+
+    public void setMethod(String method) {
+        this.method = method;
+    }
+
+    public String getBase() {
+        return base;
+    }
+
+    public void setBase(String base) {
+        this.base = base;
+    }
+
+    public Map<String, String> getParams() {
+        return params;
+    }
+
+    public void setParams(Map<String, String> params) {
+        this.params = params;
+    }
+
+    public Map<String, String> getHeaders() {
+        return headers;
+    }
+
+    public void setHeaders(Map<String, String> headers) {
+        this.headers = headers;
+    }
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
+
+    public int getConnectionTimeout() {
+        return connectionTimeout;
+    }
+
+    public void setConnectionTimeout(int connectionTimeout) {
+        this.connectionTimeout = connectionTimeout;
+    }
+
+    public int getReadTimeout() {
+        return readTimeout;
+    }
+
+    public void setReadTimeout(int readTimeout) {
+        this.readTimeout = readTimeout;
+    }
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/authentication/httpclienthelper/Token.java

@@ -0,0 +1,38 @@
+package org.talend.ms.crm.odata.authentication.httpclienthelper;
+
+public final class Token {
+    // Time to be sure to refresh the token at time
+    private final static int EXPIRATION_SECURITY_GAP_SECOND = 60;
+
+    private String access_token;
+    private String token_type;
+    private Long expires_in;
+
+    public Token(){
+    }
+
+    public String getAccess_token() {
+        return access_token;
+    }
+
+    public void setAccess_token(String access_token) {
+        this.access_token = access_token;
+    }
+
+    public String getToken_type() {
+        return token_type;
+    }
+
+    public void setToken_type(String token_type) {
+        this.token_type = token_type;
+    }
+
+    public Long getExpires_in() {
+        return expires_in;
+    }
+
+    public void setExpires_in(Long expires_in) {
+        this.expires_in = expires_in;
+    }
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/httpclientfactory/DefaultHttpClientState.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/httpclientfactory/IHttpClientFactoryObserver.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/httpclientfactory/IHttpclientFactoryObservable.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/httpclientfactory/NTLMHttpClientFactory.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/odata/httpclientfactory/OAuthHttpClientFactory.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/sdk/Instance.java

@@ -0,0 +1,85 @@
+package org.talend.ms.crm.sdk;
+
+public class Instance {
+
+	public String id;
+	
+	public String uniqueName;
+	
+	public String urlName;
+	
+	public String friendlyName;
+	
+	public String state;
+	
+	public String version;
+	
+	public String url;
+	
+	public String apiUrl;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getUniqueName() {
+		return uniqueName;
+	}
+
+	public void setUniqueName(String uniqueName) {
+		this.uniqueName = uniqueName;
+	}
+
+	public String getUrlName() {
+		return urlName;
+	}
+
+	public void setUrlName(String urlName) {
+		this.urlName = urlName;
+	}
+
+	public String getFriendlyName() {
+		return friendlyName;
+	}
+
+	public void setFriendlyName(String friendlyName) {
+		this.friendlyName = friendlyName;
+	}
+
+	public String getState() {
+		return state;
+	}
+
+	public void setState(String state) {
+		this.state = state;
+	}
+
+	public String getVersion() {
+		return version;
+	}
+
+	public void setVersion(String version) {
+		this.version = version;
+	}
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+	public String getApiUrl() {
+		return apiUrl;
+	}
+
+	public void setApiUrl(String apiUrl) {
+		this.apiUrl = apiUrl;
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/sdk/OnlineAuthenticationPolicy.java

@@ -11,7 +11,7 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import javax.activity.InvalidActivityException;
+import java.security.InvalidParameterException;
 import javax.wsdl.WSDLException;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.xpath.XPathExpressionException;
@@ -107,7 +107,7 @@ public final class OnlineAuthenticationPolicy {
         this.initialize(flatWsdlUrl, targetDocument);
     }
 
-    private void initialize(String url, String wsdl) throws URISyntaxException, InvalidActivityException {
+    private void initialize(String url, String wsdl) throws URISyntaxException, InvalidParameterException {
         final String PolicyNodeContentRegexPatern = "(?i)(<wsp:All.*?>)(.+?)(</wsp:All>)";
         final String LiveIdentityProviderTrustRegexPatern = "(?i)(<ms-xrm:LiveTrust.*?>)(.+?)(</ms-xrm:LiveTrust>)";
         final String OrgIdentityProviderTrustRegexPatern = "(?i)(<ms-xrm:OrgTrust.*?>)(.+?)(</ms-xrm:OrgTrust>)";
@@ -166,7 +166,7 @@ public final class OnlineAuthenticationPolicy {
             }
         }
 
-        throw new InvalidActivityException(
+        throw new InvalidParameterException(
                 String.format(Locale.getDefault(), "Unable to parse the authentication policy from the WSDL \"%s\".", url));
 
     }

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/main/java/org/talend/ms/crm/sdk/OrganizationServiceStubWrapper.java

@@ -100,7 +100,7 @@ public class OrganizationServiceStubWrapper extends OrganizationServiceStub {
     }
 
     private void renewToken() throws Exception {
-        orgStub = client.doGetOnlineConnection(discoveryServiceUrl);
+        orgStub = client.doGetOnlineConnection();
     }
 
     private void sleep() {

main/plugins/org.talend.designer.components.libs/libs_src/talend-mscrm/src/test/java/org/talend/ms/crm/odata/DynamicsCRMClientTest.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2022 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt

main/plugins/org.talend.designer.components.libs/libs_src/talend-orc/pom.xml

@@ -0,0 +1,191 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>org.talend.components</groupId>
+	<artifactId>talend-orc</artifactId>
+	<version>1.0-20211008</version>
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<hadoop.version>3.2.2</hadoop.version>
+		<apache.orc.version>1.7.0</apache.orc.version>
+		<junit.jupiter.version>5.7.2</junit.jupiter.version>
+		<hamcrest.version>1.3</hamcrest.version>
+		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
+	</properties>
+
+	<distributionManagement>
+		<snapshotRepository>
+			<id>talend_nexus_deployment</id>
+			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
+			<snapshots>
+				<enabled>true</enabled>
+			</snapshots>
+			<releases>
+				<enabled>false</enabled>
+			</releases>
+		</snapshotRepository>
+		<repository>
+			<id>talend_nexus_deployment</id>
+			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+		</repository>
+	</distributionManagement>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.orc</groupId>
+			<artifactId>orc-core</artifactId>
+			<version>${apache.orc.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+			<version>${hadoop.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.eclipse.jetty</groupId>
+					<artifactId>jetty-server</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.eclipse.jetty</groupId>
+					<artifactId>jetty-servlet</artifactId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jersey-server</artifactId>
+					<groupId>com.sun.jersey</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jersey-servlet</artifactId>
+					<groupId>com.sun.jersey</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>slf4j-log4j12</artifactId>
+					<groupId>org.slf4j</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>log4j</artifactId>
+					<groupId>log4j</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>zookeeper</artifactId>
+					<groupId>org.apache.zookeeper</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jetty-webapp</artifactId>
+					<groupId>org.eclipse.jetty</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>javax.servlet-api</artifactId>
+					<groupId>javax.servlet</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs-client</artifactId>
+			<version>${hadoop.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.eclipse.jetty</groupId>
+					<artifactId>jetty-server</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.eclipse.jetty</groupId>
+					<artifactId>jetty-servlet</artifactId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jersey-server</artifactId>
+					<groupId>com.sun.jersey</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jersey-servlet</artifactId>
+					<groupId>com.sun.jersey</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>slf4j-log4j12</artifactId>
+					<groupId>org.slf4j</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>log4j</artifactId>
+					<groupId>log4j</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>zookeeper</artifactId>
+					<groupId>org.apache.zookeeper</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jetty-webapp</artifactId>
+					<groupId>org.eclipse.jetty</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>javax.servlet-api</artifactId>
+					<groupId>javax.servlet</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<version>${junit.jupiter.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-api</artifactId>
+			<version>${junit.jupiter.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-engine</artifactId>
+			<version>${junit.jupiter.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-params</artifactId>
+			<version>${junit.jupiter.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.platform</groupId>
+			<artifactId>junit-platform-launcher</artifactId>
+			<version>1.7.2</version>
+			<scope>test</scope>
+		</dependency>
+				<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest-library</artifactId>
+			<version>${hamcrest.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest-library</artifactId>
+			<version>${hamcrest.version}</version>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+	
+
+	<build>
+		<plugins>
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.8.0</version>
+				<configuration>
+					<source>1.8</source>
+					<target>1.8</target>
+				</configuration>
+			</plugin>
+
+		</plugins>
+	</build>
+</project>

main/plugins/org.talend.designer.components.libs/libs_src/talend-orc/src/main/java/org/talend/orc/ORCCommonUtils.java

@@ -0,0 +1,24 @@
+package org.talend.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+
+abstract class ORCCommonUtils {
+	/**
+	 * Check that the map type for the key is BYTES, LONG or DOUBLE and that the key
+	 * type is LONG, DOUBLE, BYTES, DECIMAL or TIMESTAMP.
+	 *
+	 * @param mapVector a MapColumnVector
+	 * @return true if the key and value types conform to the limits described
+	 *         above.
+	 */
+	public static boolean checkMapColumnVectorTypes(MapColumnVector mapVector) {
+		ColumnVector.Type keyType = mapVector.keys.type;
+		ColumnVector.Type valueType = mapVector.values.type;
+		return (keyType == ColumnVector.Type.BYTES || keyType == ColumnVector.Type.LONG
+				|| keyType == ColumnVector.Type.DOUBLE)
+				&& (valueType == ColumnVector.Type.LONG || valueType == ColumnVector.Type.DOUBLE
+						|| valueType == ColumnVector.Type.BYTES || valueType == ColumnVector.Type.DECIMAL
+						|| valueType == ColumnVector.Type.TIMESTAMP);
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-orc/src/main/java/org/talend/orc/ORCReadUtils.java

@@ -0,0 +1,442 @@
+package org.talend.orc;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.TypeDescription;
+
+public class ORCReadUtils {
+	public static Object readColumnByName(VectorizedRowBatch batch, String columnName, TypeDescription schema,
+			int rowNum) {
+		List<String> allColumnNames = schema.getFieldNames();
+		int colIndex = allColumnNames.indexOf(columnName);
+		if (colIndex < 0 || colIndex > batch.cols.length - 1) {
+			return null;
+		} else {
+			org.apache.hadoop.hive.ql.exec.vector.ColumnVector colVector = batch.cols[colIndex];
+			TypeDescription fieldType = schema.getChildren().get(colIndex);
+			int colRow = colVector.isRepeating ? 0 : rowNum;
+			Object value = readColumn(colVector, fieldType, colRow);
+			return value;
+		}
+
+	}
+
+	public static Object readColumn(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object columnObj = null;
+		if (!colVec.isNull[rowNum]) {
+			switch (colVec.type) {
+			case BYTES:
+				columnObj = readBytesVal(colVec, colType, rowNum);
+				break;
+			case DECIMAL:
+				columnObj = readDecimalVal(colVec, rowNum);
+				break;
+			case DOUBLE:
+				columnObj = readDoubleVal(colVec, colType, rowNum);
+				break;
+			case LIST:
+				columnObj = readListVal(colVec, colType, rowNum);
+				break;
+			case LONG:
+				columnObj = readLongVal(colVec, colType, rowNum);
+				break;
+			case MAP:
+				columnObj = readMapVal(colVec, colType, rowNum);
+				break;
+			case STRUCT:
+				columnObj = readStructVal(colVec, colType, rowNum);
+				break;
+			case TIMESTAMP:
+				columnObj = readTimestampVal(colVec, colType, rowNum);
+				break;
+			case UNION:
+				columnObj = readUnionVal(colVec, colType, rowNum);
+				break;
+			default:
+				throw new RuntimeException("readColumn: unsupported ORC file column type: " + colVec.type.name());
+			}
+		}
+		return columnObj;
+	}
+
+	private static Object readListVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object listValues = null;
+		if (!colVec.isNull[rowNum]) {
+			ListColumnVector listVector = (ListColumnVector) colVec;
+			ColumnVector listChildVector = listVector.child;
+			TypeDescription childType = colType.getChildren().get(0);
+			switch (listChildVector.type) {
+			case BYTES:
+				listValues = readBytesListValues(listVector, childType, rowNum);
+				break;
+			case DECIMAL:
+				listValues = readDecimalListValues(listVector, rowNum);
+				break;
+			case DOUBLE:
+				listValues = readDoubleListValues(listVector, rowNum);
+				break;
+			case LONG:
+				listValues = readLongListValues(listVector, childType, rowNum);
+				break;
+			case TIMESTAMP:
+				listValues = readTimestampListValues(listVector, childType, rowNum);
+				break;
+			default:
+				throw new RuntimeException(listVector.type.name() + " is not supported for ListColumnVectors");
+			}
+		}
+		return listValues;
+	}
+
+	private static List<Object> readLongListVector(LongColumnVector longVector, TypeDescription childType, int offset,
+			int numValues) {
+		List<Object> longList = new ArrayList<>();
+		for (int i = 0; i < numValues; i++) {
+			if (!longVector.isNull[offset + i]) {
+				long longVal = longVector.vector[offset + i];
+				if (childType.getCategory() == TypeDescription.Category.BOOLEAN) {
+					Boolean boolVal = longVal == 0 ? Boolean.valueOf(false) : Boolean.valueOf(true);
+					longList.add(boolVal);
+				} else if (childType.getCategory() == TypeDescription.Category.INT) {
+					Integer intObj = (int) longVal;
+					longList.add(intObj);
+				} else {
+					longList.add(longVal);
+				}
+			} else {
+				longList.add(null);
+			}
+		}
+		return longList;
+	}
+
+	private static Object readLongListValues(ListColumnVector listVector, TypeDescription childType, int rowNum) {
+		int offset = (int) listVector.offsets[rowNum];
+		int numValues = (int) listVector.lengths[rowNum];
+		LongColumnVector longVector = (LongColumnVector) listVector.child;
+		return readLongListVector(longVector, childType, offset, numValues);
+	}
+
+	private static Object readTimestampListVector(TimestampColumnVector timestampVector, TypeDescription childType,
+			int offset, int numValues) {
+		List<Object> timestampList = new ArrayList<>();
+		for (int i = 0; i < numValues; i++) {
+			if (!timestampVector.isNull[offset + i]) {
+				int nanos = timestampVector.nanos[offset + i];
+				long millisec = timestampVector.time[offset + i];
+				Timestamp timestamp = new Timestamp(millisec);
+				timestamp.setNanos(nanos);
+				if (childType.getCategory() == TypeDescription.Category.DATE) {
+					Date date = new Date(timestamp.getTime());
+					timestampList.add(date);
+				} else {
+					timestampList.add(timestamp);
+				}
+			} else {
+				timestampList.add(null);
+			}
+		}
+		return timestampList;
+	}
+
+	/**
+	 * Read either Timestamp or Date values, depending on the definition in the
+	 * schema.
+	 */
+	private static Object readTimestampListValues(ListColumnVector listVector, TypeDescription childType, int rowNum) {
+		int offset = (int) listVector.offsets[rowNum];
+		int numValues = (int) listVector.lengths[rowNum];
+		TimestampColumnVector timestampVec = (TimestampColumnVector) listVector.child;
+		return readTimestampListVector(timestampVec, childType, offset, numValues);
+	}
+
+	private static Object readDecimalListVector(DecimalColumnVector decimalVector, int offset, int numValues,
+			int batchRowNum) {
+		List<Object> decimalList = new ArrayList<>();
+		for (int i = 0; i < numValues; i++) {
+			if (!decimalVector.isNull[offset + i]) {
+				BigDecimal bigDecimal = decimalVector.vector[batchRowNum].getHiveDecimal().bigDecimalValue();
+				decimalList.add(bigDecimal);
+			} else {
+				decimalList.add(null);
+			}
+		}
+		return decimalList;
+	}
+
+	private static Object readDecimalListValues(ListColumnVector listVector, int rowNum) {
+		int offset = (int) listVector.offsets[rowNum];
+		int numValues = (int) listVector.lengths[rowNum];
+		DecimalColumnVector decimalVec = (DecimalColumnVector) listVector.child;
+		return readDecimalListVector(decimalVec, offset, numValues, rowNum);
+	}
+
+	private static Object readBytesListVector(BytesColumnVector bytesVec, TypeDescription childType, int offset,
+			int numValues) {
+		List<Object> bytesValList = new ArrayList<>();
+		for (int i = 0; i < numValues; i++) {
+			if (!bytesVec.isNull[offset + i]) {
+				byte[] byteArray = bytesVec.vector[offset + i];
+				int vecLen = bytesVec.length[offset + i];
+				int vecStart = bytesVec.start[offset + i];
+				byte[] vecCopy = Arrays.copyOfRange(byteArray, vecStart, vecStart + vecLen);
+				if (childType.getCategory() == TypeDescription.Category.STRING) {
+					String str = new String(vecCopy);
+					bytesValList.add(str);
+				} else {
+					bytesValList.add(vecCopy);
+				}
+			} else {
+				bytesValList.add(null);
+			}
+		}
+		return bytesValList;
+	}
+
+	private static Object readBytesListValues(ListColumnVector listVector, TypeDescription childType, int rowNum) {
+		int offset = (int) listVector.offsets[rowNum];
+		int numValues = (int) listVector.lengths[rowNum];
+		BytesColumnVector bytesVec = (BytesColumnVector) listVector.child;
+		return readBytesListVector(bytesVec, childType, offset, numValues);
+	}
+
+	private static Object readDoubleListVector(DoubleColumnVector doubleVec, int offset, int numValues) {
+		List<Object> doubleList = new ArrayList<>();
+		for (int i = 0; i < numValues; i++) {
+			if (!doubleVec.isNull[offset + i]) {
+				Double doubleVal = doubleVec.vector[offset + i];
+				doubleList.add(doubleVal);
+			} else {
+				doubleList.add(null);
+			}
+		}
+		return doubleList;
+	}
+
+	private static Object readDoubleListValues(ListColumnVector listVector, int rowNum) {
+
+		int offset = (int) listVector.offsets[rowNum];
+		int numValues = (int) listVector.lengths[rowNum];
+		DoubleColumnVector doubleVec = (DoubleColumnVector) listVector.child;
+		return readDoubleListVector(doubleVec, offset, numValues);
+	}
+
+	@SuppressWarnings("unchecked")
+	private static List<Object> readMapVector(ColumnVector mapVector, TypeDescription childType, int offset,
+			int numValues, int rowNum) {
+		List<Object> mapList = null;
+		switch (mapVector.type) {
+		case BYTES:
+			mapList = (List<Object>) readBytesListVector((BytesColumnVector) mapVector, childType, offset, numValues);
+			break;
+		case DECIMAL:
+			mapList = (List<Object>) readDecimalListVector((DecimalColumnVector) mapVector, offset, numValues, rowNum);
+			;
+			break;
+		case DOUBLE:
+			mapList = (List<Object>) readDoubleListVector((DoubleColumnVector) mapVector, offset, numValues);
+			break;
+		case LONG:
+			mapList = readLongListVector((LongColumnVector) mapVector, childType, offset, numValues);
+			break;
+		case TIMESTAMP:
+			mapList = (List<Object>) readTimestampListVector((TimestampColumnVector) mapVector, childType, offset,
+					numValues);
+			break;
+		default:
+			throw new RuntimeException(mapVector.type.name() + " is not supported for MapColumnVectors");
+		}
+		return mapList;
+	}
+
+	/**
+	 * <p>
+	 * Read a Map column value (e.g., a set of keys and their associated values).
+	 * </p>
+	 * <p>
+	 * The Map key and value types are the first and second children in the children
+	 * TypeDescription List. From the TypeDescription source:
+	 * </p>
+	 * 
+	 * <pre>
+	 * result.children.add(keyType);
+	 * result.children.add(valueType);
+	 * </pre>
+	 */
+	private static Object readMapVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Map<Object, Object> objMap = new HashMap<>();
+		MapColumnVector mapVector = (MapColumnVector) colVec;
+		if (ORCCommonUtils.checkMapColumnVectorTypes(mapVector)) {
+			int mapSize = (int) mapVector.lengths[rowNum];
+			int offset = (int) mapVector.offsets[rowNum];
+			List<TypeDescription> mapTypes = colType.getChildren();
+			TypeDescription keyType = mapTypes.get(0);
+			TypeDescription valueType = mapTypes.get(1);
+			ColumnVector keyChild = mapVector.keys;
+			ColumnVector valueChild = mapVector.values;
+			List<Object> keyList = readMapVector(keyChild, keyType, offset, mapSize, rowNum);
+			List<Object> valueList = readMapVector(valueChild, valueType, offset, mapSize, rowNum);
+			assert (keyList.size() == valueList.size());
+			for (int i = 0; i < keyList.size(); i++) {
+				objMap.put(keyList.get(i), valueList.get(i));
+			}
+		} else {
+			throw new RuntimeException("readMapVal: unsupported key or value types");
+		}
+		return objMap;
+	}
+
+	private static Object readUnionVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Pair<TypeDescription, Object> columnValuePair;
+		UnionColumnVector unionVector = (UnionColumnVector) colVec;
+		int tagVal = unionVector.tags[rowNum];
+		List<TypeDescription> unionFieldTypes = colType.getChildren();
+		if (tagVal < unionFieldTypes.size()) {
+			TypeDescription fieldType = unionFieldTypes.get(tagVal);
+			if (tagVal < unionVector.fields.length) {
+				ColumnVector fieldVector = unionVector.fields[tagVal];
+				int colRow = fieldVector.isRepeating ? 0 : rowNum;
+				Object unionValue = readColumn(fieldVector, fieldType, colRow);
+				columnValuePair = new ImmutablePair<>(fieldType, unionValue);
+			} else {
+				throw new RuntimeException("readUnionVal: union tag value out of range for union column vectors");
+			}
+		} else {
+			throw new RuntimeException("readUnionVal: union tag value out of range for union types");
+		}
+		return columnValuePair;
+	}
+
+	private static Object readStructVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object structObj = null;
+		if (!colVec.isNull[rowNum]) {
+			List<Object> fieldValList = new ArrayList<>();
+			StructColumnVector structVector = (StructColumnVector) colVec;
+			ColumnVector[] fieldVec = structVector.fields;
+			List<TypeDescription> fieldTypes = colType.getChildren();
+			assert (fieldVec.length == fieldTypes.size());
+			for (int i = 0; i < fieldVec.length; i++) {
+				int colRow = fieldVec[i].isRepeating ? 0 : rowNum;
+				Object fieldObj = readColumn(fieldVec[i], fieldTypes.get(i), colRow);
+				fieldValList.add(fieldObj);
+			}
+			structObj = fieldValList;
+		}
+		return structObj;
+	}
+
+	private static Object readTimestampVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object timestampVal = null;
+		if (!colVec.isNull[rowNum]) {
+			TimestampColumnVector timestampVec = (TimestampColumnVector) colVec;
+			int nanos = timestampVec.nanos[rowNum];
+			long millisec = timestampVec.time[rowNum];
+			Timestamp timestamp = new Timestamp(millisec);
+			timestamp.setNanos(nanos);
+			timestampVal = timestamp;
+			if (colType.getCategory() == TypeDescription.Category.DATE) {
+				timestampVal = new Date(timestamp.getTime());
+			}
+		}
+		return timestampVal;
+	}
+
+	private static Object readDecimalVal(ColumnVector colVec, int rowNum) {
+		Object decimalObj = null;
+		if (!colVec.isNull[rowNum]) {
+			DecimalColumnVector decimalVec = (DecimalColumnVector) colVec;
+			decimalObj = decimalVec.vector[rowNum].getHiveDecimal().bigDecimalValue().setScale(decimalVec.scale);
+		}
+		return decimalObj;
+	}
+
+	/**
+	 * Read a Long or Boolean value
+	 * 
+	 * @param colVec  the column vector
+	 * @param colType the type of the column
+	 * @param rowNum  the ORC file row number.
+	 * @return a Boolean or Long object
+	 */
+	private static Object readLongVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object colObj = null;
+		if (!colVec.isNull[rowNum]) {
+			LongColumnVector longVec = (LongColumnVector) colVec;
+			Long longVal = longVec.vector[rowNum];
+			colObj = longVal;
+			if (colType.getCategory() == TypeDescription.Category.INT) {
+				colObj = longVal.intValue();
+			} else if (colType.getCategory() == TypeDescription.Category.BOOLEAN) {
+				colObj = longVal == 1 ? Boolean.TRUE : Boolean.FALSE;
+			} else if (colType.getCategory() == TypeDescription.Category.DATE) {
+				colObj = new Date(longVal * 86400000);
+			} else if (colType.getCategory() == TypeDescription.Category.BYTE) {
+				colObj = longVal.byteValue();
+			} else if (colType.getCategory() == TypeDescription.Category.SHORT) {
+				colObj = longVal.shortValue();
+			}
+		}
+		return colObj;
+	}
+
+	/**
+	 * Read a Double or Float value
+	 * 
+	 * @param colVec  the column vector
+	 * @param colType the type of the column
+	 * @param rowNum  the ORC file row number.
+	 * @return a Double or Float object
+	 */
+	private static Object readDoubleVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object colObj = null;
+		if (!colVec.isNull[rowNum]) {
+			DoubleColumnVector longVec = (DoubleColumnVector) colVec;
+			Double doubleVal = longVec.vector[rowNum];
+			colObj = doubleVal;
+			if (colType.getCategory() == TypeDescription.Category.FLOAT) {
+				colObj = doubleVal.floatValue();
+			}
+		}
+		return colObj;
+	}
+
+	private static Object readBytesVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
+		Object bytesObj = null;
+		if (!colVec.isNull[rowNum]) {
+			BytesColumnVector bytesVector = (BytesColumnVector) colVec;
+			byte[] columnBytes = bytesVector.vector[rowNum];
+			int vecLen = bytesVector.length[rowNum];
+			int vecStart = bytesVector.start[rowNum];
+			byte[] vecCopy = Arrays.copyOfRange(columnBytes, vecStart, vecStart + vecLen);
+			if (colType.getCategory() == TypeDescription.Category.STRING ||colType.getCategory() == TypeDescription.Category.VARCHAR) {
+				bytesObj = new String(vecCopy);
+			} else if (colType.getCategory() == TypeDescription.Category.CHAR) {
+				String charStr = new String(vecCopy);
+				bytesObj = charStr;
+			} else {
+				bytesObj = vecCopy;
+			}
+		}
+		return bytesObj;
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-orc/src/main/java/org/talend/orc/ORCWriteUtils.java

@@ -0,0 +1,884 @@
+package org.talend.orc;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MultiValuedColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.orc.TypeDescription;
+
+public class ORCWriteUtils {
+	private static final Predicate<Object> isInteger = Integer.class::isInstance;
+	private static final Predicate<Object> isLong = Long.class::isInstance;
+	private static final Predicate<Object> isDouble = Double.class::isInstance;
+	private static final Predicate<Object> isString = String.class::isInstance;
+	private static final Predicate<Object> isBigDecimal = BigDecimal.class::isInstance;
+	private static final Predicate<Object> isDate = Date.class::isInstance;
+
+	public static void setColumnByName(VectorizedRowBatch batch, String columnName, TypeDescription schema,
+			Object colVal, int rowNum) {
+		List<String> allColumnNames = schema.getFieldNames();
+		int colIndex = allColumnNames.indexOf(columnName);
+		if (colIndex < 0 || colIndex > batch.cols.length - 1) {
+			return;
+		} else {
+			org.apache.hadoop.hive.ql.exec.vector.ColumnVector colVector = batch.cols[colIndex];
+			TypeDescription fieldType = schema.getChildren().get(colIndex);
+			setColumn(colVal, fieldType, columnName, colVector, rowNum);
+		}
+
+	}
+
+	/**
+	 * Add a column value that is a String or a byte[] array.
+	 *
+	 * @param colVal         the column value object
+	 * @param fieldName      the name of the field (for error reporting)
+	 * @param bytesColVector the BytesColumnVector that the byte array will be added
+	 *                       to.
+	 * @param rowNum         the ORC file row number
+	 */
+	private static void setByteColumnVector(Object colVal, String fieldName, BytesColumnVector bytesColVector,
+			int rowNum) {
+		if (colVal instanceof byte[] || colVal instanceof String || colVal instanceof Character) {
+			byte[] byteVec;
+			if (colVal instanceof String) {
+				String strVal = (String) colVal;
+				byteVec = strVal.getBytes(StandardCharsets.UTF_8);
+			} else if (colVal instanceof Character) {
+				String strVal = String.valueOf((char) colVal);
+				byteVec = strVal.getBytes(StandardCharsets.UTF_8);
+			} else {
+				byteVec = (byte[]) colVal;
+			}
+			bytesColVector.setRef(rowNum, byteVec, 0, byteVec.length);
+		} else {
+			throw new RuntimeException(orcExceptionMsg("byte[] or String type expected for field ", fieldName, rowNum));
+		}
+	}
+
+	private static void setDecimalVector(Object colVal, String fieldName, DecimalColumnVector decimalColVector,
+			int rowNum) {
+		if (colVal instanceof BigDecimal) {
+			BigDecimal bigDecimal = (BigDecimal) colVal;
+			decimalColVector.precision = (short) bigDecimal.precision();
+			decimalColVector.scale = (short) bigDecimal.scale();
+			HiveDecimal hiveDecimal = HiveDecimal.create(bigDecimal);
+			HiveDecimalWritable writeableDecimal = new HiveDecimalWritable(hiveDecimal);
+			decimalColVector.vector[rowNum] = writeableDecimal;
+		} else {
+			throw new RuntimeException(orcExceptionMsg("BigDecimal type expected for field ", fieldName, rowNum));
+		}
+	}
+
+	private static void setDoubleVector(Object colVal, String fieldName, DoubleColumnVector doubleVector, int rowNum) {
+		if (colVal instanceof Double) {
+			doubleVector.vector[rowNum] = (Double) colVal;
+		} else if (colVal instanceof Float) {
+			Float fltVal = (Float) colVal;
+			doubleVector.vector[rowNum] = fltVal.doubleValue();
+		} else if (colVal instanceof BigDecimal) {
+			doubleVector.vector[rowNum] = ((BigDecimal) colVal).doubleValue();
+		} else {
+			throw new RuntimeException(orcExceptionMsg("Double/Float/BigDecimal type expected for field ", fieldName, rowNum));
+		}
+	}
+
+	/**
+	 * Initialize a LongColumnVector value.
+	 * 
+	 * @param colVal     an object of type Boolean, Integer, Long or BigInteger.
+	 * @param fieldName  the field name in the schema
+	 * @param longVector the LongColumnVector
+	 * @param rowNum     the row number
+	 */
+	private static void setLongColumnVector(Object colVal, String fieldName, LongColumnVector longVector, int rowNum) {
+		if (colVal instanceof Boolean) {
+			Boolean bool = (Boolean) colVal;
+			longVector.vector[rowNum] = (bool.equals(Boolean.TRUE)) ? Long.valueOf(1) : Long.valueOf(0);
+		} else if (colVal instanceof Byte) {
+			longVector.vector[rowNum] = (Byte) colVal;
+		} else if (colVal instanceof Short) {
+			longVector.vector[rowNum] = (Short) colVal;
+		} else if (colVal instanceof Integer) {
+			longVector.vector[rowNum] = (Integer) colVal;
+		} else if (colVal instanceof Long) {
+			longVector.vector[rowNum] = (Long) colVal;
+		} else if (colVal instanceof BigInteger) {
+			BigInteger bigInt = (BigInteger) colVal;
+			longVector.vector[rowNum] = bigInt.longValue();
+		} else {
+			throw new RuntimeException(orcExceptionMsg("Long or Integer type expected for field ", fieldName, rowNum));
+		}
+	}
+
+	private static void setDateColumnVector(Object colVal, String fieldName, DateColumnVector dateVector, int rowNum) {
+		if (colVal instanceof Date) {
+			Date dateVal = (Date) colVal;
+			Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+			cal.setTime(dateVal);
+			long epochDay = LocalDate.of(cal.get(Calendar.YEAR), cal.get(Calendar.MONTH)+1, cal.get(Calendar.DAY_OF_MONTH)).toEpochDay();
+			dateVector.vector[rowNum] = epochDay;
+		} else {
+			throw new RuntimeException(orcExceptionMsg("Date type expected for field ", fieldName, rowNum));
+		}
+	}
+
+	private static void setTimestampVector(Object colVal, String fieldName, TimestampColumnVector timestampVector,
+			int rowNum) {
+		if (colVal instanceof Timestamp) {
+			timestampVector.set(rowNum, (Timestamp) colVal);
+		} else if (colVal instanceof Date) {
+			Date date = (Date) colVal;
+			Timestamp ts = new Timestamp(date.getTime());
+
+			timestampVector.set(rowNum, ts);
+		} else {
+			throw new RuntimeException(
+					orcExceptionMsg("Date or Timestamp type expected for field ", fieldName, rowNum));
+		}
+	}
+
+	/**
+	 * <p>
+	 * A union column can contain column vectors of more than one type. In the
+	 * TypeDescription createUnion() is called to create a TypeDescription for a
+	 * union column. The union values are added by calling the addUnionChild()
+	 * method on this TypeDescription object.
+	 * </p>
+	 * <p>
+	 * The class fields in the UnionColumnVector are shown below:
+	 * </p>
+	 * 
+	 * <pre>
+	 *     public class UnionColumnVector extends ColumnVector {
+	 *        public int[] tags;
+	 *        public ColumnVector[] fields;
+	 * </pre>
+	 * <p>
+	 * A tag value (
+	 * 
+	 * <pre>
+	 * tags[rowNum]
+	 * </pre>
+	 * 
+	 * ) is associated with each field value (
+	 * 
+	 * <pre>
+	 * fields[rowNum])
+	 * </pre>
+	 * 
+	 * . The tag value serves as an index for the field type. For example, if there
+	 * are three field types defined:
+	 * <ol>
+	 * <li>Long</li>
+	 * <li>Double</li>
+	 * <li>String</li>
+	 * </ol>
+	 * The tag will have a value in the range of [0..2]
+	 * </p>
+	 * <p>
+	 * The tag value is needed to initialize the ColumnVector since without the tag
+	 * there is no way to know which union child should be initialized.
+	 * </p>
+	 *
+	 * @param colVal      a Pair&lt;ColumnVector.Type, Object&gt; object with the
+	 *                    union type and the object that will be used to initialize
+	 *                    the union child ColumnVector.
+	 * @param fieldName   The name of the union field
+	 * @param unionVector The UnionColumnVector to be initialized
+	 * @param rowNum      the ORC file row number.
+	 */
+	private static void setUnionColumnVector(Object colVal, TypeDescription unionTypeDesc, String fieldName,
+			UnionColumnVector unionVector, int rowNum) {
+		@SuppressWarnings("unchecked")
+		Pair<TypeDescription, Object> unionValuePair = (Pair<TypeDescription, Object>) colVal;
+		TypeDescription unionValType = unionValuePair.getLeft();
+		List<TypeDescription> unionChildrenTypes = unionTypeDesc.getChildren();
+		Object unionColVal = unionValuePair.getRight();
+		boolean found = false;
+		for (int i = 0; i < unionChildrenTypes.size(); i++) {
+			if (unionChildrenTypes.get(i).getCategory() == unionValType.getCategory()) {
+				unionVector.tags[rowNum] = i;
+				ColumnVector unionFieldVec = unionVector.fields[i];
+				setColumn(unionColVal, unionChildrenTypes.get(i), fieldName, unionFieldVec, rowNum);
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			throw new RuntimeException("writeUnionColumnVector: Bad type enumeration "
+					+ unionValType.getCategory().getName() + " passed for field " + fieldName);
+		}
+	}
+
+	private static void setLongListVector(List<Object> longValList, LongColumnVector longVector, int offset,
+			String fieldName) {
+		for (int i = 0; i < longValList.size(); i++) {
+			Object objVal = longValList.get(i);
+			if (objVal != null) {
+				if (objVal instanceof Integer) {
+					longVector.vector[offset + i] = (Integer) objVal;
+				} else if (objVal instanceof Long) {
+					longVector.vector[offset + i] = (Long) objVal;
+				} else {
+					throw new RuntimeException("List<Integer> expected for field " + fieldName);
+				}
+			} else {
+				longVector.isNull[offset + i] = true;
+				longVector.noNulls = false;
+			}
+		}
+	}
+
+	private static void setLongList(List<Object> colValList, ListColumnVector listVector, String fieldName,
+			int rowNum) {
+		LongColumnVector longVector = (LongColumnVector) listVector.child;
+		int offset = (int) listVector.offsets[rowNum];
+		setLongListVector(colValList, longVector, offset, fieldName);
+	}
+
+	private static void setDoubleListVector(List<Object> doubleValList, DoubleColumnVector doubleVector, int offset,
+			String fieldName) {
+		for (int i = 0; i < doubleValList.size(); i++) {
+			Object objVal = doubleValList.get(i);
+			if (objVal != null) {
+				if (objVal instanceof Double) {
+					doubleVector.vector[offset + i] = (Double) objVal;
+				} else if (objVal instanceof Float) {
+					Float fltVal = (Float) objVal;
+					doubleVector.vector[offset + i] = fltVal.doubleValue();
+				} else {
+					throw new RuntimeException("List<Double> expected for field " + fieldName);
+				}
+			} else {
+				doubleVector.isNull[offset + i] = true;
+				doubleVector.noNulls = false;
+			}
+		}
+	}
+
+	private static void setDoubleList(List<Object> doubleValList, ListColumnVector listVector, String fieldName,
+			int rowNum) {
+		DoubleColumnVector vecChild = (DoubleColumnVector) listVector.child;
+		int offset = (int) listVector.offsets[rowNum];
+		setDoubleListVector(doubleValList, vecChild, offset, fieldName);
+	}
+
+	private static void setTimestampListVector(List<Object> valueList, TimestampColumnVector timestampVector,
+			int offset, String fieldName) {
+		for (int i = 0; i < valueList.size(); i++) {
+			Object objVal = valueList.get(i);
+			if (objVal != null) {
+				if (objVal instanceof Date) {
+					Timestamp ts = (objVal instanceof Timestamp) ? (Timestamp) objVal
+							: new Timestamp(((Date) objVal).getTime());
+					timestampVector.time[offset + i] = ts.getTime();
+					timestampVector.nanos[offset + i] = ts.getNanos();
+				} else {
+					throw new RuntimeException("List<Date> or List<Timestamp> expected for field " + fieldName);
+				}
+			} else {
+				timestampVector.isNull[offset + i] = true;
+				timestampVector.noNulls = false;
+			}
+		}
+	}
+
+	/**
+	 * Initialize the vector values for a ListColumnVector of Date or Timestamp
+	 * values.
+	 *
+	 * @param colValList a list of Timestamp or java.util.Date objects
+	 * @param listVector A ListColumnVector with a child that will contain the
+	 *                   vector values.
+	 * @param fieldName  The field name in the schema for this ORC element
+	 * @param rowNum     The ORC file row number
+	 */
+	private static void setTimestampList(List<Object> colValList, ListColumnVector listVector, String fieldName,
+			int rowNum) {
+		TimestampColumnVector timestampVector = (TimestampColumnVector) listVector.child;
+		int offset = (int) listVector.offsets[rowNum];
+		setTimestampListVector(colValList, timestampVector, offset, fieldName);
+	}
+
+	private static void setDecimalListVector(List<Object> decimalValList, DecimalColumnVector decimalVector, int offset,
+			String fieldName) {
+		for (int i = 0; i < decimalValList.size(); i++) {
+			Object objVal = decimalValList.get(i);
+			if (objVal != null) {
+				if (objVal instanceof BigDecimal) {
+					BigDecimal bigDecimal = (BigDecimal) objVal;
+					decimalVector.precision = (short) bigDecimal.precision();
+					decimalVector.scale = (short) bigDecimal.scale();
+					HiveDecimal hiveDecimal = HiveDecimal.create(bigDecimal);
+					HiveDecimalWritable writeableDecimal = new HiveDecimalWritable(hiveDecimal);
+					decimalVector.vector[offset + i] = writeableDecimal;
+				} else {
+					throw new RuntimeException("BigDecimal value expected for field " + fieldName);
+				}
+			} else {
+				decimalVector.isNull[offset + i] = true;
+				decimalVector.noNulls = false;
+			}
+		}
+	}
+
+	/**
+	 *
+	 * @param colValList a list of BigDecimal values to initialize the
+	 *                   ListColumnVector child
+	 * @param listVector the ListColumnVector with the DecimalColumnVector child
+	 * @param fieldName  the field name for the ListColumnVector/DecimalColumnVector
+	 *                   column
+	 * @param rowNum     the ORC file row number
+	 */
+	private static void setDecimalList(List<Object> colValList, ListColumnVector listVector, String fieldName,
+			int rowNum) {
+		DecimalColumnVector decimalVector = (DecimalColumnVector) listVector.child;
+		int offset = (int) listVector.offsets[rowNum];
+		setDecimalListVector(colValList, decimalVector, offset, fieldName);
+	}
+
+	private static void setBytesListVector(List<Object> valueList, BytesColumnVector bytesVector, int offset,
+			String fieldName) {
+		for (int i = 0; i < valueList.size(); i++) {
+			Object objVal = valueList.get(i);
+			if (objVal != null) {
+				if (objVal instanceof byte[] || objVal instanceof String) {
+					byte[] byteVec = (objVal instanceof byte[]) ? (byte[]) objVal
+							: ((String) objVal).getBytes(StandardCharsets.UTF_8);
+					bytesVector.vector[offset + i] = byteVec;
+					bytesVector.length[offset + i] = byteVec.length;
+				} else {
+					throw new RuntimeException("String or byte[] value expected for field " + fieldName);
+				}
+			} else {
+				bytesVector.isNull[offset + i] = true;
+				bytesVector.length[offset + i] = 0;
+				bytesVector.noNulls = false;
+			}
+		}
+	}
+
+	/**
+	 * Initialize a ListColumnVector with a BytesColumnVector child with byte[]
+	 * values.
+	 *
+	 * @param colValList a list of byte[] or String values
+	 * @param listVector the parent ListColumnVector
+	 * @param fieldName  the field name for the ORC column that contains the
+	 *                   ListColumnVector
+	 * @param rowNum     the ORC file row number
+	 */
+	private static void setBytesList(List<Object> colValList, ListColumnVector listVector, String fieldName,
+			int rowNum) {
+		BytesColumnVector bytesVector = (BytesColumnVector) listVector.child;
+		int offset = (int) listVector.offsets[rowNum];
+		setBytesListVector(colValList, bytesVector, offset, fieldName);
+	}
+
+	private static void setMultiValuedVectorParameters(MultiValuedColumnVector multiVector, int vecLength, int rowNum) {
+		multiVector.lengths[rowNum] = vecLength;
+		if (rowNum > 0) {
+			multiVector.offsets[rowNum] = multiVector.lengths[rowNum - 1] + multiVector.offsets[rowNum - 1];
+		}
+	}
+
+	private static void setListVectorParameters(ListColumnVector listVec, int maxBatchSize, int vecLength, int rowNum) {
+		setMultiValuedVectorParameters(listVec, vecLength, rowNum);
+		listVec.child.ensureSize(maxBatchSize * vecLength, true);
+	}
+
+	/**
+	 * Initialize a ListColumnVector. The child of the vector is limited to the
+	 * scalar types long, double, String (or byte[])), BigDecimal or Date (or
+	 * Timestamp).
+	 *
+	 * @param colVal     a List&lt;Object&gt;
+	 * @param typeDesc   the schema definition for this column
+	 * @param fieldName  the column field name
+	 * @param listVector the ListColumnVector parent of the vector type child
+	 * @param rowNum     the ORC file row number.
+	 */
+	private static void setListColumnVector(Object colVal, TypeDescription typeDesc, String fieldName,
+			ListColumnVector listVector, int rowNum) {
+		if (colVal instanceof List) {
+			@SuppressWarnings("unchecked")
+			List<Object> objValList = (List<Object>) colVal;
+			final int maxBatchSize = typeDesc.createRowBatch().getMaxSize();
+			setListVectorParameters(listVector, maxBatchSize, objValList.size(), rowNum);
+			ColumnVector.Type childType = listVector.child.type;
+			switch (childType) {
+			case LONG:
+				setLongList(objValList, listVector, fieldName, rowNum);
+				break;
+			case DOUBLE:
+				setDoubleList(objValList, listVector, fieldName, rowNum);
+				break;
+			case BYTES:
+				setBytesList(objValList, listVector, fieldName, rowNum);
+				break;
+			case DECIMAL:
+				setDecimalList(objValList, listVector, fieldName, rowNum);
+				break;
+			case TIMESTAMP:
+				setTimestampList(objValList, listVector, fieldName, rowNum);
+				break;
+			default:
+				throw new RuntimeException(childType.name() + " is not supported for ListColumnVector columns");
+			}
+		} else {
+			throw new RuntimeException("List value expected for field " + fieldName);
+		}
+	}
+
+	/**
+	 * Test that all elements in an Object list are of a particular type
+	 * 
+	 * @param objList  the Object list that is tested
+	 * @param typeTest a function that compares against a particular Object type
+	 * @return true if all elements are of the test type, false if one or more
+	 *         elements are not of that type.
+	 */
+	private static boolean isListType(List<Object> objList, Predicate<Object> typeTest) {
+		return !objList.stream().map(typeTest::test).collect(Collectors.toList()).contains(false);
+	}
+
+	/**
+	 * Initialize a ColumnVector with Long values.
+	 *
+	 * @param valueList a list of Long values
+	 * @param colVector the LongColumnVector that will be initialized with the Long
+	 *                  values
+	 * @param offset    the offset[rownum] value for the array
+	 * @param fieldName the field name for the Map column
+	 */
+	private static void setLongMapValues(List<Object> valueList, ColumnVector colVector, int offset, String fieldName) {
+		if (isListType(valueList, isLong) || isListType(valueList, isInteger)) {
+			LongColumnVector longVector = (LongColumnVector) colVector;
+			setLongListVector(valueList, longVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " Long values expected");
+		}
+	}
+
+	/**
+	 * Initialize a ColumnVector with Double values.
+	 *
+	 * @param valueList a list of Double values
+	 * @param colVector the DoubleColumnVector that will be initialized with the
+	 *                  Double values
+	 * @param offset    the offset[rownum] value for the array
+	 * @param fieldName the field name for the Map column
+	 */
+	private static void setDoubleMapValues(List<Object> valueList, ColumnVector colVector, int offset,
+			String fieldName) {
+		if (isListType(valueList, isDouble)) {
+			DoubleColumnVector doubleVector = (DoubleColumnVector) colVector;
+			setDoubleListVector(valueList, doubleVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " Double values expected");
+		}
+	}
+
+	/**
+	 * Initialize a ColumnVector with String values.
+	 *
+	 * @param valueList a list of String values
+	 * @param colVector the BytesColumnVector that will be initialized with the
+	 *                  String values
+	 * @param offset    the offset[rownum] value for the array
+	 * @param fieldName the field name for the Map column
+	 */
+	private static void setStringMapValues(List<Object> valueList, ColumnVector colVector, int offset,
+			String fieldName) {
+		if (isListType(valueList, isString)) {
+			BytesColumnVector doubleVector = (BytesColumnVector) colVector;
+			setBytesListVector(valueList, doubleVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " String values expected");
+		}
+	}
+
+	/**
+	 * Initialize a ColumnVector with BigDeciml values.
+	 *
+	 * @param valueList a list of BigDecimal
+	 * @param colVector the DecimalColumnVector that will be initialized with the
+	 *                  BigDecimal values
+	 * @param offset    the offset[rownum] value for the array
+	 * @param fieldName the field name for the Map column
+	 */
+	private static void setDecimalMapValues(List<Object> valueList, ColumnVector colVector, int offset,
+			String fieldName) {
+		if (isListType(valueList, isBigDecimal)) {
+			DecimalColumnVector decimalVector = (DecimalColumnVector) colVector;
+			setDecimalListVector(valueList, decimalVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " BigDecimal values expected");
+		}
+	}
+
+	/**
+	 * Initialize a ColumnVector with timestamp values.
+	 *
+	 * @param valueList a list of Date (or Timestamp) objects
+	 * @param colVector the TimestampColumnVector that will be initialized with the
+	 *                  Timestamp values
+	 * @param offset    the offset[rownum] value for the array
+	 * @param fieldName the field name for the Map column
+	 */
+	private static void setTimestampMapValues(List<Object> valueList, ColumnVector colVector, int offset,
+			String fieldName) {
+		if (isListType(valueList, isDate)) {
+			TimestampColumnVector timestampVector = (TimestampColumnVector) colVector;
+			setTimestampListVector(valueList, timestampVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " Date or Timestamp values expected");
+		}
+	}
+
+	/**
+	 * Set the MapColumn value array vector. The type for this vector is limited to
+	 * long, double, bytes (String), Decimal and Timestamp.
+	 *
+	 * @param valueList a list of Objects to initialize the Map column value array.
+	 * @param colVector the column array vector to be initialized with the map
+	 *                  values.
+	 * @param offset    the offset[rowNum] from the parent MapColumnVector
+	 * @param fieldName the name of the field for the MapColumnVector.
+	 */
+	private static void setMapValueVector(List<Object> valueList, ColumnVector colVector, int offset,
+			String fieldName) {
+		switch (colVector.type) {
+		case LONG:
+			setLongMapValues(valueList, colVector, offset, fieldName);
+			break;
+		case DOUBLE:
+			setDoubleMapValues(valueList, colVector, offset, fieldName);
+			break;
+		case BYTES:
+			setStringMapValues(valueList, colVector, offset, fieldName);
+			break;
+		case DECIMAL:
+			setDecimalMapValues(valueList, colVector, offset, fieldName);
+			break;
+		case TIMESTAMP:
+			setTimestampMapValues(valueList, colVector, offset, fieldName);
+			break;
+		default:
+			throw new RuntimeException(
+					"For field " + fieldName + " values must be long, double, String, BigDecimal or Timestamp");
+		}
+	}
+
+	/**
+	 * <p>
+	 * Initialize a MapColumnVector with Long key values.
+	 * </p>
+	 * 
+	 * @param mapSet    a set of {key, value} pairs, where the key values are Long
+	 *                  objects. The elements of this set will be used to initialize
+	 *                  the key and value array column vectors that are children of
+	 *                  the MapColumnVector.
+	 * @param mapVector the MapColumnVector. This ColumnVector has children for the
+	 *                  key and value arrays.
+	 * @param fieldName the field name for the map column vector column.
+	 * @param rowNum    the ORC file row number.
+	 */
+	private static void setLongKeyMap(Set<Map.Entry<Object, Object>> mapSet, MapColumnVector mapVector,
+			String fieldName, int rowNum) {
+		List<Object> keyValueList = mapSet.stream().map(Map.Entry::getKey).collect(Collectors.toList());
+		if (isListType(keyValueList, isLong)) {
+			LongColumnVector longVector = (LongColumnVector) mapVector.keys;
+			int offset = (int) mapVector.offsets[rowNum];
+			// set the key vector
+			setLongListVector(keyValueList, longVector, offset, fieldName);
+			// set the value vector
+			ColumnVector valueVector = mapVector.values;
+			List<Object> valueList = mapSet.stream().map(Map.Entry::getValue).collect(Collectors.toList());
+			setMapValueVector(valueList, valueVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " Long key type expected to match schema");
+		}
+	}
+
+	/**
+	 * <p>
+	 * Initialize a MapColumnVector with Double key values.
+	 * </p>
+	 * 
+	 * @param mapSet    a set of {key, value} pairs, where the key values are Double
+	 *                  objects. The elements of this set will be used to initialize
+	 *                  the key and value array column vectors that are children of
+	 *                  the MapColumnVector.
+	 * @param mapVector the MapColumnVector. This ColumnVector has children for the
+	 *                  key and value arrays.
+	 * @param fieldName the field name for the map column vector column.
+	 * @param rowNum    the ORC file row number.
+	 */
+	private static void setDoubleKeyMap(Set<Map.Entry<Object, Object>> mapSet, MapColumnVector mapVector,
+			String fieldName, int rowNum) {
+		List<Object> keyValueList = mapSet.stream().map(Map.Entry::getKey).collect(Collectors.toList());
+		if (isListType(keyValueList, isDouble)) {
+			DoubleColumnVector doubleVector = (DoubleColumnVector) mapVector.keys;
+			int offset = (int) mapVector.offsets[rowNum];
+			// set the key vector
+			setDoubleListVector(keyValueList, doubleVector, offset, fieldName);
+			// set the value vector
+			ColumnVector valueVector = mapVector.values;
+			List<Object> valueList = mapSet.stream().map(Map.Entry::getValue).collect(Collectors.toList());
+			setMapValueVector(valueList, valueVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " Long key type expected to match schema");
+		}
+	}
+
+	/**
+	 * <p>
+	 * Initialize a MapColumnVector with String key values.
+	 * </p>
+	 * 
+	 * @param mapSet    a set of {key, value} pairs, where the key values are String
+	 *                  objects. The elements of this set will be used to initialize
+	 *                  the key and value array column vectors that are children of
+	 *                  the MapColumnVector.
+	 * @param mapVector the MapColumnVector. This ColumnVector has children for the
+	 *                  key and value arrays.
+	 * @param fieldName the field name for the map column vector column.
+	 * @param rowNum    the ORC file row number.
+	 */
+	private static void setStringKeyMap(Set<Map.Entry<Object, Object>> mapSet, MapColumnVector mapVector,
+			String fieldName, int rowNum) {
+		List<Object> keyValueList = mapSet.stream().map(Map.Entry::getKey).collect(Collectors.toList());
+		if (isListType(keyValueList, isString)) {
+			BytesColumnVector byteVector = (BytesColumnVector) mapVector.keys;
+			int offset = (int) mapVector.offsets[rowNum];
+			// set the key array vector
+			setBytesListVector(keyValueList, byteVector, offset, fieldName);
+			// set the value array vector
+			ColumnVector valueVector = mapVector.values;
+			List<Object> valueList = mapSet.stream().map(Map.Entry::getValue).collect(Collectors.toList());
+			setMapValueVector(valueList, valueVector, offset, fieldName);
+		} else {
+			throw new RuntimeException("For field " + fieldName + " Long key type expected to match schema");
+		}
+	}
+
+	private static void setMapVectorParameters(MapColumnVector mapVec, int maxBatchSize, int vecLength, int rowNum) {
+		setMultiValuedVectorParameters(mapVec, vecLength, rowNum);
+		mapVec.keys.ensureSize(maxBatchSize + vecLength, true);
+		mapVec.values.ensureSize(maxBatchSize + vecLength, true);
+	}
+
+	/**
+	 * <p>
+	 * Set the Map key and value elements for a MapColumnVector
+	 * </p>
+	 * <p>
+	 * A MapColumnVector has a single ColumnVector type for each of the map key and
+	 * map values. For example, the ColumnVector for the key values could be a
+	 * BytesColumnVector (a set of String keys). The values could be a
+	 * LongColumnVector.
+	 * </p>
+	 * <p>
+	 * In the documentation there is no restriction given for the map key type. This
+	 * code limits the key types to scalar values: string, long, double.
+	 * </p>
+	 * </p>
+	 * <p>
+	 * The documentation does not limit the map value types. This code limites the
+	 * map values to the same types that are supported for ListColumnVectors: long,
+	 * double, bytes (String), Decimal and Timestamp.
+	 * </p>
+	 *
+	 * @param colVal    a HashMap object
+	 * @param typeDesc  the schema description for the MapColumnVector column
+	 * @param fieldName the field name of the MapColumnVector column
+	 * @param mapVector The parent MapColumnVector
+	 * @param rowNum    the ORC file column number.
+	 */
+	private static void setMapColumnVector(Object colVal, TypeDescription typeDesc, String fieldName,
+			MapColumnVector mapVector, int rowNum) {
+		if (colVal == null) {
+			mapVector.isNull[rowNum] = true;
+			mapVector.noNulls = false;
+		} else {
+			if (colVal instanceof HashMap) {
+				@SuppressWarnings("unchecked")
+				Map<Object, Object> rawMap = (HashMap<Object, Object>) colVal;
+				int mapLen = rawMap.size();
+				final int maxBatchSize = typeDesc.createRowBatch().getMaxSize();
+				setMapVectorParameters(mapVector, maxBatchSize, mapLen, rowNum);
+				if (ORCCommonUtils.checkMapColumnVectorTypes(mapVector)) {
+					Set<Map.Entry<Object, Object>> mapSet = rawMap.entrySet();
+					switch (mapVector.keys.type) {
+					case LONG:
+						setLongKeyMap(mapSet, mapVector, fieldName, rowNum);
+						break;
+					case DOUBLE:
+						setDoubleKeyMap(mapSet, mapVector, fieldName, rowNum);
+						break;
+					case BYTES:
+						setStringKeyMap(mapSet, mapVector, fieldName, rowNum);
+						break;
+					default: {
+						break;
+						/* This block left intentionally empty */
+					}
+					}
+				} else {
+					throw new RuntimeException(
+							"For field " + fieldName + " key types are limited to string, long and double. "
+									+ "value types are limited to long, double, String, decimal and timestamp");
+				}
+			}
+		}
+	}
+
+	/**
+	 * Set a column value in an ORC a row that will be written to the ORC file.
+	 *
+	 * @param colVal    an Object containing the values to be written to the column
+	 * @param typeDesc  the TypeDescription from the schema that defines the column
+	 * @param fieldName the column field name
+	 * @param vector    the ColumnVector that will be initialized with the values in
+	 *                  the colVal argument.
+	 * @param rowNum    the ORC file row number.
+	 */
+	public static void setColumn(Object colVal, TypeDescription typeDesc, String fieldName, ColumnVector vector,
+			int rowNum) {
+		if (colVal == null) {
+			vector.isNull[rowNum] = true;
+			vector.noNulls = false;
+		} else {
+			switch (vector.type) {
+			case LONG: {
+				if (vector instanceof DateColumnVector) {
+					DateColumnVector dateVector = (DateColumnVector) vector;
+					setDateColumnVector(colVal, fieldName, dateVector, rowNum);
+				} else {
+					LongColumnVector longVector = (LongColumnVector) vector;
+					setLongColumnVector(colVal, fieldName, longVector, rowNum);
+				}
+				break;
+			}
+			case DOUBLE: {
+				DoubleColumnVector doubleVector = (DoubleColumnVector) vector;
+				setDoubleVector(colVal, fieldName, doubleVector, rowNum);
+				break;
+			}
+			case BYTES: {
+				BytesColumnVector bytesColVector = (BytesColumnVector) vector;
+				setByteColumnVector(colVal, fieldName, bytesColVector, rowNum);
+				break;
+			}
+			case DECIMAL: {
+				DecimalColumnVector decimalVector = (DecimalColumnVector) vector;
+				setDecimalVector(colVal, fieldName, decimalVector, rowNum);
+				break;
+			}
+			case DECIMAL_64:
+				throw new RuntimeException("Field: " + fieldName + ", Decimal64ColumnVector is not supported");
+			case TIMESTAMP: {
+				TimestampColumnVector timestampVector = (TimestampColumnVector) vector;
+				setTimestampVector(colVal, fieldName, timestampVector, rowNum);
+				break;
+			}
+			case INTERVAL_DAY_TIME:
+				throw new RuntimeException("Field: " + fieldName + ", HiveIntervalDayTime is not supported");
+			case STRUCT: {
+				StructColumnVector structVector = (StructColumnVector) vector;
+//				setStructColumnVector(colVal, typeDesc, fieldName, structVector, rowNum);
+				break;
+			}
+			case LIST: {
+				ListColumnVector listVector = (ListColumnVector) vector;
+				setListColumnVector(colVal, typeDesc, fieldName, listVector, rowNum);
+				break;
+			}
+			case MAP: {
+				MapColumnVector mapVector = (MapColumnVector) vector;
+				setMapColumnVector(colVal, typeDesc, fieldName, mapVector, rowNum);
+				break;
+			}
+			case UNION: {
+				UnionColumnVector unionVector = (UnionColumnVector) vector;
+				setUnionColumnVector(colVal, typeDesc, fieldName, unionVector, rowNum);
+				break;
+			}
+			default:
+				throw new RuntimeException("setColumn: Internal error: unexpected ColumnVector subtype");
+			} // switch
+		} // else
+	} // setColumn
+
+	private static String orcExceptionMsg(String prefixMsg, String fieldName, int rowNum) {
+		return prefixMsg + fieldName + " in row " + rowNum;
+	}
+
+	public static TypeDescription detectType(Object value) {
+		TypeDescription type = null;
+		if (value != null) {
+			if (value instanceof Boolean) {
+				type = TypeDescription.createBoolean();
+			} else if (value instanceof Short) {
+				type = TypeDescription.createShort();
+			} else if (value instanceof Integer) {
+				type = TypeDescription.createInt();
+			} else if (value instanceof Long) {
+				type = TypeDescription.createLong();
+			} else if (value instanceof Timestamp) {
+				type = TypeDescription.createTimestamp();
+			} else if (value instanceof BigDecimal) {
+				type = TypeDescription.createDecimal();
+			} else if (value instanceof Byte) {
+				type = TypeDescription.createByte();
+			} else if (value instanceof Float) {
+				type = TypeDescription.createFloat();
+			} else if (value instanceof Double) {
+				type = TypeDescription.createDouble();
+			} else if (value instanceof String) {
+				type = TypeDescription.createString();
+			} else if (value instanceof Date) {
+				type = TypeDescription.createDate();
+			} else if (value instanceof byte[]) {
+				type = TypeDescription.createBinary();
+			} else {
+				throw new RuntimeException(
+						value.getClass().getName() + " is not supported for ListColumnVector columns");
+			}
+		} else {
+			type = TypeDescription.createString();
+		}
+
+		return type;
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-orc/src/test/java/org/talend/orc/ORCUtilsTest.java

@@ -0,0 +1,253 @@
+package org.talend.orc;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
+import org.apache.orc.OrcFile.WriterOptions;
+import org.apache.orc.Reader;
+import org.apache.orc.RecordReader;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class ORCUtilsTest {
+
+	private static File localFolder;
+
+	@BeforeAll
+	static void setup() throws IOException {
+		localFolder = createTempDirectory();
+	}
+
+	@AfterAll
+	static void dispose() {
+		localFolder.delete();
+	}
+
+	/**
+	 * Test that an exception is thrown if a Date type is written.
+	 *
+	 * At the time this test was written, the ORC writer did not correctly write the
+	 * date epoch value to the ORC file. The value was written as a 32-bit int,
+	 * instead of a 64 bit long. As a result, the date is incorrect. A timestamp
+	 * value should be used instead.
+	 * 
+	 * @param tempDirPath
+	 * @throws ORCFileException
+	 * @throws InterruptedException
+	 */
+	@Test
+	void testAllDataTypes() throws Throwable {
+		String filePath = localFolder.getAbsolutePath() + "/testAllTypes.orc";
+		writeData(filePath);
+
+		readAndCheckData(filePath);
+
+	}
+
+	@Test
+	void testDetectType() {
+		Assertions.assertEquals(TypeDescription.Category.BOOLEAN, ORCWriteUtils.detectType(true).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.SHORT,
+				ORCWriteUtils.detectType(Short.valueOf("1")).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.INT, ORCWriteUtils.detectType(1).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.LONG, ORCWriteUtils.detectType(1L).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.TIMESTAMP,
+				ORCWriteUtils.detectType(new Timestamp(System.currentTimeMillis())).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.DECIMAL,
+				ORCWriteUtils.detectType(new BigDecimal("1")).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.BYTE,
+				ORCWriteUtils.detectType(Byte.valueOf("1")).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.FLOAT, ORCWriteUtils.detectType(1.0f).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.DOUBLE, ORCWriteUtils.detectType(1.0).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.STRING, ORCWriteUtils.detectType("test").getCategory());
+		Assertions.assertEquals(TypeDescription.Category.DATE, ORCWriteUtils.detectType(new Date()).getCategory());
+		Assertions.assertEquals(TypeDescription.Category.BINARY,
+				ORCWriteUtils.detectType("test".getBytes()).getCategory());
+	}
+
+	private void writeData(String filePath) throws Throwable {
+		TypeDescription schema = TypeDescription.createStruct();
+		schema.addField("t_boolean", TypeDescription.createBoolean());
+		schema.addField("t_byte", TypeDescription.createByte());
+		schema.addField("t_bytes", TypeDescription.createBinary());
+		schema.addField("t_char", TypeDescription.createChar());
+		schema.addField("t_date", TypeDescription.createDate());
+		schema.addField("t_ts", TypeDescription.createTimestamp());
+		schema.addField("t_double", TypeDescription.createDouble());
+		schema.addField("t_float", TypeDescription.createFloat());
+		schema.addField("t_decimal", TypeDescription.createDecimal().withPrecision(18).withScale(5));
+		schema.addField("t_int", TypeDescription.createInt());
+		schema.addField("t_long", TypeDescription.createLong());
+		schema.addField("t_short", TypeDescription.createShort());
+		schema.addField("t_string", TypeDescription.createString());
+		schema.addField("t_list", TypeDescription.createList(TypeDescription.createString()));
+
+		WriterOptions writerOption = OrcFile.writerOptions(new Configuration()) //
+				.overwrite(true) //
+				.compress(CompressionKind.valueOf("ZLIB")).setSchema(schema); //
+
+		Writer writer = OrcFile.createWriter(new Path(filePath), writerOption);
+		VectorizedRowBatch batch = schema.createRowBatch(100);
+		for (int r = 0; r < 1000; ++r) {
+			int row = batch.size++;
+			for (int i = 0; i < batch.cols.length; i++) {
+				ColumnVector vector = batch.cols[i];
+				TypeDescription type = schema.getChildren().get(i);
+				switch (vector.type) {
+				case BYTES:
+					if (type.getCategory() == TypeDescription.Category.BINARY) {
+						ORCWriteUtils.setColumn(("this is byte[] " + r).getBytes(), null, "t_bytes", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.STRING) {
+						if(r==666) {
+							ORCWriteUtils.setColumn(null, null, "t_string", vector, row);
+						}else {
+							ORCWriteUtils.setColumn(("this is String " + r), null, "t_string", vector, row);
+						}
+					} else if (type.getCategory() == TypeDescription.Category.CHAR) {
+						ORCWriteUtils.setColumn("talend".charAt(r % 6), null, "t_char", vector, row);
+					} else {
+						throw new RuntimeException(type.getCategory() + " is not supported as BYTES vector");
+					}
+					break;
+				case DECIMAL:
+					ORCWriteUtils.setColumn(new BigDecimal(r + ".12345"), null, "t_decimal", vector, row);
+					break;
+				case DOUBLE:
+					if (type.getCategory() == TypeDescription.Category.DOUBLE) {
+						ORCWriteUtils.setColumn(r + 0.123, null, "t_double", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.FLOAT) {
+						ORCWriteUtils.setColumn(r + 0.456f, null, "t_float", vector, row);
+					} else {
+						throw new RuntimeException(type.getCategory() + " is not supported as DOUBLE vector");
+					}
+					break;
+				case LONG:
+					if (type.getCategory() == TypeDescription.Category.BOOLEAN) {
+						ORCWriteUtils.setColumn(true, null, "t_boolean", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.BYTE) {
+						ORCWriteUtils.setColumn((byte)(r % 128), null, "t_byte", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.INT) {
+						ORCWriteUtils.setColumn(r, null, "t_int", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.SHORT) {
+						ORCWriteUtils.setColumn((short)(r % 256), null, "t_short", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.LONG) {
+						ORCWriteUtils.setColumn(r * 1000L, null, "t_long", vector, row);
+					} else if (type.getCategory() == TypeDescription.Category.DATE) {
+						Date d = new Date(1633687854031L);
+						ORCWriteUtils.setColumn(d, null, "t_date", vector, row);
+					} else {
+						throw new RuntimeException(type.getCategory() + " is not supported as LONG vector");
+					}
+					break;
+				case TIMESTAMP:
+					Timestamp ts = new java.sql.Timestamp(1633687854031L);
+					ts.setNanos(123456789);
+					ORCWriteUtils.setColumn(ts, null, "t_ts", vector, row);
+					break;
+				case LIST:
+					List<String> values = new ArrayList<>();
+					values.add("v1_" + r);
+					values.add("v2_" + r);
+					values.add("v3_" + r);
+					ORCWriteUtils.setColumn(values, ORCWriteUtils.detectType("v1_" + r), "t_ list", vector, row);
+					break;
+				default:
+					throw new RuntimeException(vector.type + " is not supported");
+
+				}
+			}
+			if (batch.size == batch.getMaxSize()) {
+				writer.addRowBatch(batch);
+				batch.reset();
+			}
+		}
+		if (batch.size != 0) {
+			writer.addRowBatch(batch);
+		}
+
+		writer.close();
+	}
+
+	private void readAndCheckData(String filePath) throws Throwable {
+
+		Reader reader = OrcFile.createReader(new Path(filePath), OrcFile.readerOptions(new Configuration()));
+		TypeDescription schema = reader.getSchema();
+		VectorizedRowBatch batch = schema.createRowBatch();
+		RecordReader rowIterator = reader.rows(reader.options().schema(schema));
+		int nuberLine = 0;
+		List<Object> nb_500 = new ArrayList<>();
+		List<Object> nb_666 = new ArrayList<>();
+		while (rowIterator.nextBatch(batch)) {
+			ColumnVector[] colVectors = batch.cols;
+			for (int row = 0; row < batch.size; ++row) {
+				nuberLine++;
+				for (String columnName : schema.getFieldNames()) {
+					ColumnVector colVector = colVectors[schema.getFieldNames().indexOf(columnName)];
+					int colRow = colVector.isRepeating ? 0 : row;
+					Object value = ORCReadUtils.readColumnByName(batch, columnName, schema, colRow);
+					if (nuberLine == 500) {
+						nb_500.add(value);
+					}else if (nuberLine == 667) {
+						nb_666.add(value);
+					}
+				}
+			}
+		}
+		Assertions.assertEquals(true, nb_500.get(0));
+		Assertions.assertEquals(Byte.valueOf("115"), nb_500.get(1));
+		Assertions.assertEquals("this is byte[] 499", new String((byte[]) nb_500.get(2)));
+		Assertions.assertEquals("a", nb_500.get(3));
+		Date t_date = (Date) nb_500.get(4);
+		Assertions.assertEquals((1633687854000L/86400000), t_date.getTime()/86400000);
+		Timestamp t_ts = (Timestamp) nb_500.get(5);
+		Assertions.assertEquals(1633687854123L, t_ts.getTime());
+		Assertions.assertEquals(123456789, t_ts.getNanos());
+		Assertions.assertEquals(499.123, nb_500.get(6));
+		Assertions.assertEquals(499.456f, (((float) nb_500.get(7)) * 1000) / 1000f);
+		Assertions.assertEquals(new BigDecimal("499.12345"), nb_500.get(8));
+		Assertions.assertEquals(499, nb_500.get(9));
+		Assertions.assertEquals(499000L, nb_500.get(10));
+		Assertions.assertEquals(Short.valueOf("243"), nb_500.get(11));
+		Assertions.assertEquals("this is String 499", nb_500.get(12));
+		Assertions.assertArrayEquals(Arrays.asList("v1_499", "v2_499", "v3_499").toArray(),
+				((List<Object>) nb_500.get(13)).toArray());
+		
+		//NB_LINE 666
+		Assertions.assertNull( nb_666.get(12));
+
+		rowIterator.close();
+
+	}
+
+	public static File createTempDirectory() throws IOException {
+		final File temp;
+
+		temp = File.createTempFile("temp", Long.toString(System.nanoTime()));
+		if (!temp.delete()) {
+			throw new IOException("Could not delete temp file: " + temp.getAbsolutePath());
+		}
+
+		if (!temp.mkdir()) {
+			throw new IOException("Could not create temp directory: " + temp.getAbsolutePath());
+		}
+
+		return temp;
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/pom.xml

@@ -0,0 +1,98 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>org.talend.components</groupId>
+	<artifactId>talend-parquet</artifactId>
+	<version>1.3</version>
+
+	<properties>
+		<parquet.version>1.10.1</parquet.version>
+		<hadoop.version>3.2.2</hadoop.version>
+		<jodd.version>6.0.1</jodd.version>
+		<hamcrest.version>1.3</hamcrest.version>
+		<junit.version>4.13.2</junit.version>
+		<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
+	</properties>
+	
+	<distributionManagement>
+		<snapshotRepository>
+			<id>talend_nexus_deployment</id>
+			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
+			<snapshots>
+				<enabled>true</enabled>
+			</snapshots>
+			<releases>
+				<enabled>false</enabled>
+			</releases>
+		</snapshotRepository>
+		<repository>
+			<id>talend_nexus_deployment</id>
+			<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+		</repository>
+	</distributionManagement>
+	
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.parquet</groupId>
+			<artifactId>parquet-hadoop</artifactId>
+			<version>${parquet.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-client</artifactId>
+			<version>${hadoop.version}</version>
+			<scope>provided</scope>
+			<exclusions>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-log4j12</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+		    <groupId>org.jodd</groupId>
+		    <artifactId>jodd-util</artifactId>
+		    <version>${jodd.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>${junit.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest-library</artifactId>
+			<version>${hamcrest.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest-library</artifactId>
+			<version>${hamcrest.version}</version>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.8.0</version>
+				<configuration>
+					<source>1.8</source>
+					<target>1.8</target>
+				</configuration>
+			</plugin>
+
+		</plugins>
+	</build>
+</project>

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/Group.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data;
+
+import org.talend.parquet.data.simple.NanoTime;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract public class Group extends GroupValueSource {
+	private static final Logger LOG = LoggerFactory.getLogger(Group.class);
+
+	public void add(String field, int value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, long value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, float value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, double value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, String value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, NanoTime value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, boolean value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, Binary value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public void add(String field, Group value) {
+		add(getType().getFieldIndex(field), value);
+	}
+
+	public Group addGroup(String field) {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("add group {} to {}", field, getType().getName());
+		}
+		return addGroup(getType().getFieldIndex(field));
+	}
+
+	@Override
+	public Group getGroup(String field, int index) {
+		return getGroup(getType().getFieldIndex(field), index);
+	}
+
+	abstract public void add(int fieldIndex, int value);
+
+	abstract public void add(int fieldIndex, long value);
+
+	abstract public void add(int fieldIndex, String value);
+
+	abstract public void add(int fieldIndex, boolean value);
+
+	abstract public void add(int fieldIndex, NanoTime value);
+
+	abstract public void add(int fieldIndex, Binary value);
+
+	abstract public void add(int fieldIndex, float value);
+
+	abstract public void add(int fieldIndex, double value);
+
+	abstract public void add(int fieldIndex, Group value);
+
+	abstract public Group addGroup(int fieldIndex);
+
+	@Override
+	abstract public Group getGroup(int fieldIndex, int index);
+
+	public Group asGroup() {
+		return this;
+	}
+
+	public Group append(String fieldName, int value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	public Group append(String fieldName, float value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	public Group append(String fieldName, double value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	public Group append(String fieldName, long value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	public Group append(String fieldName, NanoTime value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	public Group append(String fieldName, String value) {
+		add(fieldName, Binary.fromString(value));
+		return this;
+	}
+
+	public Group append(String fieldName, boolean value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	public Group append(String fieldName, Binary value) {
+		add(fieldName, value);
+		return this;
+	}
+
+	abstract public void writeValue(int field, int index, RecordConsumer recordConsumer);
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/GroupFactory.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data;
+
+abstract public class GroupFactory {
+
+	abstract public Group newGroup();
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/GroupValueSource.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.GroupType;
+
+abstract public class GroupValueSource {
+
+	public int getFieldRepetitionCount(String field) {
+		return getFieldRepetitionCount(getType().getFieldIndex(field));
+	}
+
+	public GroupValueSource getGroup(String field, int index) {
+		return getGroup(getType().getFieldIndex(field), index);
+	}
+
+	public String getString(String field, int index) {
+		return getString(getType().getFieldIndex(field), index);
+	}
+
+	public int getInteger(String field, int index) {
+		return getInteger(getType().getFieldIndex(field), index);
+	}
+
+	public long getLong(String field, int index) {
+		return getLong(getType().getFieldIndex(field), index);
+	}
+
+	public double getDouble(String field, int index) {
+		return getDouble(getType().getFieldIndex(field), index);
+	}
+
+	public float getFloat(String field, int index) {
+		return getFloat(getType().getFieldIndex(field), index);
+	}
+
+	public boolean getBoolean(String field, int index) {
+		return getBoolean(getType().getFieldIndex(field), index);
+	}
+
+	public Binary getBinary(String field, int index) {
+		return getBinary(getType().getFieldIndex(field), index);
+	}
+
+	public Binary getInt96(String field, int index) {
+		return getInt96(getType().getFieldIndex(field), index);
+	}
+
+	abstract public int getFieldRepetitionCount(int fieldIndex);
+
+	abstract public GroupValueSource getGroup(int fieldIndex, int index);
+
+	abstract public String getString(int fieldIndex, int index);
+
+	abstract public Integer getInteger(int fieldIndex, int index);
+
+	abstract public Long getLong(int fieldIndex, int index);
+
+	abstract public Double getDouble(int fieldIndex, int index);
+
+	abstract public Float getFloat(int fieldIndex, int index);
+
+	abstract public Boolean getBoolean(int fieldIndex, int index);
+
+	abstract public Binary getBinary(int fieldIndex, int index);
+
+	abstract public Binary getInt96(int fieldIndex, int index);
+
+	abstract public String getValueToString(int fieldIndex, int index);
+
+	abstract public GroupType getType();
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/GroupWriter.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data;
+
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.Type;
+
+public class GroupWriter {
+
+	private final RecordConsumer recordConsumer;
+	private final GroupType schema;
+
+	public GroupWriter(RecordConsumer recordConsumer, GroupType schema) {
+		this.recordConsumer = recordConsumer;
+		this.schema = schema;
+	}
+
+	public void write(Group group) {
+		recordConsumer.startMessage();
+		writeGroup(group, schema);
+		recordConsumer.endMessage();
+	}
+
+	private void writeGroup(Group group, GroupType type) {
+		int fieldCount = type.getFieldCount();
+		for (int field = 0; field < fieldCount; ++field) {
+			int valueCount = group.getFieldRepetitionCount(field);
+			if (valueCount > 0) {
+				Type fieldType = type.getType(field);
+				String fieldName = fieldType.getName();
+				recordConsumer.startField(fieldName, field);
+				for (int index = 0; index < valueCount; ++index) {
+					if (fieldType.isPrimitive()) {
+						group.writeValue(field, index, recordConsumer);
+					} else {
+						recordConsumer.startGroup();
+						writeGroup(group.getGroup(field, index), fieldType.asGroupType());
+						recordConsumer.endGroup();
+					}
+				}
+				recordConsumer.endField(fieldName, field);
+			}
+		}
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/BinaryValue.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class BinaryValue extends Primitive {
+
+	private final Binary binary;
+
+	public BinaryValue(Binary binary) {
+		this.binary = binary;
+	}
+
+	@Override
+	public Binary getBinary() {
+		return binary;
+	}
+
+	@Override
+	public String getString() {
+		return binary.toStringUsingUTF8();
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addBinary(binary);
+	}
+
+	@Override
+	public String toString() {
+		return getString();
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/BooleanValue.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class BooleanValue extends Primitive {
+
+	private final boolean bool;
+
+	public BooleanValue(boolean bool) {
+		this.bool = bool;
+	}
+
+	@Override
+	public String toString() {
+		return String.valueOf(bool);
+	}
+
+	@Override
+	public boolean getBoolean() {
+		return bool;
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addBoolean(bool);
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/DoubleValue.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class DoubleValue extends Primitive {
+
+	private final double value;
+
+	public DoubleValue(double value) {
+		this.value = value;
+	}
+
+	@Override
+	public double getDouble() {
+		return value;
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addDouble(value);
+	}
+
+	@Override
+	public String toString() {
+		return String.valueOf(value);
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/FloatValue.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class FloatValue extends Primitive {
+
+	private final float value;
+
+	public FloatValue(float value) {
+		this.value = value;
+	}
+
+	@Override
+	public float getFloat() {
+		return value;
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addFloat(value);
+	}
+
+	@Override
+	public String toString() {
+		return String.valueOf(value);
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/Int96Value.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class Int96Value extends Primitive {
+
+	private final Binary value;
+
+	public Int96Value(Binary value) {
+		this.value = value;
+	}
+
+	@Override
+	public Binary getInt96() {
+		return value;
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addBinary(value);
+	}
+
+	@Override
+	public String toString() {
+		return "Int96Value{" + String.valueOf(value) + "}";
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/IntegerValue.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class IntegerValue extends Primitive {
+
+	private final int value;
+
+	public IntegerValue(int value) {
+		this.value = value;
+	}
+
+	@Override
+	public String toString() {
+		return String.valueOf(value);
+	}
+
+	@Override
+	public int getInteger() {
+		return value;
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addInteger(value);
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/LongValue.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class LongValue extends Primitive {
+
+	private final long value;
+
+	public LongValue(long value) {
+		this.value = value;
+	}
+
+	@Override
+	public String toString() {
+		return String.valueOf(value);
+	}
+
+	@Override
+	public long getLong() {
+		return value;
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addLong(value);
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/NanoTime.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+
+public class NanoTime extends Primitive {
+	private final int julianDay;
+	private final long timeOfDayNanos;
+
+	public static NanoTime fromBinary(Binary bytes) {
+		Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes");
+		ByteBuffer buf = bytes.toByteBuffer();
+		buf.order(ByteOrder.LITTLE_ENDIAN);
+		long timeOfDayNanos = buf.getLong();
+		int julianDay = buf.getInt();
+		return new NanoTime(julianDay, timeOfDayNanos);
+	}
+
+	public static NanoTime fromInt96(Int96Value int96) {
+		ByteBuffer buf = int96.getInt96().toByteBuffer();
+		return new NanoTime(buf.getInt(), buf.getLong());
+	}
+
+	public NanoTime(int julianDay, long timeOfDayNanos) {
+		this.julianDay = julianDay;
+		this.timeOfDayNanos = timeOfDayNanos;
+	}
+
+	public int getJulianDay() {
+		return julianDay;
+	}
+
+	public long getTimeOfDayNanos() {
+		return timeOfDayNanos;
+	}
+
+	public Binary toBinary() {
+		ByteBuffer buf = ByteBuffer.allocate(12);
+		buf.order(ByteOrder.LITTLE_ENDIAN);
+		buf.putLong(timeOfDayNanos);
+		buf.putInt(julianDay);
+		buf.flip();
+		return Binary.fromConstantByteBuffer(buf);
+	}
+
+	public Int96Value toInt96() {
+		return new Int96Value(toBinary());
+	}
+
+	@Override
+	public void writeValue(RecordConsumer recordConsumer) {
+		recordConsumer.addBinary(toBinary());
+	}
+
+	@Override
+	public String toString() {
+		return "NanoTime{julianDay=" + julianDay + ", timeOfDayNanos=" + timeOfDayNanos + "}";
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/Primitive.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+
+public abstract class Primitive {
+
+	public String getString() {
+		throw new UnsupportedOperationException();
+	}
+
+	public int getInteger() {
+		throw new UnsupportedOperationException();
+	}
+
+	public long getLong() {
+		throw new UnsupportedOperationException();
+	}
+
+	public boolean getBoolean() {
+		throw new UnsupportedOperationException();
+	}
+
+	public Binary getBinary() {
+		throw new UnsupportedOperationException();
+	}
+
+	public Binary getInt96() {
+		throw new UnsupportedOperationException();
+	}
+
+	public float getFloat() {
+		throw new UnsupportedOperationException();
+	}
+
+	public double getDouble() {
+		throw new UnsupportedOperationException();
+	}
+
+	abstract public void writeValue(RecordConsumer recordConsumer);
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/SimpleGroup.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.talend.parquet.data.Group;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.Type;
+
+public class SimpleGroup extends Group {
+
+	private final GroupType schema;
+	private final List<Object>[] data;
+
+	@SuppressWarnings("unchecked")
+	public SimpleGroup(GroupType schema) {
+		this.schema = schema;
+		this.data = new List[schema.getFields().size()];
+		for (int i = 0; i < schema.getFieldCount(); i++) {
+			this.data[i] = new ArrayList<>();
+		}
+	}
+
+	@Override
+	public String toString() {
+		return toString("");
+	}
+
+	private StringBuilder appendToString(StringBuilder builder, String indent) {
+		int i = 0;
+		for (Type field : schema.getFields()) {
+			String name = field.getName();
+			List<Object> values = data[i];
+			++i;
+			if (values != null && !values.isEmpty()) {
+				for (Object value : values) {
+					builder.append(indent).append(name);
+					if (value == null) {
+						builder.append(": NULL\n");
+					} else if (value instanceof Group) {
+						builder.append('\n');
+						((SimpleGroup) value).appendToString(builder, indent + "  ");
+					} else {
+						builder.append(": ").append(value.toString()).append('\n');
+					}
+				}
+			}
+		}
+		return builder;
+	}
+
+	public String toString(String indent) {
+		StringBuilder builder = new StringBuilder();
+		appendToString(builder, indent);
+		return builder.toString();
+	}
+
+	@Override
+	public Group addGroup(int fieldIndex) {
+		SimpleGroup g = new SimpleGroup(schema.getType(fieldIndex).asGroupType());
+		add(fieldIndex, g);
+		return g;
+	}
+
+	@Override
+	public Group getGroup(int fieldIndex, int index) {
+		return (Group) getValue(fieldIndex, index);
+	}
+
+	private Object getValue(int fieldIndex, int index) {
+		List<Object> list;
+		try {
+			list = data[fieldIndex];
+		} catch (IndexOutOfBoundsException e) {
+			throw new RuntimeException(
+					"not found " + fieldIndex + "(" + schema.getFieldName(fieldIndex) + ") in group:\n" + this);
+		}
+		try {
+			if(list == null || list.isEmpty()) {
+				return null;
+			}
+			return list.get(index);
+		} catch (IndexOutOfBoundsException e) {
+			throw new RuntimeException("not found " + fieldIndex + "(" + schema.getFieldName(fieldIndex)
+					+ ") element number " + index + " in group:\n" + this);
+		}
+	}
+
+	private void add(int fieldIndex, Primitive value) {
+		Type type = schema.getType(fieldIndex);
+		List<Object> list = data[fieldIndex];
+		if (!type.isRepetition(Type.Repetition.REPEATED) && !list.isEmpty()) {
+			throw new IllegalStateException(
+					"field " + fieldIndex + " (" + type.getName() + ") can not have more than one value: " + list);
+		}
+		list.add(value);
+	}
+
+	@Override
+	public int getFieldRepetitionCount(int fieldIndex) {
+		List<Object> list = data[fieldIndex];
+		return list == null ? 0 : list.size();
+	}
+
+	@Override
+	public String getValueToString(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return String.valueOf(value);
+	}
+
+	@Override
+	public String getString(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((BinaryValue) value).getString();
+	}
+
+	@Override
+	public Integer getInteger(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((IntegerValue)value).getInteger();
+	}
+
+	@Override
+	public Long getLong(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((LongValue)value).getLong();
+	}
+
+	@Override
+	public Double getDouble(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((DoubleValue)value).getDouble();
+	}
+
+	@Override
+	public Float getFloat(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((FloatValue)value).getFloat();
+	}
+
+	@Override
+	public Boolean getBoolean(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((BooleanValue) value).getBoolean();
+	}
+
+	@Override
+	public Binary getBinary(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((BinaryValue) value).getBinary();
+	}
+
+	public NanoTime getTimeNanos(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return NanoTime.fromInt96((Int96Value) value);
+	}
+
+	@Override
+	public Binary getInt96(int fieldIndex, int index) {
+		Object value = getValue(fieldIndex, index);
+		if(value == null) {
+			return null;
+		}
+		return ((Int96Value) value).getInt96();
+	}
+
+	@Override
+	public void add(int fieldIndex, int value) {
+		add(fieldIndex, new IntegerValue(value));
+	}
+
+	@Override
+	public void add(int fieldIndex, long value) {
+		add(fieldIndex, new LongValue(value));
+	}
+
+	@Override
+	public void add(int fieldIndex, String value) {
+		add(fieldIndex, new BinaryValue(Binary.fromString(value)));
+	}
+
+	@Override
+	public void add(int fieldIndex, NanoTime value) {
+		add(fieldIndex, value.toInt96());
+	}
+
+	@Override
+	public void add(int fieldIndex, boolean value) {
+		add(fieldIndex, new BooleanValue(value));
+	}
+
+	@Override
+	public void add(int fieldIndex, Binary value) {
+		switch (getType().getType(fieldIndex).asPrimitiveType().getPrimitiveTypeName()) {
+		case BINARY:
+		case FIXED_LEN_BYTE_ARRAY:
+			add(fieldIndex, new BinaryValue(value));
+			break;
+		case INT96:
+			add(fieldIndex, new Int96Value(value));
+			break;
+		default:
+			throw new UnsupportedOperationException(
+					getType().asPrimitiveType().getName() + " not supported for Binary");
+		}
+	}
+
+	@Override
+	public void add(int fieldIndex, float value) {
+		add(fieldIndex, new FloatValue(value));
+	}
+
+	@Override
+	public void add(int fieldIndex, double value) {
+		add(fieldIndex, new DoubleValue(value));
+	}
+
+	@Override
+	public void add(int fieldIndex, Group value) {
+		data[fieldIndex].add(value);
+	}
+
+	@Override
+	public GroupType getType() {
+		return schema;
+	}
+
+	@Override
+	public void writeValue(int field, int index, RecordConsumer recordConsumer) {
+		((Primitive) getValue(field, index)).writeValue(recordConsumer);
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/SimpleGroupFactory.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple;
+
+import org.talend.parquet.data.Group;
+import org.talend.parquet.data.GroupFactory;
+import org.apache.parquet.schema.MessageType;
+
+public class SimpleGroupFactory extends GroupFactory {
+
+	private final MessageType schema;
+
+	public SimpleGroupFactory(MessageType schema) {
+		this.schema = schema;
+	}
+
+	@Override
+	public Group newGroup() {
+		return new SimpleGroup(schema);
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/convert/GroupRecordConverter.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple.convert;
+
+import org.talend.parquet.data.Group;
+import org.talend.parquet.data.simple.SimpleGroupFactory;
+import org.apache.parquet.io.api.GroupConverter;
+import org.apache.parquet.io.api.RecordMaterializer;
+import org.apache.parquet.schema.MessageType;
+
+public class GroupRecordConverter extends RecordMaterializer<Group> {
+
+	private final SimpleGroupFactory simpleGroupFactory;
+
+	private SimpleGroupConverter root;
+
+	public GroupRecordConverter(MessageType schema) {
+		this.simpleGroupFactory = new SimpleGroupFactory(schema);
+		this.root = new SimpleGroupConverter(null, 0, schema) {
+			@Override
+			public void start() {
+				this.current = simpleGroupFactory.newGroup();
+			}
+
+			@Override
+			public void end() {
+			}
+		};
+	}
+
+	@Override
+	public Group getCurrentRecord() {
+		return root.getCurrentRecord();
+	}
+
+	@Override
+	public GroupConverter getRootConverter() {
+		return root;
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/convert/SimpleGroupConverter.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple.convert;
+
+import org.talend.parquet.data.Group;
+import org.apache.parquet.io.api.Converter;
+import org.apache.parquet.io.api.GroupConverter;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.Type;
+
+class SimpleGroupConverter extends GroupConverter {
+	private final SimpleGroupConverter parent;
+	private final int index;
+	protected Group current;
+	private Converter[] converters;
+
+	SimpleGroupConverter(SimpleGroupConverter parent, int index, GroupType schema) {
+		this.parent = parent;
+		this.index = index;
+
+		converters = new Converter[schema.getFieldCount()];
+
+		for (int i = 0; i < converters.length; i++) {
+			final Type type = schema.getType(i);
+			if (type.isPrimitive()) {
+				converters[i] = new SimplePrimitiveConverter(this, i);
+			} else {
+				converters[i] = new SimpleGroupConverter(this, i, type.asGroupType());
+			}
+
+		}
+	}
+
+	@Override
+	public void start() {
+		current = parent.getCurrentRecord().addGroup(index);
+	}
+
+	@Override
+	public Converter getConverter(int fieldIndex) {
+		return converters[fieldIndex];
+	}
+
+	@Override
+	public void end() {
+	}
+
+	public Group getCurrentRecord() {
+		return current;
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/data/simple/convert/SimplePrimitiveConverter.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.data.simple.convert;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.PrimitiveConverter;
+
+class SimplePrimitiveConverter extends PrimitiveConverter {
+
+	private final SimpleGroupConverter parent;
+	private final int index;
+
+	SimplePrimitiveConverter(SimpleGroupConverter parent, int index) {
+		this.parent = parent;
+		this.index = index;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @see org.apache.parquet.io.api.PrimitiveConverter#addBinary(Binary)
+	 */
+	@Override
+	public void addBinary(Binary value) {
+		parent.getCurrentRecord().add(index, value);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @see org.apache.parquet.io.api.PrimitiveConverter#addBoolean(boolean)
+	 */
+	@Override
+	public void addBoolean(boolean value) {
+		parent.getCurrentRecord().add(index, value);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @see org.apache.parquet.io.api.PrimitiveConverter#addDouble(double)
+	 */
+	@Override
+	public void addDouble(double value) {
+		parent.getCurrentRecord().add(index, value);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @see org.apache.parquet.io.api.PrimitiveConverter#addFloat(float)
+	 */
+	@Override
+	public void addFloat(float value) {
+		parent.getCurrentRecord().add(index, value);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @see org.apache.parquet.io.api.PrimitiveConverter#addInt(int)
+	 */
+	@Override
+	public void addInt(int value) {
+		parent.getCurrentRecord().add(index, value);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @see org.apache.parquet.io.api.PrimitiveConverter#addLong(long)
+	 */
+	@Override
+	public void addLong(long value) {
+		parent.getCurrentRecord().add(index, value);
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/hadoop/TalendGroupReadSupport.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.hadoop;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.hadoop.api.ReadSupport;
+import org.apache.parquet.io.api.RecordMaterializer;
+import org.apache.parquet.schema.MessageType;
+import org.talend.parquet.data.Group;
+import org.talend.parquet.data.simple.convert.GroupRecordConverter;
+
+public class TalendGroupReadSupport extends ReadSupport<Group> {
+
+	@Override
+	public org.apache.parquet.hadoop.api.ReadSupport.ReadContext init(Configuration configuration,
+			Map<String, String> keyValueMetaData, MessageType fileSchema) {
+		String partialSchemaString = configuration.get(ReadSupport.PARQUET_READ_SCHEMA);
+		MessageType requestedProjection = getSchemaForRead(fileSchema, partialSchemaString);
+		return new ReadContext(requestedProjection);
+	}
+
+	@Override
+	public RecordMaterializer<Group> prepareForRead(Configuration configuration, Map<String, String> keyValueMetaData,
+			MessageType fileSchema, org.apache.parquet.hadoop.api.ReadSupport.ReadContext readContext) {
+		return new GroupRecordConverter(readContext.getRequestedSchema());
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/hadoop/TalendGroupWriteSupport.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.hadoop;
+
+import static org.apache.parquet.schema.MessageTypeParser.parseMessageType;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.MessageType;
+import org.talend.parquet.data.Group;
+import org.talend.parquet.data.GroupWriter;
+
+public class TalendGroupWriteSupport extends WriteSupport<Group> {
+
+	public static final String PARQUET_SCHEMA = "parquet.talend.schema";
+
+	public static void setSchema(MessageType schema, Configuration configuration) {
+		configuration.set(PARQUET_SCHEMA, schema.toString());
+	}
+
+	public static MessageType getSchema(Configuration configuration) {
+		return parseMessageType(Objects.requireNonNull(configuration.get(PARQUET_SCHEMA), PARQUET_SCHEMA));
+	}
+
+	private MessageType schema;
+	private GroupWriter groupWriter;
+	private Map<String, String> extraMetaData;
+
+	public TalendGroupWriteSupport() {
+		this(null, new HashMap<String, String>());
+	}
+
+	TalendGroupWriteSupport(MessageType schema) {
+		this(schema, new HashMap<String, String>());
+	}
+
+	TalendGroupWriteSupport(MessageType schema, Map<String, String> extraMetaData) {
+		this.schema = schema;
+		this.extraMetaData = extraMetaData;
+	}
+
+	@Override
+	public String getName() {
+		return "Talend";
+	}
+
+	@Override
+	public org.apache.parquet.hadoop.api.WriteSupport.WriteContext init(Configuration configuration) {
+		// if present, prefer the schema passed to the constructor
+		if (schema == null) {
+			schema = getSchema(configuration);
+		}
+		return new WriteContext(schema, this.extraMetaData);
+	}
+
+	@Override
+	public void prepareForWrite(RecordConsumer recordConsumer) {
+		groupWriter = new GroupWriter(recordConsumer, schema);
+	}
+
+	@Override
+	public void write(Group record) {
+		groupWriter.write(record);
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/hadoop/TalendInputFormat.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.hadoop;
+
+import org.apache.parquet.hadoop.ParquetInputFormat;
+import org.talend.parquet.data.Group;
+
+/**
+ * Example input format to read Parquet files
+ *
+ * This Input format uses a rather inefficient data model but works
+ * independently of higher level abstractions.
+ */
+public class TalendInputFormat extends ParquetInputFormat<Group> {
+
+	public TalendInputFormat() {
+		super(TalendGroupReadSupport.class);
+	}
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/hadoop/TalendOutputFormat.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.hadoop;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.parquet.hadoop.ParquetOutputFormat;
+import org.apache.parquet.hadoop.util.ContextUtil;
+import org.apache.parquet.schema.MessageType;
+import org.talend.parquet.data.Group;
+
+/**
+ * An example output format
+ *
+ * must be provided the schema up front
+ * 
+ * @see TalendOutputFormat#setSchema(Job, MessageType)
+ * @see TalendGroupWriteSupport#PARQUET_SCHEMA
+ */
+public class TalendOutputFormat extends ParquetOutputFormat<Group> {
+
+	/**
+	 * set the schema being written to the job conf
+	 * 
+	 * @param job    a job
+	 * @param schema the schema of the data
+	 */
+	public static void setSchema(Job job, MessageType schema) {
+		TalendGroupWriteSupport.setSchema(schema, ContextUtil.getConfiguration(job));
+	}
+
+	/**
+	 * retrieve the schema from the conf
+	 * 
+	 * @param job a job
+	 * @return the schema
+	 */
+	public static MessageType getSchema(Job job) {
+		return TalendGroupWriteSupport.getSchema(ContextUtil.getConfiguration(job));
+	}
+
+	public TalendOutputFormat() {
+		super(new TalendGroupWriteSupport());
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/hadoop/TalendParquetWriter.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.talend.parquet.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.schema.MessageType;
+import org.talend.parquet.data.Group;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An example file writer class.
+ */
+public class TalendParquetWriter extends ParquetWriter<Group> {
+
+	/**
+	 * Creates a Builder for configuring ParquetWriter with the example object
+	 *
+	 * @param file the output file to create
+	 * @return a {@link Builder} to create a {@link ParquetWriter}
+	 */
+	public static Builder builder(Path file) {
+		return new Builder(file);
+	}
+
+	/**
+	 * Creates a Builder for configuring ParquetWriter with the example object
+	 *
+	 * @param file the output file to create
+	 * @return a {@link Builder} to create a {@link ParquetWriter}
+	 */
+	public static Builder builder(OutputFile file) {
+		return new Builder(file);
+	}
+
+	/**
+	 * Create a new {@link TalendParquetWriter}.
+	 *
+	 * @param file                 The file name to write to.
+	 * @param writeSupport         The schema to write with.
+	 * @param compressionCodecName Compression code to use, or
+	 *                             CompressionCodecName.UNCOMPRESSED
+	 * @param blockSize            the block size threshold.
+	 * @param pageSize             See parquet write up. Blocks are subdivided into
+	 *                             pages for alignment and other purposes.
+	 * @param enableDictionary     Whether to use a dictionary to compress columns.
+	 * @param conf                 The Configuration to use.
+	 * @throws IOException
+	 */
+	TalendParquetWriter(Path file, WriteSupport<Group> writeSupport, CompressionCodecName compressionCodecName,
+			int blockSize, int pageSize, boolean enableDictionary, boolean enableValidation,
+			ParquetProperties.WriterVersion writerVersion, Configuration conf) throws IOException {
+		super(file, writeSupport, compressionCodecName, blockSize, pageSize, pageSize, enableDictionary,
+				enableValidation, writerVersion, conf);
+	}
+
+	public static class Builder extends ParquetWriter.Builder<Group, Builder> {
+		private MessageType type = null;
+		private Map<String, String> extraMetaData = new HashMap<String, String>();
+
+		private Builder(Path file) {
+			super(file);
+		}
+
+		private Builder(OutputFile file) {
+			super(file);
+		}
+
+		public Builder withType(MessageType type) {
+			this.type = type;
+			return this;
+		}
+
+		public Builder withExtraMetaData(Map<String, String> extraMetaData) {
+			this.extraMetaData = extraMetaData;
+			return this;
+		}
+
+		@Override
+		protected Builder self() {
+			return this;
+		}
+
+		@Override
+		protected WriteSupport<Group> getWriteSupport(Configuration conf) {
+			return new TalendGroupWriteSupport(type, extraMetaData);
+		}
+
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/utils/NanoTimeUtils.java

@@ -0,0 +1,131 @@
+package org.talend.parquet.utils;
+
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+
+import org.talend.parquet.data.simple.NanoTime;
+
+import jodd.time.JulianDate;
+
+public class NanoTimeUtils {
+
+	/**
+	 * Number of days between Julian day epoch (January 1, 4713 BC) and Unix day
+	 * epoch (January 1, 1970). The value of this constant is {@value}.
+	 */
+	public static final long JULIAN_EPOCH_OFFSET_DAYS = 2440588;
+	private static final long MILLIS_IN_DAY = TimeUnit.DAYS.toMillis(1);
+	private static final long NANOS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1);
+
+	static final long NANOS_PER_HOUR = java.util.concurrent.TimeUnit.HOURS.toNanos(1);
+	static final long NANOS_PER_MINUTE = java.util.concurrent.TimeUnit.MINUTES.toNanos(1);
+	static final long NANOS_PER_SECOND = java.util.concurrent.TimeUnit.SECONDS.toNanos(1);
+	static final long NANOS_PER_DAY = java.util.concurrent.TimeUnit.DAYS.toNanos(1);
+
+	private static final ThreadLocal<java.util.Calendar> parquetGMTCalendar = new ThreadLocal<Calendar>();
+	private static final ThreadLocal<Calendar> parquetLocalCalendar = new ThreadLocal<Calendar>();
+
+	private static Calendar getGMTCalendar() {
+		// Calendar.getInstance calculates the current-time needlessly, so cache
+		// an instance.
+		if (parquetGMTCalendar.get() == null) {
+			parquetGMTCalendar.set(Calendar.getInstance(TimeZone.getTimeZone("GMT")));
+		}
+		return parquetGMTCalendar.get();
+	}
+
+	private static Calendar getLocalCalendar() {
+		if (parquetLocalCalendar.get() == null) {
+			parquetLocalCalendar.set(Calendar.getInstance());
+		}
+		return parquetLocalCalendar.get();
+	}
+
+	private static Calendar getCalendar(boolean skipConversion) {
+		Calendar calendar = skipConversion ? getLocalCalendar() : getGMTCalendar();
+		calendar.clear();
+		return calendar;
+	}
+
+	/**
+	 * Converts a timestamp to NanoTime.
+	 */
+	public static NanoTime getNanoTime(Timestamp ts, boolean skipConversion) {
+
+		Calendar calendar = getCalendar(skipConversion);
+		calendar.setTimeInMillis(ts.getTime());
+		int year = calendar.get(Calendar.YEAR);
+		if (calendar.get(Calendar.ERA) == GregorianCalendar.BC) {
+			year = 1 - year;
+		}
+		JulianDate jDateTime;
+		jDateTime = JulianDate.of(year, calendar.get(Calendar.MONTH) + 1, // java calendar index starting at 1.
+				calendar.get(Calendar.DAY_OF_MONTH), 0, 0, 0, 0);
+		int days = jDateTime.getJulianDayNumber();
+
+		long hour = calendar.get(Calendar.HOUR_OF_DAY);
+		long minute = calendar.get(Calendar.MINUTE);
+		long second = calendar.get(Calendar.SECOND);
+		long nanos = ts.getNanos();
+		long nanosOfDay = nanos + NANOS_PER_SECOND * second + NANOS_PER_MINUTE * minute + NANOS_PER_HOUR * hour;
+
+		return new NanoTime(days, nanosOfDay);
+	}
+
+	public static Timestamp getTimestamp(NanoTime nt, boolean skipConversion) {
+		int julianDay = nt.getJulianDay();
+		long nanosOfDay = nt.getTimeOfDayNanos();
+
+		long remainder = nanosOfDay;
+		julianDay += remainder / NANOS_PER_DAY;
+		remainder %= NANOS_PER_DAY;
+		if (remainder < 0) {
+			remainder += NANOS_PER_DAY;
+			julianDay--;
+		}
+
+		JulianDate jDateTime = new JulianDate((double) julianDay);
+		LocalDateTime datetime = jDateTime.toLocalDateTime();
+		Calendar calendar = getCalendar(skipConversion);
+		calendar.set(Calendar.YEAR, datetime.getYear());
+		calendar.set(Calendar.MONTH, datetime.getMonthValue() - 1);
+		calendar.set(Calendar.DAY_OF_MONTH, datetime.getYear());
+
+		int hour = (int) (remainder / (NANOS_PER_HOUR));
+		remainder = remainder % (NANOS_PER_HOUR);
+		int minutes = (int) (remainder / (NANOS_PER_MINUTE));
+		remainder = remainder % (NANOS_PER_MINUTE);
+		int seconds = (int) (remainder / (NANOS_PER_SECOND));
+		long nanos = remainder % NANOS_PER_SECOND;
+
+		calendar.set(Calendar.HOUR_OF_DAY, hour);
+		calendar.set(Calendar.MINUTE, minutes);
+		calendar.set(Calendar.SECOND, seconds);
+		Timestamp ts = new Timestamp(calendar.getTimeInMillis());
+		ts.setNanos((int) nanos);
+		return ts;
+	}
+
+	/**
+	 * Returns timestamp millis from NanoTime type value.
+	 *
+	 * @param nt NanoTime value
+	 * @return timestamp in millis
+	 */
+	public static long getTimestampMillis(NanoTime nt) {
+		long timeOfDayNanos = nt.getTimeOfDayNanos();
+		int julianDay = nt.getJulianDay();
+
+		return (julianDay - JULIAN_EPOCH_OFFSET_DAYS) * MILLIS_IN_DAY + (timeOfDayNanos / NANOS_PER_MILLISECOND);
+	}
+
+	public static Timestamp getTimestamp(NanoTime nt) {
+		Timestamp ts = new Timestamp(getTimestampMillis(nt));
+		ts.setNanos((int) (nt.getTimeOfDayNanos() % 1000000000));
+		return ts;
+	}
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/main/java/org/talend/parquet/utils/TalendParquetUtils.java

@@ -0,0 +1,231 @@
+package org.talend.parquet.utils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.DecimalMetadata;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Type.Repetition;
+import org.apache.parquet.schema.Types;
+import org.apache.parquet.schema.Types.GroupBuilder;
+import org.talend.parquet.data.Group;
+import org.talend.parquet.data.simple.NanoTime;
+
+public class TalendParquetUtils {
+
+	public static final String ARRAY_FIELD_NAME = "array";
+
+	public static PrimitiveType createPrimitiveType(String fieldName, boolean nullable, String primitiveType,
+			String originalTypeName) {
+		OriginalType originalType = null;
+		if (originalTypeName != null) {
+			originalType = OriginalType.valueOf(originalTypeName);
+		}
+		return new PrimitiveType((nullable ? Repetition.OPTIONAL : Repetition.REQUIRED),
+				PrimitiveTypeName.valueOf(primitiveType), fieldName, originalType);
+	}
+
+	public static PrimitiveType createDecimalType(String fieldName, boolean nullable, int precision, int scale) {
+		DecimalMetadata decimalMetadata = new DecimalMetadata(precision, scale);
+		return new PrimitiveType((nullable ? Repetition.OPTIONAL : Repetition.REQUIRED),
+				PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, 16, fieldName, OriginalType.DECIMAL, decimalMetadata, null);
+	}
+
+	public static Type createGroupElementType(String fieldName, Object element) {
+		if (element == null) {
+			return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(fieldName);
+		}
+		if (String.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(fieldName);
+		} else if (Double.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.DOUBLE).named(fieldName);
+		} else if (Float.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.FLOAT).named(fieldName);
+		} else if (Byte.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.INT32).as(OriginalType.INT_8).named(fieldName);
+		} else if (Short.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.INT32).as(OriginalType.INT_16).named(fieldName);
+		} else if (Integer.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.INT32).named(fieldName);
+		} else if (Long.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.INT64).named(fieldName);
+		} else if (Boolean.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.BOOLEAN).named(fieldName);
+		} else if (Date.class.isInstance(element)) {
+			return Types.repeated(PrimitiveTypeName.INT64).as(OriginalType.TIMESTAMP_MILLIS).named(fieldName);
+		} else if (Group.class.isInstance(element)) {
+			return ((Group) element).getType();
+		} else {
+			throw new IllegalArgumentException("Unsupported type: " + element.getClass().getCanonicalName()
+					+ " for group type field'" + fieldName + "'");
+		}
+	}
+
+	public static GroupType createGroupType(String fieldName, boolean nullable, Object element) {
+		GroupBuilder<GroupType> builder = null;
+		if (nullable) {
+			builder = Types.optionalGroup();
+		} else {
+			builder = Types.requiredGroup();
+		}
+		return builder.as(OriginalType.LIST).addField(createGroupElementType("array", element)).named(fieldName);
+	}
+
+	/*
+	 * Here group only support List value with one field
+	 */
+	public static List<Object> groupFieldValueToList(Group group) {
+		if (group == null) {
+			return null;
+		}
+		List<Object> values = new ArrayList<>();
+		int listSize = group.getFieldRepetitionCount(0);
+		for (int elementIndex = 0; elementIndex < listSize; elementIndex++) {
+			Type elelemntType = group.getType().getType(0);
+			if (elelemntType.isPrimitive()) {
+				PrimitiveType pType = elelemntType.asPrimitiveType();
+				switch (pType.getPrimitiveTypeName()) {
+				case INT64:
+					if (OriginalType.TIMESTAMP_MILLIS == elelemntType.getOriginalType()) {
+						values.add(new Date(group.getLong(0, elementIndex)));
+					} else {
+						values.add(group.getLong(0, elementIndex));
+					}
+					break;
+				case INT32:
+					values.add(group.getInteger(0, elementIndex));
+					break;
+				case BOOLEAN:
+					values.add(group.getBoolean(0, elementIndex));
+					break;
+				case INT96:
+					Binary value = group.getInt96(0, elementIndex);
+					if (value != null) {
+						NanoTime nanoTime = NanoTime.fromBinary(value);
+						values.add(new Date(NanoTimeUtils.getTimestamp(nanoTime, false).getTime()));
+					} else {
+						values.add(value);
+					}
+					break;
+				case FLOAT:
+					values.add(group.getFloat(0, elementIndex));
+					break;
+				case DOUBLE:
+					values.add(group.getDouble(0, elementIndex));
+					break;
+				default:
+					values.add(group.getValueToString(0, elementIndex));
+				}
+			} else {
+				values.add(groupFieldValueToList(group.getGroup(0, elementIndex)));
+			}
+		}
+		return values;
+	}
+
+	public static void writeGroupField(Group nestGroup, List<?> values) {
+		if (values == null || values.isEmpty()) {
+			return;
+		}
+		// only support one field currently
+		for (int i = 0; i < values.size(); i++) {
+			Object element = values.get(i);
+			if (String.class.isInstance(element)) {
+				nestGroup.add(0, (String) element);
+			} else if (Double.class.isInstance(element)) {
+				nestGroup.add(0, (Double) element);
+			} else if (Float.class.isInstance(element)) {
+				nestGroup.add(0, (Float) element);
+			} else if (Byte.class.isInstance(element)) {
+				nestGroup.add(0, (Byte) element);
+			} else if (Short.class.isInstance(element)) {
+				nestGroup.add(0, (Short) element);
+			} else if (Integer.class.isInstance(element)) {
+				nestGroup.add(0, (Integer) element);
+			} else if (Long.class.isInstance(element)) {
+				nestGroup.add(0, (Long) element);
+			} else if (Boolean.class.isInstance(element)) {
+				nestGroup.add(0, (Boolean) element);
+			} else if (Date.class.isInstance(element)) {
+				nestGroup.add(0, ((Date) element).getTime());
+			} else if (Group.class.isInstance(element)) {
+				nestGroup.add(0, (Group) element);
+			} else {
+				throw new IllegalArgumentException("Unsupported type: " + element.getClass().getCanonicalName()
+						+ " for group type field'" + nestGroup + "'");
+			}
+		}
+	}
+
+	public static BigDecimal binaryToDecimal(Binary value, int precision, int scale) {
+		/*
+		 * Precision <= 18 checks for the max number of digits for an unscaled long,
+		 * else treat with big integer conversion
+		 */
+		if (precision <= 18) {
+			ByteBuffer buffer = value.toByteBuffer();
+			byte[] bytes = buffer.array();
+			int start = buffer.arrayOffset() + buffer.position();
+			int end = buffer.arrayOffset() + buffer.limit();
+			long unscaled = 0L;
+			int i = start;
+			while (i < end) {
+				unscaled = (unscaled << 8 | bytes[i] & 0xff);
+				i++;
+			}
+			int bits = 8 * (end - start);
+			long unscaledNew = (unscaled << (64 - bits)) >> (64 - bits);
+			if (scale == 0 || unscaledNew <= -Math.pow(10, 18) || unscaledNew >= Math.pow(10, 18)) {
+				return new BigDecimal(unscaledNew);
+			} else {
+				return BigDecimal.valueOf(unscaledNew / Math.pow(10, scale));
+			}
+		} else {
+			return new BigDecimal(new BigInteger(value.getBytes()), scale);
+		}
+	}
+
+    public static Binary decimalToBinary(BigDecimal decimalValue, int scale) {
+        // First we need to make sure the BigDecimal matches our schema scale:
+        decimalValue = decimalValue.setScale(scale, RoundingMode.HALF_UP);
+
+        // Next we get the decimal value as one BigInteger (like there was no decimal
+        // point)
+        BigInteger unscaledDecimalValue = decimalValue.unscaledValue();
+
+        byte[] decimalBuffer = null;
+        // Finally we serialize the integer
+        byte[] decimalBytes = unscaledDecimalValue.toByteArray();
+
+        if (decimalValue.compareTo(BigDecimal.ZERO) < 0) {
+            decimalBuffer = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 };
+        } else {
+            decimalBuffer = new byte[16];
+        }
+        if (decimalBuffer.length >= decimalBytes.length) {
+            // Because we set our fixed byte array size as 16 bytes, we need to
+            // pad-left our original value's bytes with 0 or -1
+            int decimalBufferIndex = decimalBuffer.length - 1;
+            for (int i = decimalBytes.length - 1; i >= 0; i--) {
+                decimalBuffer[decimalBufferIndex] = decimalBytes[i];
+                decimalBufferIndex--;
+            }
+        } else {
+            throw new IllegalArgumentException(String
+                    .format("Decimal size: %d was greater than the allowed max: %d",
+                            decimalBytes.length, decimalBuffer.length));
+        }
+        return Binary.fromReusedByteArray(decimalBuffer);
+    }
+
+}

main/plugins/org.talend.designer.components.libs/libs_src/talend-parquet/src/test/java/org/talend/parquet/TestUtils.java

@@ -0,0 +1,86 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.talend.parquet;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.statistics.Statistics;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+
+public class TestUtils {
+
+	public static void enforceEmptyDir(Configuration conf, Path path) throws IOException {
+		FileSystem fs = path.getFileSystem(conf);
+		if (fs.exists(path)) {
+			if (!fs.delete(path, true)) {
+				throw new IOException("can not delete path " + path);
+			}
+		}
+		if (!fs.mkdirs(path)) {
+			throw new IOException("can not create path " + path);
+		}
+	}
+
+	/**
+	 * A convenience method to avoid a large number of @Test(expected=...) tests
+	 * 
+	 * @param message  A String message to describe this assertion
+	 * @param expected An Exception class that the Runnable should throw
+	 * @param callable A Callable that is expected to throw the exception
+	 */
+	public static void assertThrows(String message, Class<? extends Exception> expected, Callable callable) {
+		try {
+			callable.call();
+			Assert.fail("No exception was thrown (" + message + "), expected: " + expected.getName());
+		} catch (Exception actual) {
+			try {
+				Assert.assertEquals(message, expected, actual.getClass());
+			} catch (AssertionError e) {
+				e.addSuppressed(actual);
+				throw e;
+			}
+		}
+	}
+
+	public static void assertStatsValuesEqual(Statistics<?> stats1, Statistics<?> stats2) {
+		assertStatsValuesEqual(null, stats1, stats2);
+	}
+
+	// To be used to assert that the values (min, max, num-of-nulls) equals. It
+	// might be used in cases when creating
+	// Statistics object for the proper Type would require too much work/code
+	// duplications etc.
+	public static void assertStatsValuesEqual(String message, Statistics<?> expected, Statistics<?> actual) {
+		if (expected == actual) {
+			return;
+		}
+		if (expected == null || actual == null) {
+			Assert.assertEquals(expected, actual);
+		}
+		Assert.assertThat(actual, CoreMatchers.instanceOf(expected.getClass()));
+		Assert.assertArrayEquals(message, expected.getMaxBytes(), actual.getMaxBytes());
+		Assert.assertArrayEquals(message, expected.getMinBytes(), actual.getMinBytes());
+		Assert.assertEquals(message, expected.getNumNulls(), actual.getNumNulls());
+	}
+}