TDI-35231: Performance of tfileOutputMSXML component

https://jira.talendforge.org/browse/TDI-35231

main/features/org.talend.commons.feature/feature.xml

@@ -109,4 +109,18 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="jackson-core-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="jackson-mapper-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

main/features/org.talend.utils.feature/feature.xml

@@ -114,4 +114,38 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="jackson-core-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="jackson-mapper-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.apache.servicemix.bundles.avro"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"/>
+
+   <plugin
+         id="com.thoughtworks.paranamer"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.apache.commons.compress"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

main/plugins/org.talend.commons.runtime/src/org/talend/commons/runtime/model/components/IComponentConstants.java

@@ -24,4 +24,9 @@ public interface IComponentConstants {
     public static final String JOBLET_NAME_CHANGED = "joblet name changed"; //$NON-NLS-1$
 
     public static final String JOBLET_SCHEMA_CHANGED = "joblet schema changed"; //$NON-NLS-1$
+
+    public static final String COMPONENT_PROPERTIES_TAG = "component.json.serialized"; //$NON-NLS-1$
+
+    public static final String COMPONENT_SCHEMA_TAG = "component.property.schema"; //$NON-NLS-1$
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/NetworkUtil.java

@@ -39,70 +39,6 @@ public class NetworkUtil {
 
     private static final String TALEND_DISABLE_INTERNET = "talend.disable.internet";//$NON-NLS-1$
 
-    private final static Set<String> getMacAddressList() throws IOException {
-        final Set<String> macAddressList = new HashSet<String>();
-
-        final String os = System.getProperty("os.name"); //$NON-NLS-1$
-
-        final String[] command;
-        if (os.startsWith("Windows")) { //$NON-NLS-1$
-            command = windowsCommand;
-        } else if (os.startsWith("Linux")) { //$NON-NLS-1$
-            command = linuxCommand;
-        } else {
-            throw new IOException("Unknown operating system: " + os); //$NON-NLS-1$
-        }
-
-        final Process process = Runtime.getRuntime().exec(command);
-        // Discard the stderr
-        new Thread() {
-
-            @Override
-            public void run() {
-                try {
-                    InputStream errorStream = process.getErrorStream();
-                    while (errorStream.read() != -1) {
-                    }
-                    errorStream.close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }.start();
-
-        // Extract the MAC addresses from stdout
-        BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
-        for (String line = null; (line = reader.readLine()) != null;) {
-            Matcher matcher = macPattern.matcher(line);
-            if (matcher.matches()) {
-                // macAddressList.add(matcher.group(1));
-                macAddressList.add(matcher.group(1).replaceAll("[-:]", "")); //$NON-NLS-1$ //$NON-NLS-2$
-            }
-        }
-        reader.close();
-        return macAddressList;
-    }
-
-    /**
-     * Note： should rename as "getFullMacAddresses" later since it takes in fact the concatenation of all mac address.
-     * 
-     * @return
-     */
-    public static String getMacAddress() {
-        try {
-            Set<String> addressList = getMacAddressList();
-            StringBuffer sb = new StringBuffer();
-            // remove empty mac address
-            addressList.remove("0000000000E0"); //$NON-NLS-1$
-            for (String curMacAddress : addressList) {
-                sb.append(curMacAddress);
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            return null;
-        }
-    }
-
     public static boolean isNetworkValid() {
         String disableInternet = System.getProperty(TALEND_DISABLE_INTERNET);
         if ("true".equals(disableInternet)) { //$NON-NLS-1$
@@ -143,12 +79,4 @@ public class NetworkUtil {
         return null;
     }
 
-    public final static void main(String[] args) {
-        try {
-            System.out.println("  MAC Address: " + getMacAddress()); //$NON-NLS-1$
-        } catch (Throwable t) {
-            t.printStackTrace();
-        }
-    }
-
 }

main/plugins/org.talend.core.repository/plugin.xml

@@ -17,6 +17,7 @@
    <extension-point id="repository_review_filter" name="repository_review_filter" schema="schema/repository_review_filter.exsd"/>
    <extension-point id="repository_context" name="Repository_context" schema="schema/repository_context.exsd"/>
    <extension-point id="document_generator" name="document_generator" schema="schema/document_generator.exsd"/>
+   <extension-point id="git_content_service" name="git_content_service" schema="schema/git_content_service.exsd"/>
 
    <extension
          point="org.talend.core.runtime.service">

main/plugins/org.talend.core.repository/schema/git_content_service.exsd

@@ -0,0 +1,102 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!-- Schema file written by PDE -->
+<schema targetNamespace="org.talend.core.repository" xmlns="http://www.w3.org/2001/XMLSchema">
+<annotation>
+      <appinfo>
+         <meta.schema plugin="org.talend.core.repository" id="git_content_service" name="git_content_service"/>
+      </appinfo>
+      <documentation>
+         [Enter description of this extension point.]
+      </documentation>
+   </annotation>
+
+   <element name="extension">
+      <annotation>
+         <appinfo>
+            <meta.element />
+         </appinfo>
+      </annotation>
+      <complexType>
+         <sequence>
+            <element ref="service" minOccurs="1" maxOccurs="unbounded"/>
+         </sequence>
+         <attribute name="point" type="string" use="required">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="id" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="name" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+               <appinfo>
+                  <meta.attribute translatable="true"/>
+               </appinfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <element name="service">
+      <complexType>
+         <attribute name="class" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+               <appinfo>
+                  <meta.attribute kind="java" basedOn=":org.talend.core.repository.services.IGitRepositoryService"/>
+               </appinfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="since"/>
+      </appinfo>
+      <documentation>
+         [Enter the first release in which this extension point appears.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="examples"/>
+      </appinfo>
+      <documentation>
+         [Enter extension point usage example here.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="apiinfo"/>
+      </appinfo>
+      <documentation>
+         [Enter API information here.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="implementation"/>
+      </appinfo>
+      <documentation>
+         [Enter information about supplied implementation of this extension point.]
+      </documentation>
+   </annotation>
+
+
+</schema>

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/i18n/messages.properties

@@ -163,6 +163,10 @@ ProjectRepositoryNode.sapFunctions.inputSchema=Input
 ProjectRepositoryNode.sapFunctions.outputSchema=Output
 ProjectRepositoryNode.sapIDocs=SAP iDocs
 ProjectRepositoryNode.sapTables=SAP Tables
+ProjectRepositoryNode.sapBWDataSource=SAP DataSource
+ProjectRepositoryNode.sapBWDataStoreObject=SAP DSO
+ProjectRepositoryNode.sapBWInfoCube=SAP InfoCube
+ProjectRepositoryNode.sapBWInfoObject=SAP InfoObject
 
 RepositoryDropAdapter_copyingItems=Copying items...
 RepositoryDropAdapter_errorMsg=This directory contains the locked item,it can not be moved now

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/IRepositoryFactory.java

@@ -38,6 +38,7 @@ import org.talend.core.model.repository.IRepositoryWorkUnitListener;
 import org.talend.core.model.repository.LockInfo;
 import org.talend.core.repository.utils.XmiResourceManager;
 import org.talend.repository.RepositoryWorkUnit;
+import org.talend.utils.json.JSONArray;
 
 /**
  * Defines all methods that a repository provider plug-in must provides to client.<br/>
@@ -414,4 +415,6 @@ public interface IRepositoryFactory {
     public void setStorage(String storage);
 
     public String getStorage();
+
+    public JSONArray getAllRemoteLocks();
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProjectRepositoryNode.java

@@ -39,7 +39,6 @@ import org.talend.commons.utils.data.container.Container;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.database.EDatabaseTypeName;
-import org.talend.core.model.genhtml.IHTMLDocConstants;
 import org.talend.core.model.metadata.MetadataManager;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
@@ -47,6 +46,7 @@ import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.metadata.builder.connection.QueriesConnection;
 import org.talend.core.model.metadata.builder.connection.Query;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.model.metadata.builder.connection.SAPConnection;
 import org.talend.core.model.metadata.builder.connection.SAPFunctionUnit;
 import org.talend.core.model.metadata.builder.connection.SAPIDocUnit;
@@ -92,9 +92,9 @@ import org.talend.core.ui.ICDCProviderService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.core.ui.branding.IBrandingService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.TableHelper;
-import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.BinRepositoryNode;
@@ -980,51 +980,6 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         return false;
     }
 
-    @SuppressWarnings("rawtypes")
-    private void convertDocumentation(org.talend.core.model.general.Project newProject, Container generatedContainer,
-            RepositoryNode parent, ERepositoryObjectType type) {
-        // for folder Documentation/generated
-        // RepositoryNode generatedFolder = getRootRepositoryNode(ERepositoryObjectType.GENERATED);
-
-        // for folder Documentation/generated/jobs
-        convertDocumentation(newProject, generatedContainer, parent, type, ERepositoryObjectType.JOBS,
-                ERepositoryObjectType.JOB_DOC);
-
-        // for folder Documentation/generated/joblets
-        convertDocumentation(newProject, generatedContainer, parent, type, ERepositoryObjectType.JOBLETS,
-                ERepositoryObjectType.JOBLET_DOC);
-
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ICamelDesignerCoreService.class)) {
-            ICamelDesignerCoreService service = (ICamelDesignerCoreService) GlobalServiceRegister.getDefault().getService(
-                    ICamelDesignerCoreService.class);
-            if (service.getRouteDocsType() != null && service.getRouteDocType() != null) {
-                convertDocumentation(newProject, generatedContainer, parent, type, service.getRouteDocsType(),
-                        service.getRouteDocType());
-            }
-        }
-
-    }
-
-    @SuppressWarnings("rawtypes")
-    private void convertDocumentation(org.talend.core.model.general.Project newProject, Container generatedContainer,
-            RepositoryNode parent, ERepositoryObjectType type, ERepositoryObjectType parentDocType, ERepositoryObjectType docType) {
-        RepositoryNode docsFolder = getRootRepositoryNode(parentDocType);
-
-        Container docsNode = null;
-        for (Object object : generatedContainer.getSubContainer()) {
-            if (((Container) object).getLabel().equalsIgnoreCase(parentDocType.name().toLowerCase())) {
-                docsNode = (Container) object;
-                break;
-            }
-        }
-
-        // get the files under generated/nodes.
-        if (docsNode != null) {
-            convert(newProject, docsNode, docsFolder, docType);
-        }
-
-    }
-
     private RepositoryNode getSQLPatternNode(String parentLabel, String label) {
         if (getMergeRefProject()) {
             List<IRepositoryNode> sqlChildren = getRootRepositoryNode(ERepositoryObjectType.SQLPATTERNS).getChildren();
@@ -1093,7 +1048,7 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             Folder oFolder = new Folder((Property) container.getProperty(), type);
             boolean found = false;
             // add for bug TDI-26084, whether or not hide folders under job_doc/joblet_doc.
-            if (ERepositoryObjectType.JOB_DOC.equals(type) || ERepositoryObjectType.JOBLET_DOC.equals(type)) {
+            if (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC)) {
                 for (FolderItem delFolder : delFolderItems) {
                     String parentName = ((FolderItem) delFolder.getParent()).getProperty().getLabel();
                     String oFolderPath = oFolder.getPath();
@@ -1126,19 +1081,6 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             if (label.equals("bin") || label.startsWith(".")) { //$NON-NLS-1$ //$NON-NLS-2$
                 continue;
             }
-            // currently, only process the docs for job, joblet and route.
-            if (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC)
-                    || type.equals(ERepositoryObjectType.JOBS) || type.equals(ERepositoryObjectType.JOBLETS)
-                    || type.equals(ERepositoryObjectType.valueOf("ROUTE_DOCS"))//$NON-NLS-1$ 
-                    || type.equals(ERepositoryObjectType.valueOf("ROUTE_DOC"))) {//$NON-NLS-1$ 
-                boolean isJobDocRootFolder = ((label.indexOf("_") != -1) && (label.indexOf(".") != -1)); //$NON-NLS-1$ //$NON-NLS-2$
-                boolean isPicFolderName = label.equals(IHTMLDocConstants.PIC_FOLDER_NAME);
-
-                // Do not show job documentation root folder and Foder "pictures" on the repository view.
-                if (isJobDocRootFolder || isPicFolderName) {
-                    continue;
-                }
-            }
             // for system folder
             if (RepositoryConstants.SYSTEM_DIRECTORY.equals(label)) {
                 if (getMergeRefProject()) {
@@ -1166,10 +1108,13 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
                     parent.getChildren().add(folder);
                 }
 
-            } else if (ERepositoryObjectType.GENERATED.name().equalsIgnoreCase(label)) {
-                if (PluginChecker.isDocumentationPluginLoaded()) {
-                    convertDocumentation(newProject, container, parent, type);
-                }
+            } else
+            // ERepositoryObjectType.GENERATED
+            if (type.equals(ERepositoryObjectType.DOCUMENTATION) && label.equalsIgnoreCase("generated")) {//$NON-NLS-1$ 
+                // if (PluginChecker.isDocumentationPluginLoaded()) {
+                // use CNF content provider instead
+                // convertDocumentation(newProject, container, parent, type);
+                // }
                 continue;
             } else {
                 if (getMergeRefProject()) {
@@ -1462,6 +1407,12 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         if (vo != null) {
             String schema = vo.getId();
             schema = schema + " - " + metadataTable.getLabel(); //$NON-NLS-1$
+            if (metadataTable instanceof SAPBWTable) {
+                String innerIOType = ((SAPBWTable) metadataTable).getInnerIOType();
+                if (innerIOType != null) {
+                    schema = schema + " - " + innerIOType; //$NON-NLS-1$
+                }
+            }
             List<IRepositoryViewObject> objs = getValidationRuleObjsFromSchema(validationRules, schema);
             if (objs.size() > 0) {
                 int num = objs.size();
@@ -1632,6 +1583,18 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             iDocNode.setChildrenObjectType(ERepositoryObjectType.METADATA_SAP_IDOC);
             node.getChildren().add(iDocNode);
             createSAPIDocNodes(repObj, metadataConnection, iDocNode);
+
+            // 4. BW DataSource:
+            createSAPBWDataSourceNodes(repObj, metadataConnection, node, validationRules);
+
+            // 5. BW DataStoreObject:
+            createSAPBWDataStoreObjectNodes(repObj, metadataConnection, node, validationRules);
+
+            // 6. BW InfoCube:
+            createSAPBWInfoCubeNodes(repObj, metadataConnection, node, validationRules);
+
+            // 7. BW InfoObject:
+            createSAPBWInfoObjectNodes(repObj, metadataConnection, node, validationRules);
         } else if (metadataConnection instanceof SalesforceSchemaConnection) {
             createSalesforceModuleNodes(repObj, metadataConnection, node, validationRules);
         } else {
@@ -1658,6 +1621,66 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
 
     }
 
+    private void createSAPBWDataSourceNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWDataSource"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_DATASOURCE);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> datasources = ((SAPConnection) metadataConnection).getBWDataSources();
+        EList tables = new BasicEList();
+        tables.addAll(datasources);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWDataStoreObjectNodes(IRepositoryViewObject repObj, Connection metadataConnection,
+            RepositoryNode node, List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWDataStoreObject"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_DATASTOREOBJECT);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> dataStoreObjects = ((SAPConnection) metadataConnection).getBWDataStoreObjects();
+        EList tables = new BasicEList();
+        tables.addAll(dataStoreObjects);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWInfoCubeNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWInfoCube"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_INFOCUBE);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> infoCubes = ((SAPConnection) metadataConnection).getBWInfoCubes();
+        EList tables = new BasicEList();
+        tables.addAll(infoCubes);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWInfoObjectNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWInfoObject"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_INFOOBJECT);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> infoObjects = ((SAPConnection) metadataConnection).getBWInfoObjects();
+        EList<SAPBWTable> tables = new BasicEList<SAPBWTable>();
+        for (SAPBWTable bwTable : infoObjects) {
+            if (!SAPBWTableHelper.IO_INNERTYPE_BASIC.equals(bwTable.getInnerIOType())) {
+                tables.add(bwTable);
+            }
+        }
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
     private void createSalesforceModuleNodes(IRepositoryViewObject rebObj, Connection metadataConnection,
             RepositoryNode connectionNode, List<IRepositoryViewObject> validationRules) {
         EList modules = ((SalesforceSchemaConnection) metadataConnection).getModules();

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -126,6 +126,7 @@ import org.talend.repository.documentation.ERepositoryActionName;
 import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.repository.model.RepositoryConstants;
 import org.talend.utils.io.FilesUtils;
+import org.talend.utils.json.JSONArray;
 import orgomg.cwm.objectmodel.core.ModelElement;
 
 /**
@@ -2227,4 +2228,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         }
     }
 
+    public JSONArray getAllRemoteLocks() {
+        return repositoryFactoryFromProvider.getAllRemoteLocks();
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/provider/GitContentServiceProviderManager.java

@@ -0,0 +1,49 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2015 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.repository.model.provider;
+
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IExtension;
+import org.eclipse.core.runtime.IExtensionPoint;
+import org.eclipse.core.runtime.IExtensionRegistry;
+import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.repository.services.IGitContentService;
+
+public class GitContentServiceProviderManager {
+
+    public static IGitContentService getGitContentService() {
+        IGitContentService service = null;
+        IExtensionRegistry extensionRegistry = Platform.getExtensionRegistry();
+        IExtensionPoint extensionPoint = extensionRegistry.getExtensionPoint("org.talend.core.repository.git_content_service"); //$NON-NLS-1$
+        if (extensionPoint != null) {
+            IExtension[] extensions = extensionPoint.getExtensions();
+            for (IExtension extension : extensions) {
+                IConfigurationElement[] configurationElements = extension.getConfigurationElements();
+                for (IConfigurationElement configurationElement : configurationElements) {
+                    try {
+                        Object object = configurationElement.createExecutableExtension("class"); //$NON-NLS-1$
+                        if (object instanceof IGitContentService) {
+                            service = (IGitContentService) object;
+                            break;
+                        }
+                    } catch (CoreException e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+            }
+        }
+        return service;
+    }
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/repositoryObject/MetadataTableRepositoryObject.java

@@ -27,6 +27,8 @@ import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
 import org.talend.core.model.metadata.builder.connection.Concept;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.MDMConnection;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
+import org.talend.core.model.metadata.builder.connection.SAPConnection;
 import org.talend.core.model.metadata.builder.connection.SAPFunctionUnit;
 import org.talend.core.model.metadata.builder.connection.SalesforceModuleUnit;
 import org.talend.core.model.properties.ConnectionItem;
@@ -38,6 +40,7 @@ import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.repository.ISubRepositoryObject;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.SwitchHelpers;
 import org.talend.cwm.relational.TdTable;
@@ -121,6 +124,12 @@ public class MetadataTableRepositoryObject extends MetadataTable implements ISub
             moduleUnit.getTables().remove(table);
             return;
         }
+        
+        if (table.eContainer() instanceof SAPConnection && table instanceof SAPBWTable) {
+            SAPBWTableHelper.removeBWTable((Connection) table.eContainer(), ((SAPBWTable) table).getModelType(),
+                    (SAPBWTable) table);
+        }
+
         if (table.getNamespace() instanceof Package) {
             Package pkg = (Package) table.getNamespace();
             if (pkg.getOwnedElement().contains(table)) {
@@ -184,6 +193,10 @@ public class MetadataTableRepositoryObject extends MetadataTable implements ISub
         } else if (table.eContainer() instanceof SAPFunctionUnit) {
             SAPFunctionUnit funUnit = (SAPFunctionUnit) table.eContainer();
             tables = ConnectionHelper.getTables(connection, funUnit);
+        } else if (table instanceof SAPBWTable) {
+            String bwTableType = ((SAPBWTable) table).getModelType();
+            String innerIOType = ((SAPBWTable) table).getInnerIOType();
+            tables = SAPBWTableHelper.getBWTables(connection, bwTableType, innerIOType, true);
         } else {
             tables = ConnectionHelper.getTables(connection);
         }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/recyclebin/RecycleBinManager.java

@@ -20,6 +20,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.eclipse.core.resources.IProject;
+import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.common.util.URI;
 import org.eclipse.emf.ecore.resource.Resource;
@@ -75,6 +76,11 @@ public class RecycleBinManager {
         projectRecyclebins.clear();
     }
 
+    public void clearCache(Project project) {
+        String projectTechnicalLabel = project.getTechnicalLabel();
+        projectRecyclebins.remove(projectTechnicalLabel);
+    }
+
     public void clearIndex(Project project) {
         loadRecycleBin(project);
         projectRecyclebins.get(project.getTechnicalLabel()).getDeletedItems().clear();
@@ -88,8 +94,15 @@ public class RecycleBinManager {
         List<TalendItem> notDeletedItems = new ArrayList<TalendItem>();
         for (TalendItem deletedItem : deletedItems) {
             try {
+                final ERepositoryObjectType type = ERepositoryObjectType.getType(deletedItem.getType());
+                // ignore the generated doc in recycle bin
+                if (type != null
+                        && (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC) || type
+                                .equals(ERepositoryObjectType.valueOf("ROUTE_DOC")))) { //$NON-NLS-1$
+                    continue;
+                }
                 IRepositoryViewObject object = ProxyRepositoryFactory.getInstance().getLastVersion(project, deletedItem.getId(),
-                        deletedItem.getPath(), ERepositoryObjectType.getType(deletedItem.getType()));
+                        deletedItem.getPath(), type);
                 if (object == null) {
                     object = ProxyRepositoryFactory.getInstance().getLastVersion(project, deletedItem.getId());
                 }
@@ -164,6 +177,11 @@ public class RecycleBinManager {
         }
     }
 
+    public RecycleBin getRecycleBin(Project project) {
+        loadRecycleBin(project);
+        return projectRecyclebins.get(project.getTechnicalLabel());
+    }
+
     private void loadRecycleBin(Project project) {
         if (projectRecyclebins.get(project.getTechnicalLabel()) != null) {
             // already loaded, nothing to do. Don't do any force reload
@@ -186,6 +204,16 @@ public class RecycleBinManager {
         }
     }
 
+    public RecycleBin loadRecycleBin(IPath recycleBinIndexPath) throws Exception {
+        Resource resource = createRecycleBinResource(recycleBinIndexPath);
+        resource.load(null);
+        return loadRecycleBin(resource);
+    }
+
+    public RecycleBin loadRecycleBin(Resource resource) {
+        return (RecycleBin) EcoreUtil.getObjectByType(resource.getContents(), RecycleBinPackage.eINSTANCE.getRecycleBin());
+    }
+
     public void saveRecycleBin(Project project) {
         if (projectRecyclebins.get(project.getTechnicalLabel()) == null) {
             loadRecycleBin(project);
@@ -219,7 +247,12 @@ public class RecycleBinManager {
 
     private Resource createRecycleBinResource(Project project) {
         IProject eclipseProject = ProjectManager.getInstance().getResourceProject(project.getEmfProject());
-        URI uri = URIHelper.convert(eclipseProject.getFullPath().append(TALEND_RECYCLE_BIN_INDEX));
+
+        return createRecycleBinResource(eclipseProject.getFullPath().append(TALEND_RECYCLE_BIN_INDEX));
+    }
+
+    public Resource createRecycleBinResource(IPath recycleBinIndexPath) {
+        URI uri = URIHelper.convert(recycleBinIndexPath);
 
         XMLResourceFactoryImpl resourceFact = new XMLResourceFactoryImpl();
         XMLResource resource = (XMLResource) resourceFact.createResource(uri);
@@ -234,5 +267,4 @@ public class RecycleBinManager {
         resource.getDefaultLoadOptions().put(XMLResource.OPTION_USE_LEXICAL_HANDLER, Boolean.TRUE);
         return resource;
     }
-
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/services/IGitContentService.java

@@ -0,0 +1,12 @@
+package org.talend.core.repository.services;
+
+public interface IGitContentService {
+
+    public abstract void setMenu(Object object);
+
+    public abstract boolean isGIT();
+
+    public abstract void createDropdownCombo(Object composite);
+
+    public abstract void configureCombo(String descriptor);
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CopyObjectAction.java

@@ -426,7 +426,8 @@ public class CopyObjectAction {
             }
         }
         if (sourceNode.getObjectType() != ERepositoryObjectType.PROCESS
-                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_MR) {
+                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_MR
+                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_STORM) {
             return false;
         }
         if (!(newItem instanceof ProcessItem)) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CreateFolderAction.java

@@ -107,8 +107,9 @@ public class CreateFolderAction extends AContextualAction {
                 canWork = false;
                 break;
             case SYSTEM_FOLDER:
-                if (ERepositoryObjectType.GENERATED.equals(property) || ERepositoryObjectType.JOBS.equals(property)
-                        || ERepositoryObjectType.JOBLETS.equals(property) || ERepositoryObjectType.SQLPATTERNS.equals(property)
+                if (property == null || property.equals(ERepositoryObjectType.GENERATED)
+                        || property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
+                        || ERepositoryObjectType.SQLPATTERNS.equals(property)
                         || ERepositoryObjectType.REFERENCED_PROJECTS.equals(property)
                         || ERepositoryObjectType.SVN_ROOT.equals(property)) {
                     canWork = false;
@@ -122,7 +123,7 @@ public class CreateFolderAction extends AContextualAction {
                 }
                 break;
             case SIMPLE_FOLDER:
-                if (ERepositoryObjectType.JOB_DOC.equals(property) || ERepositoryObjectType.JOBLET_DOC.equals(property)
+                if (property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
                         || (ERepositoryObjectType.SQLPATTERNS.equals(property) && !isUnderUserDefined(node))) {
                     canWork = false;
                 } else if (property != null

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DeleteAction.java

@@ -1197,9 +1197,6 @@ public class DeleteAction extends AContextualAction {
             return true;
         }
 
-        if (node.getContentType() == ERepositoryObjectType.JOBS) {
-            return true;
-        }
         if (node.getContentType() == ERepositoryObjectType.GENERATED) {
             return true;
         }
@@ -1495,8 +1492,8 @@ public class DeleteAction extends AContextualAction {
                         label = (String) obj;
                     }
                     boolean isGointoCondition = false;
-                    if (node.getContentType() == ERepositoryObjectType.JOB_DOC
-                            || node.getContentType() == ERepositoryObjectType.JOBLET_DOC
+                    if (node.getContentType().equals(ERepositoryObjectType.JOB_DOC)
+                            || node.getContentType().equals(ERepositoryObjectType.JOBLET_DOC)
                             || RepositoryConstants.USER_DEFINED.equals(label)) {
                         visible = false;
                         isGointoCondition = true;

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DuplicateAction.java

@@ -152,8 +152,8 @@ public class DuplicateAction extends AContextualAction {
                     } else if (node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC) {
                         canWork = false;
-                    } else if (node.getContentType() == ERepositoryObjectType.JOBS
-                            || node.getContentType() == ERepositoryObjectType.JOBLETS
+                    } else if (node.getContentType() == ERepositoryObjectType.JOB_DOC
+                            || node.getContentType() == ERepositoryObjectType.JOBLET_DOC
                             || node.getContentType() == ERepositoryObjectType.GENERATED
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC) {
@@ -511,7 +511,7 @@ public class DuplicateAction extends AContextualAction {
     private boolean isKeyword(String itemName) {
         ERepositoryObjectType itemType = sourceNode.getObjectType();
         ERepositoryObjectType[] types = { ERepositoryObjectType.PROCESS, ERepositoryObjectType.ROUTINES,
-                ERepositoryObjectType.JOBS, ERepositoryObjectType.JOBLET, ERepositoryObjectType.JOBLETS,
+                ERepositoryObjectType.JOB_DOC, ERepositoryObjectType.JOBLET, ERepositoryObjectType.JOBLET_DOC,
                 ERepositoryObjectType.JOB_SCRIPT };
         List<ERepositoryObjectType> arraysList = Arrays.asList(types);
         List<ERepositoryObjectType> typeList = new ArrayList<ERepositoryObjectType>();

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/PasteAction.java

@@ -166,8 +166,8 @@ public class PasteAction extends AContextualAction {
         IProxyRepositoryFactory proxyFactory = ProxyRepositoryFactory.getInstance();
         IRepositoryViewObject object = target.getObject();
 
-        if (target.getContentType() == ERepositoryObjectType.JOBS
-                || target.getContentType() == ERepositoryObjectType.JOBLETS
+        if (target.getContentType() == ERepositoryObjectType.JOB_DOC
+                || target.getContentType() == ERepositoryObjectType.JOBLET_DOC
                 || target.getContentType() == ERepositoryObjectType.GENERATED
                 || target.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                 || target.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/view/RepositoryLabelProvider.java

@@ -60,6 +60,7 @@ import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.repository.model.IRepositoryNode.ENodeType;
 import org.talend.repository.model.RepositoryNode;
 import org.talend.repository.ui.views.IRepositoryView;
+import org.talend.utils.string.MD5;
 
 /**
  * Label provider for the repository view. <code>DEBUG</code> boolean field specify if details (such as objects ids)
@@ -297,11 +298,12 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
 
             ImageDescriptor imageDesc = ImageUtils.createImageFromData(item.getIcon().getInnerContent());
             imageDesc = ImageUtils.scale(imageDesc, ICON_SIZE.ICON_32);
-
-            image = cachedImages.get(item.getIcon().getInnerContent());
+            String md5Desc = MD5.getMD5(item.getIcon().getInnerContent());
+            image = cachedImages.get(md5Desc);
+           
             if (image == null || image.isDisposed()) {
                 image = imageDesc.createImage();
-                cachedImages.put(item.getIcon().getInnerContent(), image);
+            	cachedImages.put(md5Desc, image);
             } else {
                 // image = imageDesc.createImage();
             }
@@ -309,7 +311,7 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
         return image;
     }
 
-    private static Map<byte[], Image> cachedImages = new HashMap<byte[], Image>();
+    private static Map<String, Image> cachedImages = new HashMap<String, Image>();
 
     @Override
     public Image getImage(Object obj) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ConvertJobsUtil.java

@@ -33,7 +33,6 @@ import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.hadoop.HadoopConstants;
 import org.talend.core.hadoop.IHadoopDistributionService;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.properties.Item;
@@ -43,6 +42,8 @@ import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.repository.i18n.Messages;
 import org.talend.core.runtime.CoreRuntimePlugin;
+import org.talend.core.runtime.hd.IHDConstants;
+import org.talend.core.runtime.hd.IHDistributionVersion;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.convert.IProcessConvertService;
 import org.talend.designer.core.convert.IProcessConvertToAllTypeService;
@@ -317,8 +318,8 @@ public class ConvertJobsUtil {
         ERepositoryObjectType oldRepType = (oldJobType == null ? null : oldJobType.getERepositoryObjectType());
         ERepositoryObjectType newRepType = (newJobType == null ? null : newJobType.getERepositoryObjectType());
 
-        boolean isNeedConvert = ProcessConvertManager.getInstance().CheckConvertProcess(oldRepType, oldFrameworkValue, newRepType,
-                newFrameworkValue);
+        boolean isNeedConvert = ProcessConvertManager.getInstance().CheckConvertProcess(oldRepType, oldFrameworkValue,
+                newRepType, newFrameworkValue);
 
         // if need popup warning, then do this check
         if (isNeedConvert && needPopupWarning) {
@@ -327,29 +328,29 @@ public class ConvertJobsUtil {
                 boolean isSparkStreaming = SPARKSTREAMING_FRAMEWORK.equals(newFrameworkValue);
                 if (isSpark || isSparkStreaming) {
                     try {
-                        IProcessConvertService converter = ProcessConvertManager.getInstance()
-                                .extractConvertService(ProcessConverterType.CONVERTER_FOR_MAPREDUCE);
+                        IProcessConvertService converter = ProcessConvertManager.getInstance().extractConvertService(
+                                ProcessConverterType.CONVERTER_FOR_MAPREDUCE);
 
                         IProcess process = converter.getProcessFromItem(originalItem, false);
                         IElementParameter mrVersion = process.getElementParameter(MR_VERSION);
-                        if (mrVersion != null) {
-                            EHadoopVersion4Drivers hadoopVersion = EHadoopVersion4Drivers
-                                    .indexOfByVersion((String) mrVersion.getValue());
-                            if (hadoopVersion != null) {
-                                if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
-                                    IHadoopDistributionService hadoopService = (IHadoopDistributionService) GlobalServiceRegister
-                                            .getDefault().getService(IHadoopDistributionService.class);
-                                    boolean isSupport = false;
-                                    if (isSpark) {
-                                        isSupport = hadoopService.isSupportSpark(hadoopVersion);
-                                    } else if (isSparkStreaming) {
-                                        isSupport = hadoopService.isSupportSparkStreaming(hadoopVersion);
-                                    }
-                                    if (!isSupport) {
-                                        MessageDialog.openWarning(Display.getDefault().getActiveShell(),
-                                                Messages.getString("ConvertJobsUtil.warning.title"), //$NON-NLS-1$
-                                                Messages.getString("ConvertJobsUtil.warning.message")); //$NON-NLS-1$
-                                    }
+                        if (mrVersion != null
+                                && GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
+                            IHadoopDistributionService hadoopService = (IHadoopDistributionService) GlobalServiceRegister
+                                    .getDefault().getService(IHadoopDistributionService.class);
+                            IHDistributionVersion distributionVersion = hadoopService.getHadoopDistributionVersion(
+                                    (String) mrVersion.getValue(), false);
+                            if (distributionVersion != null) {
+                                boolean isSupport = false;
+                                if (isSpark) {
+                                    isSupport = hadoopService.doSupportService(distributionVersion, IHDConstants.SERVICE_SPARK);
+                                } else if (isSparkStreaming) {
+                                    isSupport = hadoopService.doSupportService(distributionVersion,
+                                            IHDConstants.SERVICE_SPARK_STREAMING);
+                                }
+                                if (!isSupport) {
+                                    MessageDialog.openWarning(Display.getDefault().getActiveShell(),
+                                            Messages.getString("ConvertJobsUtil.warning.title"), //$NON-NLS-1$
+                                            Messages.getString("ConvertJobsUtil.warning.message")); //$NON-NLS-1$
                                 }
                             }
                         }

main/plugins/org.talend.core.runtime/META-INF/MANIFEST.MF

@@ -15,7 +15,6 @@ Export-Package: org.talend.commons.utils.generation,
  org.talend.core.database.conn,
  org.talend.core.database.conn.template,
  org.talend.core.database.conn.version,
- org.talend.core.database.hbase.conn.version,
  org.talend.core.database.utils,
  org.talend.core.download,
  org.talend.core.exception,
@@ -54,6 +53,8 @@ Export-Package: org.talend.commons.utils.generation,
  org.talend.core.repository,
  org.talend.core.runtime,
  org.talend.core.runtime.conf,
+ org.talend.core.runtime.hd,
+ org.talend.core.runtime.hd.hive,
  org.talend.core.runtime.maven,
  org.talend.core.runtime.preference,
  org.talend.core.runtime.process,
@@ -110,7 +111,9 @@ Require-Bundle: org.eclipse.jdt.core,
  org.talend.commons.runtime;visibility:=reexport,
  org.talend.commons.ui,
  org.ops4j.pax.url.mvn,
- org.talend.components.api
+ org.talend.components.api,
+ org.apache.servicemix.bundles.avro,
+ jackson-core-asl
 Bundle-Activator: org.talend.core.runtime.CoreRuntimePlugin
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,

main/plugins/org.talend.core.runtime/mappings/mapping_Cassandra22_datastax.xml

@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<mapping>
+    <dbms product="Cassandra" id="cassandra22_datastax_id" label="Mapping Cassandra"
+        default="true">
+        <dbTypes>
+            <dbType type="ascii" ignorePre="true" />
+            <dbType type="bigint" ignorePre="true" />
+            <dbType type="blob" ignorePre="true" />
+            <dbType type="boolean" ignorePre="true" />
+            <dbType type="counter" ignorePre="true" />
+            <dbType type="date" ignorePre="true" />
+            <dbType type="decimal" ignorePre="true" />
+            <dbType type="double" ignorePre="true" />
+            <dbType type="float" ignorePre="true" />
+            <dbType type="inet" ignorePre="true" />
+            <dbType type="int" ignorePre="true" />
+            <dbType type="list" ignorePre="true" />
+            <dbType type="map" ignorePre="true" />
+            <dbType type="set" ignorePre="true" />
+            <dbType type="smallint" ignorePre="true" />
+            <dbType type="text" ignorePre="true" />
+            <dbType type="time" ignorePre="true" />
+            <dbType type="timestamp" ignorePre="true" />
+            <dbType type="timeuuid" ignorePre="true" />
+            <dbType type="tinyint" ignorePre="true" />
+            <dbType type="uuid" ignorePre="true" />
+            <dbType type="varchar" ignorePre="true" />
+            <dbType type="varint" ignorePre="true" />
+        </dbTypes>
+
+        <language name="java">
+            <talendToDbTypes>
+                <talendType type="id_List">
+                    <dbType type="list" default="true"/>
+                </talendType>
+                <talendType type="id_Boolean">
+                    <dbType type="boolean" default="true" />
+                </talendType>
+                <talendType type="id_Byte">
+                    <dbType type="tinyint" default="true" />
+                </talendType>
+                <talendType type="id_byte[]">
+                    <dbType type="blob" default="true" />
+                </talendType>
+                <talendType type="id_Character">
+                    <dbType type="ascii" default="true" />
+                    <dbType type="text" />
+                    <dbType type="varchar" />
+                </talendType>
+                <talendType type="id_Date">
+                    <dbType type="timestamp" default="true" />
+                    <dbType type="date" />
+                </talendType>
+                <talendType type="id_BigDecimal">
+                    <dbType type="decimal" default="true" />
+                </talendType>
+                <talendType type="id_Double">
+                    <dbType type="double" default="true" />
+                </talendType>
+                <talendType type="id_Float">
+                    <dbType type="float" default="true" />
+                </talendType>
+                <talendType type="id_Integer">
+                    <dbType type="int" default="true" />
+                </talendType>
+                <talendType type="id_Long">
+                    <dbType type="bigint" default="true" />
+                    <dbType type="counter" />
+                    <dbType type="time" />
+                </talendType>
+                <talendType type="id_Object">
+                    <dbType type="varint" default="true" />
+                    <dbType type="inet" />
+                    <dbType type="map" />
+                    <dbType type="set" />
+                </talendType>    
+                <talendType type="id_Short">
+                    <dbType type="smallint" default="true" />
+                </talendType>
+                <talendType type="id_String">
+                    <dbType type="ascii" default="true" />
+                    <dbType type="text" />
+                    <dbType type="timeuuid" />
+                    <dbType type="uuid" />
+                    <dbType type="varchar" />
+                </talendType>
+            </talendToDbTypes>
+            <dbToTalendTypes>
+                <dbType type="ascii">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="bigint">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="blob">
+                    <talendType type="id_byte[]" default="true" />
+                </dbType>
+                <dbType type="boolean">
+                    <talendType type="id_Boolean" default="true" />
+                </dbType>
+                <dbType type="counter">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="date">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="decimal">
+                    <talendType type="id_BigDecimal" default="true" />
+                </dbType>
+                <dbType type="double">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="float">
+                    <talendType type="id_Float" default="true" />
+                </dbType>
+                <dbType type="inet">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="int">
+                    <talendType type="id_Integer" default="true" />
+                </dbType>
+                <dbType type="list">
+                    <talendType type="id_List" default="true" />
+                </dbType>
+                <dbType type="map">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="set">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="smallint">
+                    <talendType type="id_Short" default="true" />
+                </dbType>
+                <dbType type="text">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="time">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="timestamp">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="timeuuid">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="tinyint">
+                    <talendType type="id_Byte" default="true" />
+                </dbType>
+                <dbType type="uuid">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="varchar">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="varint">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+            </dbToTalendTypes>
+        </language>
+    </dbms>
+
+</mapping>

main/plugins/org.talend.core.runtime/mappings/mapping_Cassandra_datastax.xml

@@ -82,7 +82,6 @@
             <dbToTalendTypes>
                 <dbType type="ascii">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="bigint">
                     <talendType type="id_Long" default="true" />
@@ -110,8 +109,6 @@
                 </dbType>
                 <dbType type="int">
                     <talendType type="id_Integer" default="true" />
-                    <talendType type="id_Short" />
-                    <talendType type="id_Byte"/>
                 </dbType>
                 <dbType type="list">
                     <talendType type="id_List" default="true" />
@@ -124,7 +121,6 @@
                 </dbType>
                 <dbType type="text">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="timestamp">
                     <talendType type="id_Date" default="true" />
@@ -137,7 +133,6 @@
                 </dbType>
                 <dbType type="varchar">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="varint">
                     <talendType type="id_Object" default="true" />

main/plugins/org.talend.core.runtime/mappings/mapping_Hive.xml

@@ -9,11 +9,13 @@
 			<dbType type="INT" ignoreLen="true" ignorePre="true"/>
 			<dbType type="TINYINT" ignoreLen="true" ignorePre="true"/>
 			<dbType type="STRING" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DECIMAL" ignoreLen="false" ignorePre="false"/>
 			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true"/>
 			<dbType type="STRUCT" ignoreLen="true" ignorePre="true"/>
 			<dbType type="MAP" ignoreLen="true" ignorePre="true"/>
 			<dbType type="ARRAY" ignoreLen="true" ignorePre="true"/>
 			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DATE" ignoreLen="true" ignorePre="true"/>
 		</dbTypes>
 		
 		<language name="java">
@@ -34,9 +36,11 @@
 				</talendType>
 				<talendType type="id_Date">
 				    <dbType type="TIMESTAMP" default="true"/>
+				    <dbType type="DATE"/>
 				</talendType>
 				<talendType type="id_BigDecimal">
-					<dbType type="BIGINT" default="true"/>
+				    <dbType type="DECIMAL" default="true"/>
+					<dbType type="BIGINT"/>
 					<dbType type="FLOAT"/>
 					<dbType type="DOUBLE"/>
 				</talendType>
@@ -80,6 +84,9 @@
 					<talendType type="id_BigDecimal"/>
 					<talendType type="id_Float"/>
 				</dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true"/>
+                </dbType>
 				<dbType type="BIGINT">
 					<talendType type="id_BigDecimal" default="true"/>
 					<talendType type="id_Byte"/>
@@ -111,7 +118,10 @@
 				<dbType type="ARRAY">
 					<talendType type="id_Object" default="true"/>
 				</dbType>
-				<dbType type="TIMESTAMP">
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true"/>
+                </dbType>
+                <dbType type="DATE">
                     <talendType type="id_Date" default="true"/>
                 </dbType>
 			</dbToTalendTypes>

main/plugins/org.talend.core.runtime/mappings/mapping_Redshift_new.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0"?>
+<mapping>
+	<dbms product="Redshift" id="redshift_new_id" label="Mapping Redshift"
+		default="true">
+		<dbTypes>
+            <dbType type="SMALLINT" ignoreLen="true" ignorePre="true" />
+            <dbType type="INTEGER" ignoreLen="true" ignorePre="true" />
+            <dbType type="BIGINT" ignoreLen="true" ignorePre="true" />
+            <dbType type="DECIMAL" ignoreLen="true" />
+            <dbType type="REAL" ignoreLen="true" />
+            <dbType type="DOUBLE PRECISION" ignoreLen="true" />
+            <dbType type="BOOLEAN" ignoreLen="true" ignorePre="true" />
+            <dbType type="CHAR" ignoreLen="true" ignorePre="true" />
+            <dbType type="VARCHAR" ignorePre="true" />
+            <dbType type="DATE" ignoreLen="true" ignorePre="true" />
+            <dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true" />
+		</dbTypes>
+		<language name="java">
+            <talendToDbTypes>
+                <talendType type="id_List">
+                </talendType>
+                <talendType type="id_Boolean">
+                    <dbType type="BOOLEAN" default="true" />
+                </talendType>
+                <talendType type="id_Byte">
+                </talendType>
+                <talendType type="id_byte[]">
+                </talendType>
+                <talendType type="id_Character">
+                </talendType>
+                <talendType type="id_Date">
+                    <dbType type="DATE" default="true" />
+                    <dbType type="TIMESTAMP" />
+                </talendType>
+                <talendType type="id_BigDecimal">
+                    <dbType type="DECIMAL" default="true" />
+                    <dbType type="BIGINT" />
+                </talendType>
+                <talendType type="id_Double">
+                    <dbType type="DOUBLE PRECISION" default="true" />
+                    <dbType type="REAL" />
+                </talendType>
+                <talendType type="id_Float">
+                </talendType>
+                <talendType type="id_Integer">
+                    <dbType type="INTEGER" default="true" />
+                    <dbType type="SMALLINT" />
+                </talendType>
+                <talendType type="id_Long">
+                    <dbType type="BIGINT" default="true" />
+                    <dbType type="INTEGER" />
+                </talendType>
+                <talendType type="id_Object">
+                </talendType>    
+                <talendType type="id_Short">
+                </talendType>
+                <talendType type="id_String">
+                    <dbType type="VARCHAR" default="true" />
+                    <dbType type="CHAR" />
+                </talendType>
+            </talendToDbTypes>
+            <dbToTalendTypes>
+                <dbType type="SMALLINT">
+                    <talendType type="id_Integer" default="true" />
+                </dbType>
+                <dbType type="INTEGER">
+                    <talendType type="id_Integer" default="true" />
+                    <talendType type="id_Long" />
+                </dbType>
+                <dbType type="BIGINT">
+                    <talendType type="id_Long" default="true" />
+                    <talendType type="id_BigDecimal" />
+                </dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true" />
+                </dbType>
+                <dbType type="REAL">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="DOUBLE PRECISION">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="BOOLEAN">
+                    <talendType type="id_Boolean" default="true" />
+                </dbType>
+                <dbType type="CHAR">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="VARCHAR">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="DATE">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+            </dbToTalendTypes>		  
+		</language>
+	</dbms>
+</mapping>
+        

main/plugins/org.talend.core.runtime/resources/hadoop/hadoop_default_confs.json

@@ -10,6 +10,11 @@
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
         "JOBHISTORY_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -35,6 +40,11 @@
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
         "JOBHISTORY_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -56,6 +66,11 @@
         "RESOURCEMANAGER_SCHEDULER_ADDRESS": "localhost:8030",
         "JOBHISTORY_ADDRESS": "0.0.0.0:10020",
         "STAGING_DIRECTORY": "/user",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -79,6 +94,11 @@
         "NAMENODE_PRINCIPAL": "nn/_HOST@EXAMPLE.COM",
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -150,6 +170,11 @@
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
         "JOBHISTORY_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -174,6 +199,11 @@
         "NAMENODE_PRINCIPAL": "nn/_HOST@EXAMPLE.COM",
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/classloader/ClassLoaderFactory.java

@@ -34,7 +34,8 @@ import org.talend.core.ILibraryManagerService;
 import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.IMetadataConnection;
-import org.talend.core.model.metadata.connection.hive.HiveConnUtils;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.hive.HiveMetadataHelper;
 import org.talend.repository.ProjectManager;
 import org.talend.utils.io.FilesUtils;
 
@@ -43,21 +44,21 @@ import org.talend.utils.io.FilesUtils;
  */
 public class ClassLoaderFactory {
 
-    private final static String EXTENSION_POINT_ID = "org.talend.core.runtime.classloader_provider"; //$NON-NLS-1$
+    public final static String EXTENSION_POINT_ID = "org.talend.core.runtime.classloader_provider"; //$NON-NLS-1$
 
     private static IConfigurationElement[] configurationElements = null;
 
     private static Map<String, DynamicClassLoader> classLoadersMap = null;
 
-    private final static String SEPARATOR = ";"; //$NON-NLS-1$
+    public final static String SEPARATOR = ";"; //$NON-NLS-1$
 
     private final static String PATH_SEPARATOR = "/"; //$NON-NLS-1$
 
-    private final static String INDEX_ATTR = "index"; //$NON-NLS-1$
+    public final static String INDEX_ATTR = "index"; //$NON-NLS-1$
 
-    private final static String LIB_ATTR = "libraries"; //$NON-NLS-1$
+    public final static String LIB_ATTR = "libraries"; //$NON-NLS-1$
 
-    private final static String PARENT_ATTR = "parent"; //$NON-NLS-1$
+    public final static String PARENT_ATTR = "parent"; //$NON-NLS-1$
 
     static {
         IExtensionRegistry registry = Platform.getExtensionRegistry();
@@ -166,44 +167,67 @@ public class ClassLoaderFactory {
         classLoadersMap = new ConcurrentHashMap<String, DynamicClassLoader>();
     }
 
-    private static synchronized DynamicClassLoader findLoader(String index, ClassLoader parentLoader,
-            boolean showDownloadIfNotExist) {
-        if (index != null && configurationElements != null) {
+    public static IConfigurationElement findIndex(String index) {
+        if (StringUtils.isNotEmpty(index) && configurationElements != null) {
             for (IConfigurationElement current : configurationElements) {
                 String key = current.getAttribute(INDEX_ATTR);
                 if (index.equals(key)) {
-                    String libraries = current.getAttribute(LIB_ATTR);
-                    if (StringUtils.isNotEmpty(index)) {
-                        DynamicClassLoader classLoader = null;
-                        ClassLoader parentClassLoader = null;
-                        String parentKey = current.getAttribute(PARENT_ATTR);
-                        // take parent classlaoder in extensions first
-                        if (StringUtils.isNotEmpty(parentKey)) {
-                            parentClassLoader = getClassLoader(parentKey, showDownloadIfNotExist);
-                        }
-                        if (parentClassLoader == null) {
-                            parentClassLoader = parentLoader;
-                        }
-
-                        if (parentClassLoader == null) {
-                            classLoader = new DynamicClassLoader();
-                        } else {
-                            classLoader = new DynamicClassLoader(parentClassLoader);
-                        }
-                        boolean putInCache = true;
-                        if (StringUtils.isNotEmpty(libraries)) {
-                            String[] librariesArray = libraries.split(SEPARATOR);
-                            putInCache = loadLibraries(classLoader, librariesArray, showDownloadIfNotExist);
-                        }
-                        if (putInCache) {
-                            // if any libraries can't be retreived , do not put it in cache
-                            classLoadersMap.put(index, classLoader);
-                        }
-                        return classLoader;
-                    }
+                    return current;
                 }
             }
         }
+        return null;
+    }
+
+    public static String[] getLibs(String index) {
+        IConfigurationElement current = findIndex(index);
+        return getLibs(current);
+    }
+
+    public static String[] getLibs(IConfigurationElement current) {
+        if (current != null) {
+            String libraries = current.getAttribute(LIB_ATTR);
+            if (StringUtils.isNotEmpty(libraries)) {
+                return libraries.split(SEPARATOR);
+            }
+        }
+        return new String[0];
+    }
+
+    private static synchronized DynamicClassLoader findLoader(String index, ClassLoader parentLoader,
+            boolean showDownloadIfNotExist) {
+        IConfigurationElement current = findIndex(index);
+        if (current != null) {
+            // String key = current.getAttribute(INDEX_ATTR);
+            // String libraries = current.getAttribute(LIB_ATTR);
+            String parentKey = current.getAttribute(PARENT_ATTR);
+
+            ClassLoader parentClassLoader = null;
+            // take parent classlaoder in extensions first
+            if (StringUtils.isNotEmpty(parentKey)) {
+                parentClassLoader = getClassLoader(parentKey, showDownloadIfNotExist);
+            }
+            if (parentClassLoader == null) {
+                parentClassLoader = parentLoader;
+            }
+
+            DynamicClassLoader classLoader = null;
+            if (parentClassLoader == null) {
+                classLoader = new DynamicClassLoader();
+            } else {
+                classLoader = new DynamicClassLoader(parentClassLoader);
+            }
+            boolean putInCache = true;
+            String[] librariesArray = getLibs(current);
+            if (librariesArray.length > 0) {
+                putInCache = loadLibraries(classLoader, librariesArray, showDownloadIfNotExist);
+            }
+            if (putInCache) {
+                // if any libraries can't be retreived , do not put it in cache
+                classLoadersMap.put(index, classLoader);
+            }
+            return classLoader;
+        }
 
         return null;
     }
@@ -275,7 +299,9 @@ public class ClassLoaderFactory {
         String distroKey = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_DISTRIBUTION);
         String distroVersion = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_VERSION);
         String hiveModel = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_MODE);
-        if (HiveConnUtils.isCustomDistro(distroKey)) {
+
+        IHDistribution distribution = HiveMetadataHelper.getDistribution(distroKey, false);
+        if (distribution != null && distribution.useCustom()) {
             String jarsStr = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HADOOP_CUSTOM_JARS);
             moduleList = jarsStr.split(";"); //$NON-NLS-1$
         } else {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -23,7 +23,7 @@ import org.talend.core.database.conn.template.DbConnStrForHive;
 import org.talend.core.database.conn.template.EDatabaseConnTemplate;
 import org.talend.core.database.conn.version.EDatabaseVersion4Drivers;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
-import org.talend.core.model.metadata.connection.hive.HiveConnVersionInfo;
+import org.talend.core.model.metadata.connection.hive.HiveModeInfo;
 import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.utils.TalendQuoteUtils;
 
@@ -191,15 +191,16 @@ public class DatabaseConnStrUtil {
         }
         // DbConnStrForHive.URL_HIVE_2_TEMPLATE or DbConnStrForHive.URL_HIVE_1_TEMPLATE
         // set a default
+        boolean isEmbedded = (HiveModeInfo.get(hiveModel) == HiveModeInfo.EMBEDDED);
         String url = null;
         if (template.startsWith(DbConnStrForHive.URL_HIVE_2_TEMPLATE)) {
-            if (HiveConnVersionInfo.MODE_EMBEDDED.getKey().equalsIgnoreCase(hiveModel)) {
+            if (isEmbedded) {
                 url = getHive2EmbeddedURLString();
             } else {
                 url = getHive2StandaloneURLString(false, server, port, sidOrDatabase, hivePrincipal);
             }
         } else if (template.startsWith(DbConnStrForHive.URL_HIVE_1_TEMPLATE)) {
-            if (HiveConnVersionInfo.MODE_EMBEDDED.getKey().equalsIgnoreCase(hiveModel)) {
+            if (isEmbedded) {
                 url = getHive1EmbeddedURLString();
             } else {
                 url = getHive1StandaloneURLString(false, server, port, sidOrDatabase);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/EDatabaseConnTemplate.java

@@ -81,7 +81,7 @@ public enum EDatabaseConnTemplate {
             "jdbc:odbc:<datasource>")), //$NON-NLS-1$
 
     ACCESS(new DbConnStr(EDatabaseTypeName.ACCESS, //
-            "jdbc:ucanaccess://<filename>")), //$NON-NLS-1$
+            "jdbc:ucanaccess://<filename>;<property>",null,"memory=false")), //$NON-NLS-1$
 
     SYBASEASE(new DbConnStr(EDatabaseTypeName.SYBASEASE, //
             "jdbc:sybase:Tds:<host>:<port>/<sid>?<property>", //$NON-NLS-1$ 
@@ -357,6 +357,7 @@ public enum EDatabaseConnTemplate {
             case TERADATA: // for feature 0013719
             case VERTICA:
             case EXASOL:
+            case ACCESS:
                 return true;
             default:
             }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/hbase/conn/version/EHBaseDistribution4Versions.java

@@ -1,208 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.database.hbase.conn.version;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
-
-/**
- * DOC ycbai class global comment. Detailled comment
- */
-public enum EHBaseDistribution4Versions {
-    HDP_2_3(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_3.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_3
-            .getVersionValue()),
-
-    HDP_2_2(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_2.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_2
-            .getVersionValue()),
-
-    HDP_2_1(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_1.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_1
-            .getVersionValue()),
-
-    HDP_2_0(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_0.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_0
-            .getVersionValue()),
-
-    HDP_1_3(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_1_3.getVersionDisplay(), EHadoopVersion4Drivers.HDP_1_3
-            .getVersionValue()),
-
-    HDP_1_2(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_1_2.getVersionDisplay(), EHadoopVersion4Drivers.HDP_1_2
-            .getVersionValue()),
-
-    APACHE_2_4_0_EMR(
-                     EHBaseDistributions.AMAZON_EMR,
-                     EHadoopVersion4Drivers.APACHE_2_4_0_EMR.getVersionDisplay(),
-                     EHadoopVersion4Drivers.APACHE_2_4_0_EMR.getVersionValue()),
-
-    APACHE_1_0_3_EMR(
-                     EHBaseDistributions.AMAZON_EMR,
-                     EHadoopVersion4Drivers.APACHE_1_0_3_EMR.getVersionDisplay(),
-                     EHadoopVersion4Drivers.APACHE_1_0_3_EMR.getVersionValue()),
-
-    APACHE_1_0_0(
-                 EHBaseDistributions.APACHE,
-                 EHadoopVersion4Drivers.APACHE_1_0_0.getVersionDisplay(),
-                 EHadoopVersion4Drivers.APACHE_1_0_0.getVersionValue()),
-
-    CLOUDERA_CDH5_5(
-                    EHBaseDistributions.CLOUDERA,
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_5.getVersionDisplay(),
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_5.getVersionValue()),
-
-    CLOUDERA_CDH5_4(
-                    EHBaseDistributions.CLOUDERA,
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_4.getVersionDisplay(),
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_4.getVersionValue()),
-
-    CLOUDERA_CDH5_1(
-                    EHBaseDistributions.CLOUDERA,
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_1.getVersionDisplay(),
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_1.getVersionValue()),
-
-    CLOUDERA_CDH5_1_MR1(
-                        EHBaseDistributions.CLOUDERA,
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionDisplay(),
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionValue()),
-
-    CLOUDERA_CDH5(
-                  EHBaseDistributions.CLOUDERA,
-                  EHadoopVersion4Drivers.CLOUDERA_CDH5.getVersionDisplay(),
-                  EHadoopVersion4Drivers.CLOUDERA_CDH5.getVersionValue()),
-
-    CLOUDERA_CDH4_YARN(
-                       EHBaseDistributions.CLOUDERA,
-                       EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN.getVersionDisplay(),
-                       EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN.getVersionValue()),
-
-    CLOUDERA_CDH4(
-                  EHBaseDistributions.CLOUDERA,
-                  EHadoopVersion4Drivers.CLOUDERA_CDH4.getVersionDisplay(),
-                  EHadoopVersion4Drivers.CLOUDERA_CDH4.getVersionValue()),
-
-    MAPR_5_0_0(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR500.getVersionDisplay(), EHadoopVersion4Drivers.MAPR500
-            .getVersionValue()),
-
-    MAPR_4_1_0(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR410.getVersionDisplay(), EHadoopVersion4Drivers.MAPR410
-            .getVersionValue()),
-
-    MAPR_4_0_1(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR401.getVersionDisplay(), EHadoopVersion4Drivers.MAPR401
-            .getVersionValue()),
-
-    MAPR_3_1_0(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR310.getVersionDisplay(), EHadoopVersion4Drivers.MAPR310
-            .getVersionValue()),
-
-    MAPR_3_0_1(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR301.getVersionDisplay(), EHadoopVersion4Drivers.MAPR301
-            .getVersionValue()),
-
-    MAPR_2_1_3(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR213.getVersionDisplay(), EHadoopVersion4Drivers.MAPR213
-            .getVersionValue()),
-
-    MAPR_2_1_2(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR212.getVersionDisplay(), EHadoopVersion4Drivers.MAPR212
-            .getVersionValue()),
-
-    MAPR(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR2.getVersionDisplay(), EHadoopVersion4Drivers.MAPR2
-            .getVersionValue()),
-
-    PIVOTAL_HD_2_0(
-                   EHBaseDistributions.PIVOTAL_HD,
-                   EHadoopVersion4Drivers.PIVOTAL_HD_2_0.getVersionDisplay(),
-                   EHadoopVersion4Drivers.PIVOTAL_HD_2_0.getVersionValue()),
-
-    PIVOTAL_HD_1_0_1(
-                     EHBaseDistributions.PIVOTAL_HD,
-                     EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1.getVersionDisplay(),
-                     EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1.getVersionValue()),
-
-    CUSTOM(EHBaseDistributions.CUSTOM, EHadoopVersion4Drivers.CUSTOM.getVersionDisplay(), EHadoopVersion4Drivers.CUSTOM
-            .getVersionValue());
-
-    private EHBaseDistributions distribution;
-
-    private String versionDisplayName;
-
-    private String versionValue;
-
-    EHBaseDistribution4Versions(EHBaseDistributions distribution, String versionDisplayName, String versionValue) {
-        this.distribution = distribution;
-        this.versionDisplayName = versionDisplayName;
-        this.versionValue = versionValue;
-    }
-
-    public static List<EHBaseDistribution4Versions> indexOfByDistribution(String distribution) {
-        List<EHBaseDistribution4Versions> distribution4Versions = new ArrayList<EHBaseDistribution4Versions>();
-        if (distribution != null) {
-            for (EHBaseDistribution4Versions d4v : EHBaseDistribution4Versions.values()) {
-                if (d4v.getDistribution().getName().equals(distribution)
-                        || d4v.getDistribution().getDisplayName().equals(distribution)) {
-                    distribution4Versions.add(d4v);
-                }
-            }
-        }
-        return distribution4Versions;
-    }
-
-    public static EHBaseDistribution4Versions indexOfByVersionDisplay(String displayName) {
-        return indexOf(displayName, true);
-    }
-
-    public static EHBaseDistribution4Versions indexOfByVersion(String value) {
-        return indexOf(value, false);
-    }
-
-    private static EHBaseDistribution4Versions indexOf(String name, boolean display) {
-        if (name != null) {
-            for (EHBaseDistribution4Versions version : EHBaseDistribution4Versions.values()) {
-                if (display) {
-                    if (name.equalsIgnoreCase(version.getVersionDisplayName())) {
-                        return version;
-                    }
-                } else {
-                    if (name.equalsIgnoreCase(version.getVersionValue())) {
-                        return version;
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    public static List<String> getHadoopDistributionVersions(String distribution) {
-        return getHadoopDistributionVersions(distribution, true);
-    }
-
-    public static List<String> getHadoopDistributionVersions(String distribution, boolean display) {
-        List<String> result = new ArrayList<String>();
-        List<EHBaseDistribution4Versions> d4vList = EHBaseDistribution4Versions.indexOfByDistribution(distribution);
-        for (EHBaseDistribution4Versions d4v : d4vList) {
-            if (display) {
-                result.add(d4v.getVersionDisplayName());
-            } else {
-                result.add(d4v.getVersionValue());
-            }
-        }
-        return result;
-    }
-
-    public EHBaseDistributions getDistribution() {
-        return this.distribution;
-    }
-
-    public String getVersionDisplayName() {
-        return this.versionDisplayName;
-    }
-
-    public String getVersionValue() {
-        return this.versionValue;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/hbase/conn/version/EHBaseDistributions.java

@@ -1,92 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.database.hbase.conn.version;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.talend.core.hadoop.version.EHadoopDistributions;
-
-/**
- * DOC ycbai class global comment. Detailled comment
- */
-public enum EHBaseDistributions {
-
-    HORTONWORKS(EHadoopDistributions.HORTONWORKS.getDisplayName()),
-
-    CLOUDERA(EHadoopDistributions.CLOUDERA.getDisplayName()),
-
-    MAPR(EHadoopDistributions.MAPR.getDisplayName()),
-
-    APACHE(EHadoopDistributions.APACHE.getDisplayName()),
-
-    AMAZON_EMR(EHadoopDistributions.AMAZON_EMR.getDisplayName()),
-
-    PIVOTAL_HD(EHadoopDistributions.PIVOTAL_HD.getDisplayName()),
-
-    CUSTOM(EHadoopDistributions.CUSTOM.getDisplayName());
-
-    private String displayName;
-
-    EHBaseDistributions(String displayName) {
-        this.displayName = displayName;
-    }
-
-    public String getName() {
-        return name();
-    }
-
-    public String getDisplayName() {
-        return this.displayName;
-    }
-
-    public static List<String> getAllDistributionDisplayNames() {
-        return getAllDistributionNames(true);
-    }
-
-    public static List<String> getAllDistributionNames(boolean display) {
-        List<String> names = new ArrayList<String>();
-        EHBaseDistributions[] values = values();
-        for (EHBaseDistributions distribution : values) {
-            if (display) {
-                names.add(distribution.getDisplayName());
-            } else {
-                names.add(distribution.getName());
-            }
-        }
-
-        return names;
-    }
-
-    public static EHBaseDistributions getDistributionByDisplayName(String name) {
-        return getDistributionByName(name, true);
-    }
-
-    public static EHBaseDistributions getDistributionByName(String name, boolean display) {
-        if (name != null) {
-            for (EHBaseDistributions distribution : values()) {
-                if (display) {
-                    if (name.equalsIgnoreCase(distribution.getDisplayName())) {
-                        return distribution;
-                    }
-                } else {
-                    if (name.equalsIgnoreCase(distribution.getName())) {
-                        return distribution;
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConstants.java

@@ -32,6 +32,8 @@ public class HadoopConstants {
     public static final String FRAMEWORK_STORM = "Storm";
 
     public static final String FRAMEWORK_SPARKSTREAMING = "Spark Streaming";
+    
+    public static final String FRAMEWORK_SPARK_STREAMING = "Spark_Streaming";
 
     public static final String FRAMEWORK_MAPREDUCE = "MapReduce";
 
@@ -40,6 +42,10 @@ public class HadoopConstants {
     public static final String JOBTYPEBDSTREAMING = "Big Data Streaming";
 
     public static final String JOBTYPEBDBATCH = "Big Data Batch";
+    
+    public static final String JOBTYPE_BDSTREAMING = "Big_Data_Streaming";
+
+    public static final String JOBTYPE_BDBATCH = "Big_Data_Batch";
 
     public static final String SPARKSTREAMING_TYPE = "SPARKSTREAMING";
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopClusterService.java

@@ -216,6 +216,8 @@ public interface IHadoopClusterService extends IService {
      */
     public boolean hasDiffsFromClusterToProcess(Item item, IProcess process);
 
+    public Connection getHadoopClusterConnectionBySubConnection(Connection hadoopSubConnection);
+
     public String getHadoopClusterProperties(Connection hadoopSubConnection);
 
     /**

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopDistributionService.java

@@ -12,18 +12,85 @@
 // ============================================================================
 package org.talend.core.hadoop;
 
-import org.talend.core.IService;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
+import java.util.Map;
 
+import org.talend.core.IService;
+import org.talend.core.runtime.hd.IDistributionsManager;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.IHDistributionVersion;
 
 /**
- * created by cmeng on Jan 15, 2016
- * Detailled comment
+ * created by cmeng on Jan 15, 2016 Detailled comment
  *
  */
 public interface IHadoopDistributionService extends IService {
 
-    public boolean isSupportSpark(EHadoopVersion4Drivers version);
+    /**
+     * 
+     * According to the service, find the distributions. The service is name of HadoopComponent service.
+     * {@link org.talend.hadoop.distribution.component.HadoopComponent}.
+     */
+    IHDistribution[] getDistributions(String service);
+
+    /**
+     * Check the distribution version support the service or not.
+     * 
+     * The service must be full name, and be sub class of
+     * {@link org.talend.hadoop.distribution.component.HadoopComponent)
+     */
+    boolean doSupportService(IHDistributionVersion distributionVersion, String service);
+
+    /**
+     * Check the boolean methods for sub class of {@link org.talend.hadoop.distribution.component.HadoopComponent)
+     * 
+     * If no the method in the HadoopComponent object, won't contain in the result map.
+     */
+    Map<String, Boolean> doSupportMethods(IHDistributionVersion distributionVersion, String... methods) throws Exception;
+
+    boolean doSupportMethod(IHDistributionVersion distributionVersion, String method) throws Exception;
+
+    /**
+     * Find the matched distribution via name.
+     * 
+     * If the display is true, the name match for display one of distribution.
+     */
+    IHDistribution getHadoopDistribution(String name, boolean byDisplay);
+
+    /**
+     * Find the matched distribution via version .
+     * 
+     * If the display is true, the version match for display one of distribution.
+     */
+    IHDistributionVersion getHadoopDistributionVersion(String version, boolean byDisplay);
+
+    /**
+     * 
+     * for Hadoop Distribution {@link HadoopComponent}.
+     */
+    IDistributionsManager getHadoopDistributionManager();
+
+    /**
+     * 
+     * for HBase in Database wizard.{@link HBaseComponent}.
+     */
+    IDistributionsManager getHBaseDistributionManager();
+
+    /**
+     * 
+     * for Hive Distribution {@link HiveComponent}.
+     */
+    IDistributionsManager getSparkDistributionManager();
+
+    /**
+     * 
+     * for Spark Distribution {@link SparkComponent}.
+     */
+    IDistributionsManager getHiveDistributionManager();
+
+    /**
+     * 
+     * Temp for Oozie.
+     */
+    IHDistribution[] getOozieDistributions();
 
-    public boolean isSupportSparkStreaming(EHadoopVersion4Drivers version);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/conf/EHadoopProperties.java

@@ -45,6 +45,16 @@ public enum EHadoopProperties {
     DATABASE,
 
     PORT,
+    
+    CLOUDERA_NAVIGATOR_USERNAME,
+    
+    CLOUDERA_NAVIGATOR_PASSWORD,
+    
+    CLOUDERA_NAVIGATOR_URL,
+    
+    CLOUDERA_NAVIGATOR_METADATA_URL,
+    
+    CLOUDERA_NAVIGATOR_CLIENT_URL,
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/repository/HadoopRepositoryUtil.java

@@ -139,8 +139,11 @@ public class HadoopRepositoryUtil {
         }
         List<Map<String, Object>> parentProperties = null;
         if (hadoopClusterService != null) {
-            parentProperties = getHadoopPropertiesList(hadoopClusterService.getHadoopClusterProperties(connection),
-                    connection.isContextMode(), true);
+            Connection hadoopClusterConnection = hadoopClusterService.getHadoopClusterConnectionBySubConnection(connection);
+            if (hadoopClusterConnection != null) {
+                parentProperties = getHadoopPropertiesList(hadoopClusterService.getHadoopClusterProperties(connection),
+                        hadoopClusterConnection.isContextMode(), true);
+            }
         }
         List<Map<String, Object>> properties = getHadoopPropertiesList(propertiesJsonStr, connection.isContextMode(), true);
         Map<String, Map<String, Object>> propertiesMap = new HashMap<String, Map<String, Object>>();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/EHadoopDistributions.java

@@ -12,11 +12,10 @@
 // ============================================================================
 package org.talend.core.hadoop.version;
 
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * DOC ycbai class global comment. Detailled comment
+ * 
+ * @deprecated @link HadoopComponent via osgi service instead.
  */
 public enum EHadoopDistributions {
 
@@ -50,28 +49,6 @@ public enum EHadoopDistributions {
         return this.displayName;
     }
 
-    public static List<String> getAllDistributionDisplayNames() {
-        return getAllDistributionNames(true);
-    }
-
-    public static List<String> getAllDistributionNames(boolean display) {
-        List<String> names = new ArrayList<String>();
-        EHadoopDistributions[] values = values();
-        for (EHadoopDistributions distribution : values) {
-            if (display) {
-                names.add(distribution.getDisplayName());
-            } else {
-                names.add(distribution.getName());
-            }
-        }
-
-        return names;
-    }
-
-    public static EHadoopDistributions getDistributionByDisplayName(String name) {
-        return getDistributionByName(name, true);
-    }
-
     public static EHadoopDistributions getDistributionByName(String name, boolean display) {
         if (name != null) {
             for (EHadoopDistributions distribution : values()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/EHadoopVersion4Drivers.java

@@ -16,15 +16,20 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.ArrayUtils;
-import org.talend.commons.exception.ExceptionHandler;
-import org.talend.core.GlobalServiceRegister;
-import org.talend.core.hadoop.IHadoopDistributionService;
 
 /**
  * DOC ycbai class global comment. Detailled comment
+ * 
+ * @deprecated @link HadoopComponent via osgi service instead.
  */
 public enum EHadoopVersion4Drivers {
 
+    HDP_2_4(EHadoopDistributions.HORTONWORKS, "Hortonworks Data Platform V2.4.0", //$NON-NLS-1$
+            "HDP_2_4", //$NON-NLS-1$
+            true,
+            false,
+            new EMRVersion[] { EMRVersion.YARN }),
+
     HDP_2_3(EHadoopDistributions.HORTONWORKS, "Hortonworks Data Platform V2.3.0", //$NON-NLS-1$
             "HDP_2_3", //$NON-NLS-1$
             true,
@@ -93,6 +98,8 @@ public enum EHadoopVersion4Drivers {
 
     CLOUDERA_CDH4(EHadoopDistributions.CLOUDERA, "Cloudera CDH4.X(MR1 mode)", "Cloudera_CDH4", true, false), //$NON-NLS-1$ //$NON-NLS-2$
 
+    MAPR510(EHadoopDistributions.MAPR, "MapR 5.1.0(YARN mode)", "MAPR510", true, true, new EMRVersion[] { EMRVersion.YARN }), //$NON-NLS-1$ //$NON-NLS-2$
+
     MAPR500(EHadoopDistributions.MAPR, "MapR 5.0.0(YARN mode)", "MAPR500", true, true, new EMRVersion[] { EMRVersion.YARN }), //$NON-NLS-1$ //$NON-NLS-2$
 
     MAPR410(EHadoopDistributions.MAPR, "MapR 4.1.0(YARN mode)", "MAPR410", false, true, new EMRVersion[] { EMRVersion.YARN }), //$NON-NLS-1$ //$NON-NLS-2$
@@ -115,6 +122,12 @@ public enum EHadoopVersion4Drivers {
               false,
               new EMRVersion[] { EMRVersion.YARN }),
 
+    EMR_4_5_0(EHadoopDistributions.AMAZON_EMR, "EMR 4.5.0 (Apache 2.7.2)", //$NON-NLS-1$
+              "EMR_4_5_0", //$NON-NLS-1$
+              false,
+              false,
+              new EMRVersion[] { EMRVersion.YARN }),
+
     APACHE_2_4_0_EMR(EHadoopDistributions.AMAZON_EMR, "Apache 2.4.0", //$NON-NLS-1$
                      "APACHE_2_4_0_EMR", //$NON-NLS-1$
                      false,
@@ -245,15 +258,4 @@ public enum EHadoopVersion4Drivers {
         return ArrayUtils.contains(getMrVersions(), EMRVersion.YARN);
     }
 
-    public boolean isSupportSpark() {
-        boolean isSupport = false;
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
-            IHadoopDistributionService hadoopDistributionService = (IHadoopDistributionService) GlobalServiceRegister.getDefault()
-                    .getService(IHadoopDistributionService.class);
-            isSupport = hadoopDistributionService.isSupportSpark(this);
-        } else {
-            ExceptionHandler.process(new Exception(IHadoopDistributionService.class.getName() + " is not registed.")); //$NON-NLS-1$
-        }
-        return isSupport;
-    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/EMRVersion.java

@@ -18,12 +18,13 @@ import java.util.List;
 /**
  * created by ycbai on 2013-9-16 Detailled comment
  * 
+ * @deprecated @link EHadoopVersion instead.
  */
 public enum EMRVersion {
 
-    MR1,
+    MR1, // EHadoopVersion.HADOOP_1
 
-    YARN,
+    YARN, // EHadoopVersion.HADOOP_2
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/HadoopVersionDialog.java

@@ -32,14 +32,17 @@ import org.talend.commons.ui.runtime.image.ImageProvider;
 import org.talend.commons.ui.swt.formtools.LabelledCombo;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.hadoop.HadoopConstants;
+import org.talend.core.hadoop.IHadoopDistributionService;
 import org.talend.core.hadoop.IHadoopService;
-import org.talend.core.hadoop.version.EHadoopDistributions;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.runtime.CoreRuntimePlugin;
+import org.talend.core.runtime.hd.IDistributionsManager;
+import org.talend.core.runtime.hd.IHDConstants;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.IHDistributionVersion;
 import org.talend.core.runtime.i18n.Messages;
 
 /**
@@ -99,6 +102,13 @@ public class HadoopVersionDialog extends TitleAreaDialog {
         this.types = types;
     }
 
+    private IHadoopDistributionService getHadoopDistributionService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
+            return (IHadoopDistributionService) GlobalServiceRegister.getDefault().getService(IHadoopDistributionService.class);
+        }
+        return null;
+    }
+
     @Override
     protected void configureShell(Shell newShell) {
         super.configureShell(newShell);
@@ -267,16 +277,24 @@ public class HadoopVersionDialog extends TitleAreaDialog {
         }
     }
 
+    private IHDistribution getHadoopDistribution() {
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            final IHDistribution distributionByDisplay = hadoopDistributionService.getHadoopDistribution(
+                    distributionCombo.getText(), true);
+            return distributionByDisplay;
+        }
+        return null;
+    }
+
     private void addListener() {
         distributionCombo.addModifyListener(new ModifyListener() {
 
             @Override
             public void modifyText(final ModifyEvent e) {
-                String newDistributionDisplayName = distributionCombo.getText();
-                EHadoopDistributions newDistribution = EHadoopDistributions
-                        .getDistributionByDisplayName(newDistributionDisplayName);
-                if (newDistribution != null) {
-                    distribution = newDistribution.getName();
+                final IHDistribution distributionByDisplay = getHadoopDistribution();
+                if (distributionByDisplay != null) {
+                    distribution = distributionByDisplay.getName();
                     updateVersionPart();
                 }
             }
@@ -286,10 +304,14 @@ public class HadoopVersionDialog extends TitleAreaDialog {
 
             @Override
             public void modifyText(final ModifyEvent e) {
+                final IHDistribution distributionByDisplay = getHadoopDistribution();
                 String newVersionDisplayName = versionCombo.getText();
-                EHadoopVersion4Drivers newVersion4Drivers = EHadoopVersion4Drivers.indexOfByVersionDisplay(newVersionDisplayName);
-                if (newVersion4Drivers != null) {
-                    version = newVersion4Drivers.getVersionValue();
+
+                if (distributionByDisplay != null) {
+                    final IHDistributionVersion hdVersion = distributionByDisplay.getHDVersion(newVersionDisplayName, true);
+                    if (hdVersion != null) {
+                        version = hdVersion.getVersion();
+                    }
                 }
             }
         });
@@ -336,48 +358,27 @@ public class HadoopVersionDialog extends TitleAreaDialog {
     }
 
     private void init() {
-        distributionCombo.getCombo().setItems(getDistributions().toArray(new String[0]));
-        distributionCombo.select(0);
-    }
-
-    protected List<String> getDistributions() {
-        List<String> distributions = null;
-        if (isSparkJob()) {
-            EHadoopVersion4Drivers hadoopVersions[] = EHadoopVersion4Drivers.values();
-            Set<EHadoopDistributions> hadoopDistributions = new HashSet<EHadoopDistributions>();
-            for (EHadoopVersion4Drivers hadoopVersion : hadoopVersions) {
-                if (hadoopVersion.isSupportSpark()) {
-                    hadoopDistributions.add(hadoopVersion.getDistribution());
+        List<String> distributionsDisplay = new ArrayList<String>();
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            IDistributionsManager distributionManager = null;
+            if (isSparkJob()) {
+                distributionManager = hadoopDistributionService.getSparkDistributionManager();
+            } else {
+                distributionManager = hadoopDistributionService.getHadoopDistributionManager();
+            }
+            IHDistribution[] distributions = distributionManager.getDistributions();
+            if (distributions != null) {
+                for (IHDistribution d : distributions) {
+                    if (!d.useCustom()) {
+                        distributionsDisplay.add(d.getDisplayName());
+                    }
                 }
             }
-            distributions = new ArrayList<String>(hadoopDistributions.size());
-            for (EHadoopDistributions hadoopDistribution : hadoopDistributions) {
-                distributions.add(hadoopDistribution.getDisplayName());
-            }
-
-        } else {
-            distributions = EHadoopDistributions.getAllDistributionDisplayNames();
         }
-        distributions.remove(EHadoopDistributions.CUSTOM.getDisplayName());
 
-        return distributions;
-    }
-
-    protected List<String> getVersions(EHadoopDistributions dis) {
-        List<String> result = new ArrayList<String>();
-        boolean isSparkJob = isSparkJob();
-        List<EHadoopVersion4Drivers> v4dList = EHadoopVersion4Drivers.indexOfByDistribution(dis);
-        for (EHadoopVersion4Drivers v4d : v4dList) {
-            boolean needToAdd = true;
-            if (isSparkJob && !v4d.isSupportSpark()) {
-                needToAdd = false;
-            }
-
-            if (needToAdd) {
-                result.add(v4d.getVersionDisplay());
-            }
-        }
-        return result;
+        distributionCombo.getCombo().setItems(distributionsDisplay.toArray(new String[0]));
+        distributionCombo.select(0);
     }
 
     protected boolean isSparkJob() {
@@ -390,13 +391,34 @@ public class HadoopVersionDialog extends TitleAreaDialog {
     }
 
     private void updateVersionPart() {
-        EHadoopDistributions dis = EHadoopDistributions.getDistributionByName(distribution, false);
-        List<String> items = getVersions(dis);
-        String[] versions = new String[items.size()];
-        items.toArray(versions);
-        versionCombo.getCombo().setItems(versions);
-        if (versions.length > 0) {
-            versionCombo.getCombo().select(0);
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            final IHDistribution hDistribution = hadoopDistributionService.getHadoopDistribution(distribution, false);
+            if (hDistribution != null) {
+                boolean isSparkJob = isSparkJob();
+                final IHDistributionVersion[] hdVersions = hDistribution.getHDVersions();
+                List<String> versionsDisplay = new ArrayList<String>();
+                for (IHDistributionVersion hdv : hdVersions) {
+                    final String displayVersion = hdv.getDisplayVersion();
+                    if (displayVersion == null) {
+                        continue;
+                    }
+                    if (isSparkJob) {
+                        if (hadoopDistributionService.doSupportService(hdv, IHDConstants.SERVICE_SPARK)) {
+                            versionsDisplay.add(displayVersion);
+                        }
+                    } else {
+                        versionsDisplay.add(displayVersion);
+                    }
+                }
+                versionCombo.getCombo().setItems(versionsDisplay.toArray(new String[0]));
+                IHDistributionVersion defaultVersion = hDistribution.getDefaultVersion();
+                if (defaultVersion != null) {
+                    versionCombo.getCombo().setText(defaultVersion.getDisplayVersion());
+                } else if (!versionsDisplay.isEmpty()) {
+                    versionCombo.getCombo().select(0);
+                }
+            }
         }
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/ComponentCategory.java

@@ -17,6 +17,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.service.IMRProcessService;
+import org.talend.core.service.IStormProcessService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 
 /**
@@ -96,6 +97,16 @@ public enum ComponentCategory {
                 return ComponentCategory.CATEGORY_4_MAPREDUCE;
             }
         }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IStormProcessService.class)) {
+            IStormProcessService stormService = (IStormProcessService) GlobalServiceRegister.getDefault().getService(
+                    IStormProcessService.class);
+            if (stormService.isSparkStreaming(item)) {
+                return ComponentCategory.CATEGORY_4_SPARKSTREAMING;
+            }
+            if (stormService.isStormItem(item)) {
+                return ComponentCategory.CATEGORY_4_STORM;
+            }
+        }
         if (item instanceof ProcessItem || item instanceof JobletProcessItem) {
             return ComponentCategory.CATEGORY_4_DI;
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleNeeded.java

@@ -162,7 +162,12 @@ public class ModuleNeeded {
     public ModuleNeeded(String context, String informationMsg, boolean required, String mvnUri) {
         this(context, null, informationMsg, required, null, null, mvnUri, ELibraryInstallStatus.UNKNOWN);
         MavenArtifact mavenArtifact = MavenUrlHelper.parseMvnUrl(mvnUri);
-        setModuleName(mavenArtifact.getArtifactId() + "." + mavenArtifact.getType());
+        if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(mavenArtifact.getGroupId())
+                || StringUtils.isEmpty(mavenArtifact.getVersion())) {
+            setModuleName(mavenArtifact.getArtifactId() + "." + mavenArtifact.getType()); //$NON-NLS-1$
+        } else {
+            setModuleName(mavenArtifact.getArtifactId() + "-" + mavenArtifact.getVersion() + "." + mavenArtifact.getType());  //$NON-NLS-1$//$NON-NLS-2$
+        }
 
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleToInstall.java

@@ -43,6 +43,8 @@ public class ModuleToInstall {
 
     private String distribution = "";
 
+    private boolean fromCustomNexus;
+
     public String getName() {
         return this.name;
     }
@@ -176,4 +178,16 @@ public class ModuleToInstall {
         return this.mavenUris;
     }
 
+    /**
+     * Sets the fromCustomNexus.
+     * 
+     * @param fromCustomNexus the fromCustomNexus to set
+     */
+    public void setFromCustomNexus(boolean fromCustomNexus) {
+        this.fromCustomNexus = fromCustomNexus;
+    }
+
+    public boolean isFromCustomNexus() {
+        return this.fromCustomNexus;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -158,26 +158,15 @@ public class MetadataTable implements IMetadataTable, Cloneable {
 
     @Override
     public boolean isDynamicSchema() {
-
-        int sizeListColumns = listColumns.size();
-        boolean hasDynamic = false;
-        for (int i = 0; i < sizeListColumns; i++) {
-            if (listColumns.get(i).getTalendType().equals("id_Dynamic")) {
-                hasDynamic = true;
-                break;
-            }
-        }
-        return hasDynamic;
+        return getDynamicColumn() != null;
 
     }
 
     @Override
     public IMetadataColumn getDynamicColumn() {
-        if (isDynamicSchema()) {
-            for (int i = 0; i < listColumns.size(); i++) {
-                if (listColumns.get(i).getTalendType().equals("id_Dynamic")) {
-                    return listColumns.get(i);
-                }
+        for (IMetadataColumn column : listColumns) {
+            if (column.getTalendType().equals("id_Dynamic")) {
+                return column;
             }
         }
         return null;
@@ -264,6 +253,7 @@ public class MetadataTable implements IMetadataTable, Cloneable {
                                 if (!otherColumn.sameMetacolumnAs(myColumn, options)) {
                                     return false;
                                 }
+                                break;
                             }
                         }
                         if (!exist) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolAvroHelper.java

@@ -0,0 +1,579 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.metadata;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.SchemaBuilder.BaseFieldTypeBuilder;
+import org.apache.avro.SchemaBuilder.FieldAssembler;
+import org.apache.avro.SchemaBuilder.FieldBuilder;
+import org.apache.avro.SchemaBuilder.PropBuilder;
+import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.talend.core.model.metadata.builder.connection.ConnectionFactory;
+import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.types.JavaTypesManager;
+import org.talend.core.utils.TalendQuoteUtils;
+import org.talend.cwm.helper.TaggedValueHelper;
+import org.talend.daikon.avro.util.AvroTypes;
+import org.talend.daikon.avro.util.AvroUtils;
+import org.talend.daikon.talend6.Talend6SchemaConstants;
+import orgomg.cwm.objectmodel.core.Expression;
+import orgomg.cwm.objectmodel.core.TaggedValue;
+
+/**
+ * Utility classes for conversion between a {@link MetadataTable} and an Avro {@link Schema}.
+ */
+public final class MetadataToolAvroHelper {
+
+    /**
+     * @return An Avro schema with enriched properties from the incoming metadata table.
+     */
+    public static org.apache.avro.Schema convertToAvro(MetadataTable in) {
+        RecordBuilder<Schema> builder = SchemaBuilder.builder().record(in.getLabel());
+        copyTableProperties(builder, in);
+
+        FieldAssembler<Schema> fa = builder.fields();
+        int dynamicPosition = -1;
+        org.talend.core.model.metadata.builder.connection.MetadataColumn dynColumn = null;
+        int i = 0;
+        for (org.talend.core.model.metadata.builder.connection.MetadataColumn column : in.getColumns()) {
+            if ("id_Dynamic".equals(column.getTalendType())) { //$NON-NLS-1$
+                dynamicPosition = i;
+                dynColumn = column;
+            } else {
+                fa = convertToAvro(fa, column);
+            }
+            i++;
+        }
+
+        Schema schema = fa.endRecord();
+
+        if (dynColumn != null) {
+            // store all the dynamic column's properties
+            schema = copyDynamicColumnProperties(schema, dynColumn);
+            // store dynamic position
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION,
+                    String.valueOf(dynamicPosition));
+            // tag avro schema with include-all-columns
+            schema = AvroUtils.setIncludeAllFields(schema, true);
+        }
+        return schema;
+    }
+
+    /**
+     * Copy all of the information from the MetadataTable in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyTableProperties(PropBuilder<T> builder, MetadataTable in) {
+
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Table-specific properties.
+        if (in.getName() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_NAME, in.getName());
+        }
+        if (in.getTableType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_TYPE, in.getTableType());
+        }
+
+        return builder;
+    }
+
+    /**
+     * Build a field into a schema using enriched properties from the incoming column.
+     */
+    private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        FieldBuilder<Schema> fb = fa.name(in.getLabel());
+        copyColumnProperties(fb, in);
+        BaseFieldTypeBuilder<Schema> ftb = in.isNullable() ? fb.type().nullable() : fb.type();
+
+        Object defaultValue = null;
+        Expression initialValue = in.getInitialValue();
+        if (initialValue != null) {
+            defaultValue = initialValue.getBody();
+        }
+
+        String tt = in.getTalendType();
+
+        Schema type = null;
+        // Numeric types.
+        if (JavaTypesManager.LONG.getId().equals(tt)) {
+            type = AvroTypes._long();
+            defaultValue = defaultValue == null ? null : Long.parseLong(defaultValue.toString());
+        } else if (JavaTypesManager.INTEGER.getId().equals(tt)) {
+            type = AvroTypes._int();
+            defaultValue = defaultValue == null ? null : Integer.parseInt(defaultValue.toString());
+        } else if (JavaTypesManager.SHORT.getId().equals(tt)) {
+            type = AvroTypes._short();
+            defaultValue = defaultValue == null ? null : Integer.parseInt(defaultValue.toString());
+        } else if (JavaTypesManager.BYTE.getId().equals(tt)) {
+            type = AvroTypes._byte();
+            defaultValue = defaultValue == null ? null : Integer.parseInt(defaultValue.toString());
+        } else if (JavaTypesManager.DOUBLE.getId().equals(tt)) {
+            type = AvroTypes._double();
+            defaultValue = defaultValue == null ? null : Double.parseDouble(defaultValue.toString());
+        } else if (JavaTypesManager.FLOAT.getId().equals(tt)) {
+            type = AvroTypes._float();
+            defaultValue = defaultValue == null ? null : Float.parseFloat(defaultValue.toString());
+        } else if (JavaTypesManager.BIGDECIMAL.getId().equals(tt)) {
+            // decimal(precision, scale) == column length and precision?
+            type = AvroTypes._decimal();
+        }
+
+        // Other primitive types that map directly to Avro.
+        else if (JavaTypesManager.BOOLEAN.getId().equals(tt)) {
+            type = AvroTypes._boolean();
+            defaultValue = defaultValue == null ? null : Boolean.parseBoolean(defaultValue.toString());
+        } else if (JavaTypesManager.BYTE_ARRAY.getId().equals(tt)) {
+            type = AvroTypes._bytes();
+        } else if (JavaTypesManager.DATE.getId().equals(tt)) {
+            type = AvroTypes._date();
+        }
+        // String-ish types.
+        else if (JavaTypesManager.STRING.getId().equals(tt) || JavaTypesManager.FILE.getId().equals(tt)
+                || JavaTypesManager.DIRECTORY.getId().equals(tt) || JavaTypesManager.VALUE_LIST.getId().equals(tt)
+                || JavaTypesManager.CHARACTER.getId().equals(tt) || JavaTypesManager.PASSWORD.getId().equals(tt)) {
+            type = AvroTypes._string();
+        }
+
+        // Types with unknown elements, store as binary
+        if (JavaTypesManager.OBJECT.getId().equals(tt)) {
+            // FIXME it's not right, as it don't store all the information about the object
+        }
+
+        if (JavaTypesManager.LIST.getId().equals(tt)) {
+            // FIXME it's not right, as it don't store all the information about the object
+        }
+        // Can this occur?
+        if (type == null) {
+            throw new UnsupportedOperationException("Unrecognized type " + tt); //$NON-NLS-1$
+        }
+
+        type = in.isNullable() ? AvroUtils.wrapAsNullable(type) : type;
+        return defaultValue == null ? fb.type(type).noDefault() : fb.type(type).withDefault(defaultValue);
+    }
+
+    private static Schema copyDynamicColumnProperties(Schema schema,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        Map<String, String> props = new HashMap<String, String>();
+        if (in.getId() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_NAME, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_ADDITIONAL_PROPERTIES
+                        + additionalTag, tv.getValue());
+            }
+        }
+
+        // Column-specific properties.
+        if (in.isKey()) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
+        }
+        if (in.getType() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
+        }
+        if (in.getTalendType() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
+        }
+        if (in.getPattern() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_PATTERN,
+                    TalendQuoteUtils.removeQuotesIfExist(in.getPattern()));
+        }
+        if (in.getLength() >= 0) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_LENGTH,
+                    String.valueOf((int) in.getLength()));
+        }
+        if (in.getOriginalLength() >= 0) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH,
+                    String.valueOf(in.getOriginalLength()));
+        }
+        if (in.isNullable()) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
+        }
+        if (in.getPrecision() >= 0) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_PRECISION,
+                    String.valueOf(in.getPrecision()));
+        }
+        if (in.getInitialValue() != null && in.getInitialValue().getBody() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
+        }
+        if (in.getName() != null) {
+            // keyword fixes?
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
+        }
+        if (in.getRelatedEntity() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
+        }
+        if (in.getRelationshipType() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE,
+                    in.getRelationshipType());
+        }
+        return schema;
+    }
+
+    /**
+     * Copy all of the information from the IMetadataColumn in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyColumnProperties(PropBuilder<T> builder,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Column-specific properties.
+        if (in.isKey()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
+        }
+        if (in.getType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
+        }
+        if (in.getTalendType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
+        }
+        if (in.getPattern() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN, TalendQuoteUtils.removeQuotesIfExist(in.getPattern()));
+        }
+        if (in.getLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH, String.valueOf((int) in.getLength()));
+        }
+        if (in.getOriginalLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH, String.valueOf(in.getOriginalLength()));
+        }
+        if (in.isNullable()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
+        }
+        if (in.getPrecision() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION, String.valueOf(in.getPrecision()));
+        }
+        if (in.getInitialValue() != null && in.getInitialValue().getBody() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
+        }
+        if (in.getName() != null) {
+            // keyword fixes?
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
+        }
+        if (in.getRelatedEntity() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
+        }
+        if (in.getRelationshipType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE, in.getRelationshipType());
+        }
+        return builder;
+    }
+
+    /**
+     * @param in A schema which must be of type {@link Schema.Type#RECORD}.
+     * @return A MetadataTable containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static MetadataTable convertFromAvro(org.apache.avro.Schema in) {
+        MetadataTable table = ConnectionFactory.eINSTANCE.createMetadataTable();
+
+        // Set the defaults values to the name (the only information guaranteed to be present in the schema.)
+        table.setId(in.getName());
+        table.setLabel(in.getName());
+        table.setName(in.getName());
+        table.setSourceName(in.getName());
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            table.setId(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            table.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            table.setLabel(null);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            table.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : in.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
+                table.getTaggedValue().add(tv);
+            }
+        }
+
+        // Table-specific properties.
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_NAME))) {
+            table.setName(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_TYPE))) {
+            table.setTableType(prop);
+        }
+
+        // Add the columns.
+        List<org.talend.core.model.metadata.builder.connection.MetadataColumn> columns = new ArrayList<>(in.getFields().size());
+        for (Schema.Field f : in.getFields()) {
+            columns.add(convertFromAvro(f));
+        }
+        boolean isDynamic = AvroUtils.isIncludeAllFields(in);
+        if (isDynamic) {
+            org.talend.core.model.metadata.builder.connection.MetadataColumn col = convertFromAvroForDynamic(in);
+            // get dynamic position
+            int dynPosition = Integer.valueOf(in.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION));
+            columns.add(dynPosition, col);
+        }
+        table.getColumns().addAll(columns);
+        return table;
+    }
+
+    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvroForDynamic(Schema schema) {
+        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
+        String prop;
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_ID))) {
+            col.setId(prop);
+        }
+
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_COMMENT))) {
+            col.setComment(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_NAME))) {
+            col.setLabel(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_IS_READ_ONLY))) {
+            col.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : schema.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, schema.getProp(key));
+                col.getTaggedValue().add(tv);
+            }
+        }
+
+        // Column-specific properties.
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
+            col.setKey(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
+            col.setSourceType(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
+            col.setTalendType(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
+            col.setPattern(TalendQuoteUtils.addQuotesIfNotExist(prop));
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setOriginalLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
+            col.setNullable(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
+            Long value = Long.parseLong(prop);
+            col.setPrecision(value > 0 ? value : -1);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
+            col.setDefaultValue(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
+            col.setName(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
+            col.setRelatedEntity(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
+            col.setRelationshipType(prop);
+        }
+
+        col.setTalendType("id_Dynamic"); //$NON-NLS-1$
+        return col;
+    }
+
+    /**
+     * @param in A field from an incoming schema
+     * @return A MetadataColumn containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvro(Schema.Field field) {
+        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
+        Schema in = field.schema();
+
+        // Set the defaults values to the name (the only information guaranteed to be available in every field).
+        col.setId(field.name());
+        col.setLabel(field.name());
+        col.setName(field.name());
+        Schema nonnullable = AvroUtils.unwrapIfNullable(in);
+        if (AvroTypes.isSameType(nonnullable, AvroTypes._boolean())) {
+            col.setTalendType(JavaTypesManager.BOOLEAN.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._byte())) {
+            col.setTalendType(JavaTypesManager.BYTE.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._bytes())) {
+            col.setTalendType(JavaTypesManager.BYTE_ARRAY.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._character())) {
+            col.setTalendType(JavaTypesManager.CHARACTER.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._date())) {
+            col.setTalendType(JavaTypesManager.DATE.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._decimal())) {
+            col.setTalendType(JavaTypesManager.BIGDECIMAL.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._double())) {
+            col.setTalendType(JavaTypesManager.DOUBLE.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._float())) {
+            col.setTalendType(JavaTypesManager.FLOAT.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._int())) {
+            col.setTalendType(JavaTypesManager.INTEGER.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._long())) {
+            col.setTalendType(JavaTypesManager.LONG.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._short())) {
+            col.setTalendType(JavaTypesManager.SHORT.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._string())) {
+            col.setTalendType(JavaTypesManager.STRING.getId());
+        }
+        // FIXME missing List and Object here
+
+        // TODO setSourceType from the field Schema type.
+        col.setNullable(AvroUtils.isNullable(in));
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            col.setId(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            col.setComment(field.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            col.setLabel(null);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            col.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : field.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, field.getProp(key));
+                col.getTaggedValue().add(tv);
+            }
+        }
+
+        // Column-specific properties.
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
+            col.setKey(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
+            col.setSourceType(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
+            col.setTalendType(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
+            col.setPattern(TalendQuoteUtils.addQuotesIfNotExist(prop));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setOriginalLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
+            col.setNullable(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
+            Long value = Long.parseLong(prop);
+            col.setPrecision(value > 0 ? value : -1);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
+            col.setDefaultValue(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
+            col.setName(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
+            col.setRelatedEntity(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
+            col.setRelationshipType(prop);
+        }
+
+        // If the source type wasn't set, there is an issue. Can this occur in the studio.
+        if (col.getTalendType() == null) {
+            throw new UnsupportedOperationException("Unrecognized type " + in); //$NON-NLS-1$
+        }
+
+        return col;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -604,6 +604,7 @@ public final class MetadataToolHelper {
         target.getListColumns().addAll(columnsTAdd);
         target.sortCustomColumns();
         target.setLabel(source.getLabel());
+        target.getAdditionalProperties().putAll(source.getAdditionalProperties());
     }
 
     public static void copyTable(List<IMetadataColumn> sourceColumns, IMetadataTable target, List<IMetadataColumn> targetColumns) {
@@ -1064,7 +1065,8 @@ public final class MetadataToolHelper {
         String uniqueName = null;
         for (int i = 0; i < elementParameters.size(); i++) {
             IElementParameter param = elementParameters.get(i);
-            if (param.getFieldType().equals(EParameterFieldType.SCHEMA_TYPE)
+            if ((param.getFieldType().equals(EParameterFieldType.SCHEMA_TYPE) || param.getFieldType().equals(
+                    EParameterFieldType.SCHEMA_REFERENCE))
                     && param.getContext().equals(metadataTable.getAttachedConnector())) {
                 if (param.getValue() instanceof IMetadataTable) {
                     param.setValueToDefault(elementParameters);
@@ -1310,7 +1312,6 @@ public final class MetadataToolHelper {
             }
         }
         return null;
-
     }
 
     public static IMetadataTable convert(MetadataTable old) {
@@ -1398,5 +1399,4 @@ public final class MetadataToolHelper {
         }
         return result;
     }
-
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/builder/ConvertionHelper.java

@@ -40,9 +40,11 @@ import org.talend.core.model.metadata.builder.connection.DelimitedFileConnection
 import org.talend.core.model.metadata.builder.connection.MDMConnection;
 import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.utils.KeywordsValidator;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.model.bridge.ReponsitoryContextBridge;
 import orgomg.cwm.objectmodel.core.TaggedValue;
 
@@ -337,6 +339,16 @@ public final class ConvertionHelper {
         result.setId(old.getId());
         result.setLabel(old.getLabel());
         result.setTableType(old.getTableType());
+        if (old instanceof SAPBWTable) {
+            String innerType = ((SAPBWTable) old).getInnerIOType();
+            String sourceSysName = ((SAPBWTable) old).getSourceSystemName();
+            if (innerType != null) {
+                result.getAdditionalProperties().put(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE, innerType);
+            }
+            if (sourceSysName != null) {
+                result.getAdditionalProperties().put(SAPBWTableHelper.SAP_DATASOURCE_SOURCESYSNAME, sourceSysName);
+            }
+        }
         String sourceName = old.getName();
         if (sourceName == null) {
             sourceName = old.getLabel();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveConnUtils.java

@@ -1,503 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.model.metadata.connection.hive;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang.ArrayUtils;
-
-/**
- * @author Marvin Wang
- * @version 1.0 jdk1.6
- * @date Aug 9, 2012
- */
-public class HiveConnUtils {
-
-    /**
-     * Returns the distribution names that can be used to show in UI. If no items, return <code>new String[0]</code>.
-     * Added by Marvin Wang on Aug. 9, 2012.
-     * 
-     * @see #getDistributionNameList()
-     * @return
-     */
-    public static String[] getDistributionNames() {
-        List<String> list = getDistributionNameList();
-        if (list == null) {
-            return new String[0];
-        } else {
-            return list.toArray(new String[list.size()]);
-        }
-    }
-
-    /**
-     * Returns the name list of distribution. If no items, return <code>null</code>. These names are from
-     * {@link HiveConnVersionInfo} that defines the level <code>0</code>. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @see #extractVersionInfo(int)
-     * @return
-     */
-    public static List<String> getDistributionNameList() {
-        return getNameListByLevel(0);
-    }
-
-    /**
-     * Returns the children of the given object, refer to the method {@link #getFollowersByKeyAndLevel(String, int)}.
-     * Added by Marvin Wang on Aug 10, 2012.
-     * 
-     * @param obj
-     * @return
-     */
-    protected static List<HiveConnVersionInfo> getFollowersOfObject(HiveConnVersionInfo obj) {
-        String key = obj.getKey();
-        int level = obj.getLevel();
-        List<HiveConnVersionInfo> beans = getObjectsByLevel(level + 1);
-        if (beans != null && beans.size() > 0) {
-            List<HiveConnVersionInfo> listFollowingKey = new ArrayList<HiveConnVersionInfo>();
-            for (HiveConnVersionInfo bean : beans) {
-                extractObjectFollowsKey(key, bean, listFollowingKey);
-            }
-            return listFollowingKey;
-        }
-        return null;
-    }
-
-    /**
-     * If the object from the given array <code>HiveConnVersionInfo[]</code> includes the key, then put the object into
-     * the given list to store. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param key
-     * @param beans
-     * @param listFollowingKey
-     */
-    protected static void extractObjectFollowsKey(String key, HiveConnVersionInfo bean, List<HiveConnVersionInfo> listFollowingKey) {
-        HiveConnVersionInfo[] followsBeans = bean.getFollows();
-        if (followsBeans != null && followsBeans.length > 0) {
-            for (HiveConnVersionInfo followsBean : followsBeans) {
-                if (followsBean.getKey().equals(key)) {
-                    listFollowingKey.add(bean);
-                }
-            }
-        }
-    }
-
-    /**
-     * Returns all names with a specified level. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param level
-     * @return
-     */
-    protected static List<String> getNameListByLevel(int level) {
-        List<HiveConnVersionInfo> beans = getObjectsByLevel(level);
-        if (beans != null && beans.size() > 0) {
-            List<String> nameList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : beans) {
-                nameList.add(bean.getDisplayName());
-            }
-            return nameList;
-        }
-        return null;
-    }
-
-    /**
-     * Returns all {@link HiveConnVersionInfo} objects with the given level. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param level
-     * @return
-     */
-    protected static List<HiveConnVersionInfo> getObjectsByLevel(int level) {
-        HiveConnVersionInfo[] beans = HiveConnVersionInfo.values();
-        if (beans != null && beans.length > 0) {
-            List<HiveConnVersionInfo> list = new ArrayList<HiveConnVersionInfo>();
-            for (HiveConnVersionInfo bean : beans) {
-                int beanLevel = bean.getLevel();
-                if (level == beanLevel) {
-                    list.add(bean);
-                }
-            }
-            // ADD msjian TDQ-6407 2012-11-26:for top not support hive embedded mode
-            // if (PluginChecker.isOnlyTopLoaded() && (level == 0 || level == 2)) {
-            // list.remove(0);
-            // }
-            // TDQ-6407~
-            return list;
-        }
-        return null;
-    }
-
-    /**
-     * Returns the index of distribution that is level 0 by display name. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param displayName
-     * @return
-     */
-    public static int getDistributionIndex(String displayName) {
-        String[] names = getDistributionNames();
-        if (names != null && names.length > 0) {
-            for (int i = 0; i < names.length; i++) {
-                if (displayName != null && displayName.equals(names[i])) {
-                    return i;
-                }
-            }
-        }
-        return -1;
-    }
-
-    /**
-     * Returns the key by display name and level. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param displayName
-     * @param level
-     * @return
-     */
-    protected static String getKeyByDisplayName(String displayName, int level) {
-        List<HiveConnVersionInfo> beans = getObjectsByLevel(level);
-        if (beans != null && beans.size() > 0) {
-            for (HiveConnVersionInfo bean : beans) {
-                if (displayName != null && displayName.equals(bean.getDisplayName())) {
-                    return bean.getKey();
-                }
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Returns all hive versions by distribution index, it means these hive versions follow the distribution. Added by
-     * Marvin Wang on Aug 10, 2012.
-     * 
-     * @param level0Index
-     * @return
-     */
-    protected static List<HiveConnVersionInfo> getHiveVersions(int distributionIndex) {
-        List<HiveConnVersionInfo> level0Objects = getObjectsByLevel(0);
-        HiveConnVersionInfo objOfLevel0ByIndex = level0Objects.get(distributionIndex);
-        return getFollowersOfObject(objOfLevel0ByIndex);
-    }
-
-    protected static List<String> getHiveVersionNameList(int distributionIndex) {
-        List<HiveConnVersionInfo> hiveVersionObjs = getHiveVersions(distributionIndex);
-        if (hiveVersionObjs != null && hiveVersionObjs.size() > 0) {
-            List<String> hiveVersionNameList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : hiveVersionObjs) {
-                hiveVersionNameList.add(bean.getDisplayName());
-            }
-            return hiveVersionNameList;
-        }
-        return null;
-    }
-
-    public static String[] getHiveVersionNames(int distributionIndex) {
-        List<String> hiveVersionNameList = getHiveVersionNameList(distributionIndex);
-        if (hiveVersionNameList != null && hiveVersionNameList.size() > 0) {
-            return hiveVersionNameList.toArray(new String[hiveVersionNameList.size()]);
-        } else {
-            return new String[0];
-        }
-    }
-
-    protected static List<HiveConnVersionInfo> getHiveModes(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> supportedModes = new ArrayList<HiveConnVersionInfo>();
-        HiveConnVersionInfo hiveVersionObj = getHiveVersionObj(distributionIndex, versionIndex);
-        if (hiveVersionObj != null) {
-            boolean supportStandalone = isSupportStandalone(hiveVersionObj, hiveServerIndex);
-            List<HiveConnVersionInfo> modes = getFollowersOfObject(hiveVersionObj);
-            if (modes != null && modes.size() > 0) {
-                for (HiveConnVersionInfo mode : modes) {
-                    if (HiveConnVersionInfo.MODE_EMBEDDED.equals(mode) || HiveConnVersionInfo.MODE_STANDALONE.equals(mode)
-                            && supportStandalone) {
-                        supportedModes.add(mode);
-                    }
-                }
-            }
-        }
-
-        return supportedModes;
-    }
-
-    private static boolean isSupportStandalone(HiveConnVersionInfo hiveVersionObj, int hiveServerIndex) {
-        boolean isHiveServer1 = "HIVE".equals(HiveServerVersionUtils.extractKey(hiveServerIndex)); //$NON-NLS-1$
-        return !(isHiveServer1 && (HiveConnVersionInfo.HDP_1_2.equals(hiveVersionObj)
-                || HiveConnVersionInfo.HDP_1_3.equals(hiveVersionObj) || HiveConnVersionInfo.HDP_2_0.equals(hiveVersionObj)));
-    }
-
-    protected static List<String> getHiveModeNameList(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> hiveModeObjs = getHiveModes(distributionIndex, versionIndex, hiveServerIndex);
-        // ADD msjian TDQ-6407 2012-11-26: for top, not support hive embeded mode,hide this menu
-        // if (PluginChecker.isOnlyTopLoaded() && hiveModeObjs.size() > 1) {
-        // hiveModeObjs.remove(0);
-        // }
-        // TDQ-6407~
-        if (hiveModeObjs != null && hiveModeObjs.size() > 0) {
-            List<String> hiveModeNameList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : hiveModeObjs) {
-                hiveModeNameList.add(bean.getDisplayName());
-            }
-            return hiveModeNameList;
-        }
-        return null;
-    }
-
-    public static String[] getHiveModeNames(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<String> hiveModeNameList = getHiveModeNameList(distributionIndex, versionIndex, hiveServerIndex);
-        if (hiveModeNameList != null && hiveModeNameList.size() > 0) {
-            return hiveModeNameList.toArray(new String[hiveModeNameList.size()]);
-        } else {
-            return new String[0];
-        }
-    }
-
-    protected static List<String> getHiveModeKeyList(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> hiveModeObjs = getHiveModes(distributionIndex, versionIndex, hiveServerIndex);
-        // ADD msjian TDQ-6407 2012-11-26: for top, not support hive embeded mode,hide this menu
-        // if (PluginChecker.isOnlyTopLoaded() && hiveModeObjs.size() > 1) {
-        // hiveModeObjs.remove(0);
-        // }
-        // TDQ-6407~
-        if (hiveModeObjs != null && hiveModeObjs.size() > 0) {
-            List<String> hiveModeKeyList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : hiveModeObjs) {
-                hiveModeKeyList.add(bean.getKey());
-            }
-            return hiveModeKeyList;
-        }
-        return null;
-    }
-
-    public static String[] getHiveModeKeys(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<String> hiveModeKeyList = getHiveModeKeyList(distributionIndex, versionIndex, hiveServerIndex);
-        if (hiveModeKeyList != null && hiveModeKeyList.size() > 0) {
-            return hiveModeKeyList.toArray(new String[hiveModeKeyList.size()]);
-        } else {
-            return new String[0];
-        }
-    }
-
-    public static int getIndexOfDistribution(String distributionKey) {
-        int level = 0;
-        List<HiveConnVersionInfo> distributions = getObjectsByLevel(level);
-        if (distributions != null && distributions.size() > 0) {
-            for (int i = 0; i < distributions.size(); i++) {
-                if (distributions.get(i).getKey().equals(distributionKey)) {
-                    return i;
-                }
-            }
-        }
-        return -1;
-    }
-
-    protected static int getIndexOfDistribution(HiveConnVersionInfo distribution) {
-        String key = distribution.getKey();
-        return getIndexOfDistribution(key);
-    }
-
-    public static int getIndexOfHiveVersion(String distributionKey, String hiveVersionKey) {
-        int distributionIndex = getIndexOfDistribution(distributionKey);
-        List<HiveConnVersionInfo> hiveVersions = getHiveVersions(distributionIndex);
-        if (hiveVersions != null && hiveVersions.size() > 0) {
-            for (int i = 0; i < hiveVersions.size(); i++) {
-                if (hiveVersions.get(i).getKey().equals(hiveVersionKey)) {
-                    return i;
-                }
-            }
-        }
-        return 0;
-    }
-
-    protected static int getIndexOfHiveVersion(HiveConnVersionInfo distribution, HiveConnVersionInfo hiveVersion) {
-        String distributionKey = distribution.getKey();
-        String hiveVersionKey = hiveVersion.getKey();
-        return getIndexOfHiveVersion(distributionKey, hiveVersionKey);
-    }
-
-    public static int getIndexOfHiveMode(String distributionKey, String hiveVersionKey, String hiveModeKey, String hiveServerKey) {
-        int distributionIndex = getIndexOfDistribution(distributionKey);
-        int hiveVersionIndex = getIndexOfHiveVersion(distributionKey, hiveVersionKey);
-        int hiveServerIndex = getIndexOfHiveServer(hiveServerKey);
-        List<HiveConnVersionInfo> hiveModes = getHiveModes(distributionIndex, hiveVersionIndex, hiveServerIndex);
-        if (hiveModes != null && hiveModes.size() > 0) {
-            for (int i = 0; i < hiveModes.size(); i++) {
-                if (hiveModes.get(i).getKey().equals(hiveModeKey)) {
-                    return i;
-                }
-            }
-        }
-        return 0;
-    }
-
-    protected static int getIndexOfHiveMode(HiveConnVersionInfo distribution, HiveConnVersionInfo hiveVersion,
-            HiveConnVersionInfo hiveMode, String hiveServerKey) {
-        String distributionKey = distribution.getKey();
-        String hiveModeKey = hiveMode.getKey();
-        String hiveVersionkey = hiveVersion.getKey();
-        return getIndexOfHiveMode(distributionKey, hiveVersionkey, hiveModeKey, hiveServerKey);
-    }
-
-    public static int getIndexOfHiveServer(String hiveServerKey) {
-        List<String> hiveServers = HiveServerVersionUtils.extractListKeys();
-        if (hiveServers != null && hiveServers.size() > 0) {
-            for (int i = 0; i < hiveServers.size(); i++) {
-                if (hiveServers.get(i).equals(hiveServerKey)) {
-                    return i;
-                }
-            }
-        }
-
-        return 0;
-    }
-
-    public static int getIndexOfHiveServerByDisplayName(String hiveServerDisplayName) {
-        List<String> hiveServers = HiveServerVersionUtils.extractListDisplayNames();
-        if (hiveServers != null && hiveServers.size() > 0) {
-            for (int i = 0; i < hiveServers.size(); i++) {
-                if (hiveServers.get(i).equals(hiveServerDisplayName)) {
-                    return i;
-                }
-            }
-        }
-
-        return 0;
-    }
-
-    public static boolean isEmbeddedMode(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> hiveModes = getHiveModes(distributionIndex, hiveVersionIndex, hiveServerIndex);
-        if (hiveModes != null && hiveModes.size() > 0) {
-            HiveConnVersionInfo hiveMode = hiveModes.get(hiveModeIndex);
-            if (HiveConnVersionInfo.MODE_EMBEDDED.getKey().equals(hiveMode.getKey())) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Returns the distribution <code>HiveConnVersionInfo</code> object by the given <code>index</code> from level 0.
-     * Added by Marvin Wang on Aug 13, 2012.
-     * 
-     * @param index
-     * @return
-     */
-    public static HiveConnVersionInfo getDistributionObj(int index) {
-        List<HiveConnVersionInfo> distributions = getObjectsByLevel(0);
-        if (distributions != null && distributions.size() > 0 && index != -1) {
-            return distributions.get(index);
-        }
-        return null;
-    }
-
-    /**
-     * Returns the hive version object by the given index of distribution and hive version. Added by Marvin Wang on Aug
-     * 13, 2012.
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @return
-     */
-    public static HiveConnVersionInfo getHiveVersionObj(int distributionIndex, int hiveVersionIndex) {
-        List<HiveConnVersionInfo> objs = getHiveVersions(distributionIndex);
-        if (objs != null && objs.size() > 0) {
-            return objs.get(hiveVersionIndex);
-        }
-        return null;
-    }
-
-    /**
-     * Returns the hive mode object by the given index of distribution, version and mode. Added by Marvin Wang on Aug
-     * 13, 2012.
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @param hiveModeIndex
-     * @return
-     */
-    public static HiveConnVersionInfo getHiveModeObj(int distributionIndex, int hiveVersionIndex, int hiveModeIndex,
-            int hiveServerIndex) {
-        List<HiveConnVersionInfo> objs = getHiveModes(distributionIndex, hiveVersionIndex, hiveServerIndex);
-        if (objs != null && objs.size() > 0) {
-            return objs.get(hiveModeIndex);
-        }
-        return null;
-    }
-
-    public static String getHiveModeObjKey(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, int hiveServerIndex) {
-        HiveConnVersionInfo hiveModeObj = getHiveModeObj(distributionIndex, hiveVersionIndex, hiveModeIndex, hiveServerIndex);
-        if (hiveModeObj != null) {
-            return hiveModeObj.getKey();
-        }
-        return null;
-    }
-
-    /**
-     * Checkes if the selected distro version supports hive server2. Added by Marvin Wang on Mar 25, 2013.
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @return
-     */
-    public static boolean isSupportHiveServer2(int distributionIndex, int hiveVersionIndex) {
-        HiveConnVersionInfo distroVersion = getHiveVersionObj(distributionIndex, hiveVersionIndex);
-        return distroVersion.isSupportHive2();
-    }
-
-    /**
-     * Checks if the current distro is {@link HiveConnVersionInfo#CUSTOM}. If yes, then return <code>true</code> Added
-     * by Marvin Wang on Mar 26, 2013.
-     * 
-     * @param distributionIndex
-     * @return
-     */
-    public static boolean isCustomDistro(int distributionIndex) {
-        HiveConnVersionInfo obj = getDistributionObj(distributionIndex);
-        if (HiveConnVersionInfo.DISTRO_CUSTOM == obj) {
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    public static boolean isCustomDistro(String distroKey) {
-        int distroIndex = getIndexOfDistribution(distroKey);
-        return isCustomDistro(distroIndex);
-    }
-
-    /**
-     * This is a temporary method to estimate whether or not one hive version support security. We need to refactor
-     * {@link HiveConnVersionInfo}(A really bad class) to support this function later...
-     * <P>
-     * DOC ycbai Comment method "isSupportSecurity".
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @param hiveModeIndex
-     * @param isHive2
-     * @return
-     */
-    public static boolean isSupportSecurity(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, boolean isHive2,
-            int hiveServerIndex) {
-        HiveConnVersionInfo hiveVersionObj = getHiveVersionObj(distributionIndex, hiveVersionIndex);
-        if (hiveVersionObj.isSupportSecurity()
-                && (isHive2 || isEmbeddedMode(distributionIndex, hiveVersionIndex, hiveModeIndex, hiveServerIndex))) {
-            return true;
-        }
-
-        return false;
-    }
-
-    public static boolean isSupportTez(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, int hiveServerIndex) {
-        HiveConnVersionInfo hiveVersionObj = getHiveVersionObj(distributionIndex, hiveVersionIndex);
-        boolean versionSupportTez = ArrayUtils.contains(HiveConnVersionInfo.getHiveVersionsSupportingTez(), hiveVersionObj);
-        boolean isEmbeddedMode = isEmbeddedMode(distributionIndex, hiveVersionIndex, hiveModeIndex, hiveServerIndex);
-        return versionSupportTez && isEmbeddedMode;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveConnVersionInfo.java

@@ -1,423 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.model.metadata.connection.hive;
-
-import org.talend.core.hadoop.version.EHadoopDistributions;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
-
-/**
- * This class provides the info for Hive. There are 3 levels as follows:
- *
- * <pre>
- * <li> 0 -- stands for "Distribution"
- * <li> 1 -- stands for "Distro version"
- * <li> 2 -- stands for "Hive mode"
- * </pre>
- *
- * The <code>key</code> of this enum is matched to the config file of component.
- *
- * @author Marvin Wang
- * @version 1.0 jdk1.6
- * @date Aug 9, 2012
- */
-public enum HiveConnVersionInfo {
-
-    HORTONWORKS(0, EHadoopDistributions.HORTONWORKS.getName(), EHadoopDistributions.HORTONWORKS.getDisplayName(), true),
-
-    APACHE(0, EHadoopDistributions.APACHE.getName(), EHadoopDistributions.APACHE.getDisplayName(), false),
-
-    CLOUDERA(0, EHadoopDistributions.CLOUDERA.getName(), EHadoopDistributions.CLOUDERA.getDisplayName(), false),
-
-    MAPR(0, EHadoopDistributions.MAPR.getName(), EHadoopDistributions.MAPR.getDisplayName(), false),
-
-    AMAZON_EMR(0, EHadoopDistributions.AMAZON_EMR.getName(), EHadoopDistributions.AMAZON_EMR.getDisplayName(), false),
-
-    PIVOTAL_HD(0, EHadoopDistributions.PIVOTAL_HD.getName(), EHadoopDistributions.PIVOTAL_HD.getDisplayName(), false),
-
-    DISTRO_CUSTOM(0, EHadoopDistributions.CUSTOM.getName(), EHadoopDistributions.CUSTOM.getDisplayName(), false),
-
-    HDP_2_3(
-            1,
-            EHadoopVersion4Drivers.HDP_2_3.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_3.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_2_2(
-            1,
-            EHadoopVersion4Drivers.HDP_2_2.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_2.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_2_1(
-            1,
-            EHadoopVersion4Drivers.HDP_2_1.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_1.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_2_0(
-            1,
-            EHadoopVersion4Drivers.HDP_2_0.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_0.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_1_3(
-            1,
-            EHadoopVersion4Drivers.HDP_1_3.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_1_3.getVersionDisplay(),
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_1_2(
-            1,
-            EHadoopVersion4Drivers.HDP_1_2.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_1_2.getVersionDisplay(),
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    APACHE_1_0_0(1, EHadoopVersion4Drivers.APACHE_1_0_0.getVersionValue(), EHadoopVersion4Drivers.APACHE_1_0_0
-            .getVersionDisplay(), false, HiveConnVersionInfo.APACHE),
-
-    Cloudera_CDH5_5(1, EHadoopVersion4Drivers.CLOUDERA_CDH5_5.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5_5
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5_4(1, EHadoopVersion4Drivers.CLOUDERA_CDH5_4.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5_4
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5_1(1, EHadoopVersion4Drivers.CLOUDERA_CDH5_1.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5_1
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5_1_MR1(
-                        1,
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionValue(),
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionDisplay(),
-                        true,
-                        false,
-                        true,
-                        true,
-                        HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5(1, EHadoopVersion4Drivers.CLOUDERA_CDH5.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH4_YARN(1, EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN
-            .getVersionDisplay(), true, false, true, false, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH4(1, EHadoopVersion4Drivers.CLOUDERA_CDH4.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH4
-            .getVersionDisplay(), true, true, HiveConnVersionInfo.CLOUDERA),
-
-    MAPR5_0_0(
-              1,
-              EHadoopVersion4Drivers.MAPR500.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR500.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR4_1_0(
-              1,
-              EHadoopVersion4Drivers.MAPR410.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR410.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR4_0_1(
-              1,
-              EHadoopVersion4Drivers.MAPR401.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR401.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR3_1_0(
-              1,
-              EHadoopVersion4Drivers.MAPR310.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR310.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR3_0_1(
-              1,
-              EHadoopVersion4Drivers.MAPR301.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR301.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR2_1_3(
-              1,
-              EHadoopVersion4Drivers.MAPR213.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR213.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR2_1_2(
-              1,
-              EHadoopVersion4Drivers.MAPR212.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR212.getVersionDisplay(),
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR2(
-          1,
-          EHadoopVersion4Drivers.MAPR2.getVersionValue(),
-          EHadoopVersion4Drivers.MAPR2.getVersionDisplay(),
-          false,
-          HiveConnVersionInfo.MAPR),
-
-    EMR_4_0_0(1, EHadoopVersion4Drivers.EMR_4_0_0.getVersionValue(), "EMR 4.0.0 (Hive 1.0.0)", //$NON-NLS-1$
-              true,
-              false,
-              HiveConnVersionInfo.AMAZON_EMR),
-
-    APACHE_2_4_0_EMR_0_13_1(1, "APACHE_2_4_0_EMR_0_13_1", "Apache 2.4.0 (Hive 0.13.1)", false, HiveConnVersionInfo.AMAZON_EMR), //$NON-NLS-1$ //$NON-NLS-2$
-
-    APACHE_2_4_0_EMR(
-                     1,
-                     EHadoopVersion4Drivers.APACHE_2_4_0_EMR.getVersionValue(),
-                     "Apache 2.4.0 (Hive 0.11.0)", false, HiveConnVersionInfo.AMAZON_EMR), //$NON-NLS-1$
-
-    APACHE_1_0_3_EMR(
-                     1,
-                     EHadoopVersion4Drivers.APACHE_1_0_3_EMR.getVersionValue(),
-                     "Apache 1.0.3 (Hive 0.8.1)", false, HiveConnVersionInfo.AMAZON_EMR), //$NON-NLS-1$
-
-    PIVOTAL_HD_2_0(1, EHadoopVersion4Drivers.PIVOTAL_HD_2_0.getVersionValue(), EHadoopVersion4Drivers.PIVOTAL_HD_2_0
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.PIVOTAL_HD),
-
-    PIVOTAL_HD_1_0_1(1, EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1.getVersionValue(), EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1
-            .getVersionDisplay(), false, false, true, false, HiveConnVersionInfo.PIVOTAL_HD),
-
-    DISTRO_VERSION_CUSTOM(1, "DISTRO_VERSION_CUSTOM", "Customized Version", true, true, HiveConnVersionInfo.DISTRO_CUSTOM), //$NON-NLS-1$//$NON-NLS-2$
-
-    MODE_EMBEDDED(2, "EMBEDDED",//$NON-NLS-1$
-                  "Embedded",//$NON-NLS-1$
-                  false,
-                  HiveConnVersionInfo.HDP_1_2,
-                  HiveConnVersionInfo.HDP_1_3,
-                  HiveConnVersionInfo.HDP_2_0,
-                  HiveConnVersionInfo.HDP_2_1,
-                  HiveConnVersionInfo.HDP_2_2,
-                  // HiveConnVersionInfo.HDP_2_3,
-                  HiveConnVersionInfo.APACHE_1_0_0,
-                  HiveConnVersionInfo.Cloudera_CDH4,
-                  HiveConnVersionInfo.Cloudera_CDH4_YARN,
-                  HiveConnVersionInfo.Cloudera_CDH5,
-                  HiveConnVersionInfo.Cloudera_CDH5_1,
-                  HiveConnVersionInfo.Cloudera_CDH5_1_MR1,
-                  HiveConnVersionInfo.Cloudera_CDH5_4,
-                  HiveConnVersionInfo.MAPR2,
-                  HiveConnVersionInfo.MAPR2_1_2,
-                  HiveConnVersionInfo.MAPR2_1_3,
-                  HiveConnVersionInfo.MAPR3_0_1,
-                  HiveConnVersionInfo.MAPR3_1_0,
-                  HiveConnVersionInfo.MAPR4_0_1,
-                  HiveConnVersionInfo.MAPR4_1_0,
-                  // HiveConnVersionInfo.MAPR5_0_0,
-                  HiveConnVersionInfo.APACHE_1_0_3_EMR,
-                  HiveConnVersionInfo.APACHE_2_4_0_EMR,
-                  HiveConnVersionInfo.PIVOTAL_HD_1_0_1,
-                  HiveConnVersionInfo.PIVOTAL_HD_2_0,
-                  HiveConnVersionInfo.DISTRO_VERSION_CUSTOM),
-
-    MODE_STANDALONE(2, "STANDALONE",//$NON-NLS-1$
-                    "Standalone",//$NON-NLS-1$
-                    false,
-                    HiveConnVersionInfo.HDP_1_2,
-                    HiveConnVersionInfo.HDP_1_3,
-                    HiveConnVersionInfo.HDP_2_0,
-                    HiveConnVersionInfo.HDP_2_1,
-                    HiveConnVersionInfo.HDP_2_2,
-                    HiveConnVersionInfo.HDP_2_3,
-                    HiveConnVersionInfo.APACHE_1_0_0,
-                    HiveConnVersionInfo.Cloudera_CDH4,
-                    HiveConnVersionInfo.Cloudera_CDH4_YARN,
-                    HiveConnVersionInfo.Cloudera_CDH5,
-                    HiveConnVersionInfo.Cloudera_CDH5_1,
-                    HiveConnVersionInfo.Cloudera_CDH5_1_MR1,
-                    HiveConnVersionInfo.Cloudera_CDH5_4,
-                    HiveConnVersionInfo.Cloudera_CDH5_5,
-                    HiveConnVersionInfo.MAPR2,
-                    HiveConnVersionInfo.MAPR2_1_2,
-                    HiveConnVersionInfo.MAPR2_1_3,
-                    HiveConnVersionInfo.MAPR3_0_1,
-                    HiveConnVersionInfo.MAPR3_1_0,
-                    HiveConnVersionInfo.MAPR4_0_1,
-                    HiveConnVersionInfo.MAPR4_1_0,
-                    HiveConnVersionInfo.MAPR5_0_0,
-                    HiveConnVersionInfo.APACHE_1_0_3_EMR,
-                    HiveConnVersionInfo.APACHE_2_4_0_EMR,
-                    HiveConnVersionInfo.APACHE_2_4_0_EMR_0_13_1,
-                    HiveConnVersionInfo.EMR_4_0_0,
-                    HiveConnVersionInfo.PIVOTAL_HD_1_0_1,
-                    HiveConnVersionInfo.PIVOTAL_HD_2_0,
-                    HiveConnVersionInfo.DISTRO_VERSION_CUSTOM);
-
-    private int level;
-
-    private String key;// The key is mapped to item name of component in t*_java.xml file.
-
-    private String displayName;// That is mapped to the property of component in t*_messages.properties file.
-
-    private HiveConnVersionInfo[] follows;// That stores all are followed by the current object.
-
-    private boolean isSupportHive2; // Till now only MapR2, CDH4, and HDP1.2 support hive server2.
-
-    private boolean isSupportMR1;
-
-    private boolean isSupportYARN;
-
-    private boolean supportSecurity;
-
-    private static HiveConnVersionInfo[] hiveVersions = new HiveConnVersionInfo[] { HiveConnVersionInfo.Cloudera_CDH5,
-            HiveConnVersionInfo.Cloudera_CDH5_1, HiveConnVersionInfo.Cloudera_CDH5_1_MR1, HiveConnVersionInfo.Cloudera_CDH5_4,
-            HiveConnVersionInfo.Cloudera_CDH5_5, HiveConnVersionInfo.HDP_2_3, HiveConnVersionInfo.HDP_2_2,
-            HiveConnVersionInfo.HDP_2_1, HiveConnVersionInfo.HDP_2_0, HiveConnVersionInfo.PIVOTAL_HD_2_0,
-            HiveConnVersionInfo.APACHE_2_4_0_EMR, HiveConnVersionInfo.APACHE_2_4_0_EMR_0_13_1, HiveConnVersionInfo.EMR_4_0_0 };
-
-    private static HiveConnVersionInfo[] hiveVersionsSupportingTez = new HiveConnVersionInfo[] { HiveConnVersionInfo.HDP_2_2,
-            HiveConnVersionInfo.HDP_2_1, HiveConnVersionInfo.MAPR4_0_1, HiveConnVersionInfo.MAPR4_1_0,
-            HiveConnVersionInfo.MAPR5_0_0, HiveConnVersionInfo.DISTRO_VERSION_CUSTOM };
-
-    private HiveConnVersionInfo(int level, String key, String displayName, boolean supportSecurity,
-            HiveConnVersionInfo... follows) {
-        this(level, key, displayName, false, supportSecurity, follows);
-    }
-
-    private HiveConnVersionInfo(int level, String key, String displayName, boolean isSupportHive2, boolean supportSecurity,
-            HiveConnVersionInfo... follows) {
-        this(level, key, displayName, isSupportHive2, true, false, supportSecurity, follows);
-    }
-
-    private HiveConnVersionInfo(int level, String key, String displayName, boolean isSupportHive2, boolean isSupportMR1,
-            boolean isSupportYARN, boolean supportSecurity, HiveConnVersionInfo... follows) {
-        this.level = level;
-        this.key = key;
-        this.displayName = displayName;
-        this.isSupportHive2 = isSupportHive2;
-        this.isSupportMR1 = isSupportMR1;
-        this.isSupportYARN = isSupportYARN;
-        this.follows = follows;
-        this.supportSecurity = supportSecurity;
-    }
-
-    public int getLevel() {
-        return this.level;
-    }
-
-    public String getKey() {
-        return this.key;
-    }
-
-    public String getDisplayName() {
-        return this.displayName;
-    }
-
-    public HiveConnVersionInfo[] getFollows() {
-        return this.follows;
-    }
-
-    /**
-     * Getter for isSupportHive2.
-     *
-     * @return the isSupportHive2
-     */
-    public boolean isSupportHive2() {
-        return this.isSupportHive2;
-    }
-
-    public boolean isSupportHiveServerVersion(HiveServerVersionInfo hiveServerVersion) {
-        boolean isSupport = false;
-
-        if (hiveServerVersion == null) {
-            return isSupport;
-        }
-
-        switch (hiveServerVersion) {
-        case HIVE_SERVER_1:
-            switch (this) {
-            // make them same with components
-            case HDP_2_3:
-            case EMR_4_0_0:
-            case Cloudera_CDH5_4:
-            case Cloudera_CDH5_5:
-            case MAPR4_1_0:
-            case MAPR5_0_0:
-                isSupport = false;
-                break;
-            default:
-                isSupport = true;
-            }
-            break;
-        case HIVE_SERVER_2:
-            isSupport = isSupportHive2();
-            break;
-        default:
-            isSupport = false;
-        }
-
-        return isSupport;
-    }
-
-    public boolean isSupportMR1() {
-        return this.isSupportMR1;
-    }
-
-    public boolean isSupportYARN() {
-        return this.isSupportYARN;
-    }
-
-    public boolean isSupportSecurity() {
-        return this.supportSecurity;
-    }
-
-    public static HiveConnVersionInfo[] getHiveVersionsNotSupportOnWindows() {
-        return hiveVersions;
-    }
-
-    public static HiveConnVersionInfo[] getHiveVersionsSupportingTez() {
-        return hiveVersionsSupportingTez;
-    }
-
-    public static HiveConnVersionInfo getVersionByKey(String key) {
-        HiveConnVersionInfo[] values = values();
-        for (HiveConnVersionInfo version : values) {
-            if (version.getLevel() == 1 && version.getKey().equals(key)) {
-                return version;
-            }
-        }
-
-        return null;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveModeInfo.java

@@ -0,0 +1,54 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.metadata.connection.hive;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public enum HiveModeInfo {
+    STANDALONE("Standalone"), //$NON-NLS-1$
+    EMBEDDED("Embedded"), //$NON-NLS-1$
+    ;
+
+    private String display;
+
+    private HiveModeInfo(String diaplyName) {
+        this.display = diaplyName;
+    }
+
+    public String getName() {
+        return name();
+    }
+
+    public String getDisplayName() {
+        return display;
+    }
+
+    public static HiveModeInfo getByDisplay(String display) {
+        for (HiveModeInfo m : HiveModeInfo.values()) {
+            if (m.getDisplayName().equals(display)) {
+                return m;
+            }
+        }
+        return null;
+    }
+
+    public static HiveModeInfo get(String name) {
+        for (HiveModeInfo m : HiveModeInfo.values()) {
+            if (m.getName().equals(name)) {
+                return m;
+            }
+        }
+        return null;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveServerVersionInfo.java

@@ -42,15 +42,6 @@ public enum HiveServerVersionInfo {
         return this.key;
     }
 
-    /**
-     * Sets the key.
-     * 
-     * @param key the key to set
-     */
-    public void setKey(String key) {
-        this.key = key;
-    }
-
     /**
      * Getter for name.
      * 
@@ -60,15 +51,6 @@ public enum HiveServerVersionInfo {
         return this.name;
     }
 
-    /**
-     * Sets the name.
-     * 
-     * @param name the name to set
-     */
-    public void setName(String name) {
-        this.name = name;
-    }
-
     /**
      * Getter for displayName.
      * 
@@ -78,13 +60,21 @@ public enum HiveServerVersionInfo {
         return this.displayName;
     }
 
-    /**
-     * Sets the displayName.
-     * 
-     * @param displayName the displayName to set
-     */
-    public void setDisplayName(String displayName) {
-        this.displayName = displayName;
+    public static HiveServerVersionInfo getByDisplay(String display) {
+        for (HiveServerVersionInfo s : HiveServerVersionInfo.values()) {
+            if (s.getDisplayName().equals(display)) {
+                return s;
+            }
+        }
+        return null;
     }
 
+    public static HiveServerVersionInfo getByKey(String key) {
+        for (HiveServerVersionInfo s : HiveServerVersionInfo.values()) {
+            if (s.getKey().equals(key)) {
+                return s;
+            }
+        }
+        return null;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveServerVersionUtils.java

@@ -1,132 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.model.metadata.connection.hive;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Created by Marvin Wang on Mar 25, 2013.
- */
-public class HiveServerVersionUtils {
-
-    /**
-     * Extracts all display names for hive server version, all versions are from {@link HiveServerVersionInfo}. Added by
-     * Marvin Wang on Mar 25, 2013.
-     * 
-     * @return
-     */
-    public static List<String> extractListDisplayNames() {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        List<String> displayNames = new ArrayList<String>();
-        for (HiveServerVersionInfo info : infos) {
-            displayNames.add(info.getDisplayName());
-        }
-
-        return displayNames;
-    }
-
-    public static List<String> extractAvailableListDisplayNames(HiveConnVersionInfo hadoopVersion) {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        List<String> keys = new ArrayList<String>();
-        for (HiveServerVersionInfo info : infos) {
-            if (hadoopVersion.isSupportHiveServerVersion(info)) {
-                keys.add(info.getDisplayName());
-            }
-        }
-
-        return keys;
-    }
-
-    /**
-     * Extracts the array of display names, refer to this method {@link #extractListDisplayNames()}. Added by Marvin
-     * Wang on Mar 25, 2013.
-     * 
-     * @return
-     */
-    public static String[] extractArrayDisplayNames() {
-        List<String> list = extractListDisplayNames();
-        return list.toArray(new String[list.size()]);
-    }
-
-    public static String[] extractAvailableArrayDisplayNames(HiveConnVersionInfo hadoopVersion) {
-        List<String> list = extractAvailableListDisplayNames(hadoopVersion);
-        return list.toArray(new String[list.size()]);
-    }
-
-    public static List<String> extractListKeys() {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        List<String> keys = new ArrayList<String>();
-        for (HiveServerVersionInfo info : infos) {
-            keys.add(info.getKey());
-        }
-
-        return keys;
-    }
-
-    /**
-     * Gets the index of hive server by display name, if no index found, then return 0. Added by Marvin Wang on Mar 25,
-     * 2013.
-     * 
-     * @param hiveServerDisplayName
-     * @return 0 if there is no index found.
-     */
-    public static int getIndexofHiveServer(String hiveServerDisplayName) {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        for (int i = 0; i < infos.length; i++) {
-            if (infos[i].getDisplayName().equalsIgnoreCase(hiveServerDisplayName)) {
-                return i;
-            }
-        }
-        return 0;
-    }
-
-    public static int getIndexofHiveServerByKey(String hiveKey) {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        for (int i = 0; i < infos.length; i++) {
-            if (infos[i].getKey().equalsIgnoreCase(hiveKey)) {
-                return i;
-            }
-        }
-        return 0;
-    }
-
-    /**
-     * Extracts the display name by the index. Added by Marvin Wang on Mar 25, 2013.
-     * 
-     * @param index
-     * @return
-     */
-    public static String extractDisplayName(int index) {
-        List<String> displayNames = extractListDisplayNames();
-        if (index < 0) {
-            return null;
-        }
-        return displayNames.get(index);
-    }
-
-    /**
-     * Extracts the key which is mapped to t*component.xml. Added by Marvin Wang on Mar 26, 2013.
-     * 
-     * @param index
-     * @return
-     */
-    public static String extractKey(int index) {
-        List<String> keys = extractListKeys();
-        if (index < 0) {
-            return null;
-        }
-        return keys.get(index);
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -21,11 +21,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.emf.common.util.EList;
 import org.talend.commons.ui.runtime.exception.ExceptionHandler;
 import org.talend.commons.ui.utils.PathUtils;
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.database.EDatabaseTypeName;
 import org.talend.core.database.conn.ConnParameterKeys;
@@ -910,18 +910,10 @@ public class RepositoryToComponentProperty {
             }
         }
         if (value.equals("SERVER_NAME")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
         if (value.equals("PORT")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getPort())) {
-                return connection.getPort();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getPort());
-            }
+            return getAppropriateValue(connection, connection.getPort());
         }
         if (value.equals("SID") || value.equals("DATABASE_ALIAS")) { //$NON-NLS-1$ //$NON-NLS-2$
             if (("").equals(connection.getSID()) || connection.getSID() == null) { //$NON-NLS-1$
@@ -939,54 +931,25 @@ public class RepositoryToComponentProperty {
             }
         }
         if (value.equals("DATASOURCE")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getDatasourceName())) {
-                return connection.getDatasourceName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDatasourceName());
-            }
+            return getAppropriateValue(connection, connection.getDatasourceName());
         }
         if (value.equals("USERNAME")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getUsername())) {
-                return connection.getUsername();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getUsername());
-            }
+            return getAppropriateValue(connection, connection.getUsername());
         }
         if (value.equals("PASSWORD")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getPassword())) {
-                return connection.getPassword();
-            } else {
-                String pwd = TalendQuoteUtils.checkAndAddBackslashes(connection.getRawPassword());
-                return TalendQuoteUtils.addQuotes(pwd);
-            }
+            return getAppropriateValue(connection, connection.getValue(connection.getPassword(), false));
         }
         if (value.equals("NULL_CHAR")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getNullChar())) {
-                return connection.getNullChar();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getNullChar());
-            }
+            return getAppropriateValue(connection, connection.getNullChar());
         }
         if (value.equals("SCHEMA")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getUiSchema())) {
-                return connection.getUiSchema();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getUiSchema());
-            }
+            return getAppropriateValue(connection, connection.getUiSchema());
         }
         if (value.equals("FILE")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getFileFieldName())) {
-                return connection.getFileFieldName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getFileFieldName());
-            }
+            return getAppropriateValue(connection, connection.getFileFieldName());
         }
         if (value.equals("PROPERTIES_STRING")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getAdditionalParams())) {
-                return connection.getAdditionalParams();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getAdditionalParams());
-            }
+            return getAppropriateValue(connection, connection.getAdditionalParams());
         }
 
         if (value.equals("CDC_MODE")) { //$NON-NLS-1$
@@ -1163,12 +1126,7 @@ public class RepositoryToComponentProperty {
         }
         // add this for tJavaDB embeded "DB Root Path"
         if (value.equals("DIRECTORY")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDBRootPath())) {
-                return connection.getDBRootPath();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDBRootPath());
-            }
-
+            return getAppropriateValue(connection, connection.getDBRootPath());
         }
         // add for feature 11674
         if (value.equals("RUNNING_MODE")) {//$NON-NLS-1$       
@@ -1183,25 +1141,13 @@ public class RepositoryToComponentProperty {
             return runningMode;
         }
         if (value.equals("DBPATH")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDBRootPath())) {
-                return connection.getDBRootPath();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDBRootPath());
-            }
+            return getAppropriateValue(connection, connection.getDBRootPath());
         }
         if (value.equals("DBNAME")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDatasourceName())) {
-                return connection.getDatasourceName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDatasourceName());
-            }
+            return getAppropriateValue(connection, connection.getDatasourceName());
         }
         if (value.equals("RAC_URL")) {
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
 
         if (value.equals("DISTRIBUTION")) {
@@ -1229,13 +1175,15 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("HBASE_MASTER_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_MASTERPRINCIPAL));
+            String hbaseMasterPrinc = connection.getParameters().get(
+                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_MASTERPRINCIPAL);
+            return getAppropriateValue(connection, hbaseMasterPrinc);
         }
 
         if (value.equals("HBASE_REGIONSERVER_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_REGIONSERVERPRINCIPAL));
+            String hbaseRegPrinc = connection.getParameters().get(
+                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_REGIONSERVERPRINCIPAL);
+            return getAppropriateValue(connection, hbaseRegPrinc);
         }
 
         if (value.equals("HIVE_SERVER")) {
@@ -1300,11 +1248,7 @@ public class RepositoryToComponentProperty {
         if (value.equals(EParameterNameForComponent.PARA_NAME_MAPRED_JT.getName())
                 || value.equals(EParameterNameForComponent.PARA_NAME_RESOURCE_MANAGER.getName())) {
             String jobTrackerURL = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_URL);
-            if (isContextMode(connection, jobTrackerURL)) {
-                return jobTrackerURL;
-            } else {
-                return TalendQuoteUtils.addQuotes(jobTrackerURL);
-            }
+            return getAppropriateValue(connection, jobTrackerURL);
         }
 
         if (value.equals(EParameterNameForComponent.PARA_NAME_USE_YARN.getName())) {
@@ -1343,20 +1287,12 @@ public class RepositoryToComponentProperty {
 
         if (value.equals("MAPRED_JOB_TRACKER") || value.equals("MAPRED_RESOURCE_MANAGER")) {
             String mapredJobTracker = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_URL);
-            if (mapredJobTracker == null) {
-                return mapredJobTracker;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(mapredJobTracker));
-            }
+            return getAppropriateValue(connection, mapredJobTracker);
         }
 
         if (value.equals("NAMENODE_PRINCIPAL")) {
             String nameNodePrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_NAME_NODE_PRINCIPAL);
-            if (nameNodePrincipal == null) {
-                return nameNodePrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(nameNodePrincipal));
-            }
+            return getAppropriateValue(connection, nameNodePrincipal);
         }
 
         /**
@@ -1366,36 +1302,20 @@ public class RepositoryToComponentProperty {
          */
         if (value.equals("JOBTRACKER_PRINCIPAL") || value.equals("RESOURCEMANAGER_PRINCIPAL")) {
             String jtOrRmPrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_PRINCIPAL);
-            if (jtOrRmPrincipal == null) {
-                return jtOrRmPrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(jtOrRmPrincipal));
-            }
+            return getAppropriateValue(connection, jtOrRmPrincipal);
         }
 
         if (value.equals("JOBHISTORY_PRINCIPAL")) {
             String jobHistoryPrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_HISTORY_PRINCIPAL);
-            if (jobHistoryPrincipal == null) {
-                return jobHistoryPrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(jobHistoryPrincipal));
-            }
+            return getAppropriateValue(connection, jobHistoryPrincipal);
         }
 
         if (value.equals("ZOOKEEPER_QUORUM")) {
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
 
         if (value.equals("ZOOKEEPER_CLIENT_PORT")) {
-            if (isContextMode(connection, connection.getPort())) {
-                return connection.getPort();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getPort());
-            }
+            return getAppropriateValue(connection, connection.getPort());
         }
 
         if (value.equals("COLUMN_MAPPING")) { //$NON-NLS-1$
@@ -1403,24 +1323,28 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("HIVE_PRINCIPAL")) {
-            return TalendQuoteUtils
-                    .addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_HIVEPRINCIPLA));
+            String hivePrincipal = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_HIVEPRINCIPLA);
+            return getAppropriateValue(connection, hivePrincipal);
         }
 
         if (value.equals("METASTORE_JDBC_URL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_METASTOREURL));
+            String metadataUrl = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_METASTOREURL);
+            return getAppropriateValue(connection, metadataUrl);
         }
 
         if (value.equals("METASTORE_CLASSNAME")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_DRIVERCLASS));
+            String metastoreDC = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_DRIVERCLASS);
+            return getAppropriateValue(connection, metastoreDC);
         }
 
         if (value.equals("METASTORE_USERNAME")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_USERNAME));
+            String metastoreUser = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_USERNAME);
+            return getAppropriateValue(connection, metastoreUser);
         }
 
         if (value.equals("METASTORE_PASSWORD")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_PASSWORD));
+            String metastorePwd = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_PASSWORD);
+            return getAppropriateValue(connection, metastorePwd);
         }
 
         if (value.equals("USE_KEYTAB")) {
@@ -1433,15 +1357,18 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("PRINCIPAL") || value.equals("KEYTAB_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB_PRINCIPAL));
+            String principal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB_PRINCIPAL);
+            return getAppropriateValue(connection, principal);
         }
 
         if (value.equals("KEYTAB_PATH")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB));
+            String keytab = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB);
+            return getAppropriateValue(connection, keytab);
         }
 
         if (value.equals("IMPALA_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA));
+            String principal = connection.getParameters().get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA);
+            return getAppropriateValue(connection, principal);
         }
 
         if (value.equals("IMPALA_VERSION")) {
@@ -1461,8 +1388,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("RESOURCEMANAGER_SCHEDULER_ADDRESS")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_RESOURCEMANAGER_SCHEDULER_ADDRESS));
+            String rmSchAdr = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_RESOURCEMANAGER_SCHEDULER_ADDRESS);
+            return getAppropriateValue(connection, rmSchAdr);
         }
 
         if (value.equals("SET_JOBHISTORY_ADDRESS")) {
@@ -1470,7 +1397,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("JOBHISTORY_ADDRESS")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOBHISTORY_ADDRESS));
+            String jhAddr = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOBHISTORY_ADDRESS);
+            return getAppropriateValue(connection, jhAddr);
         }
 
         if (value.equals("SET_STAGING_DIRECTORY")) {
@@ -1478,7 +1406,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("STAGING_DIRECTORY")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_STAGING_DIRECTORY));
+            String stageDir = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_STAGING_DIRECTORY);
+            return getAppropriateValue(connection, stageDir);
         }
 
         if (value.equals("USE_DATANODE_HOSTNAME")) {
@@ -1490,6 +1419,14 @@ public class RepositoryToComponentProperty {
 
     }
 
+    private static String getAppropriateValue(Connection connection, String rawValue) {
+        if (isContextMode(connection, rawValue)) {
+            return rawValue;
+        } else {
+            return TalendQuoteUtils.addQuotesIfNotExist(rawValue);
+        }
+    }
+
     /**
      * DOC nrousseau Comment method "getDatabaseValue".
      * 
@@ -2694,4 +2631,14 @@ public class RepositoryToComponentProperty {
         }
         return value;
     }
+
+    public static boolean isGenericRepositoryValue(Connection connection, List<ComponentProperties> componentProperties,
+            String paramName) {
+        for (IDragAndDropServiceHandler handler : DragAndDropManager.getHandlers()) {
+            if (handler.canHandle(connection)) {
+                return handler.isGenericRepositoryValue(componentProperties, paramName);
+            }
+        }
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/types/Java2STLangTypesHelper.java

@@ -60,8 +60,8 @@ public class Java2STLangTypesHelper {
         return "string"; //$NON-NLS-1$
 
     }
-	
-	/**
+
+    /**
      * DOC return as: int, long, double, float, string.
      * 
      * @author rdubois
@@ -69,32 +69,34 @@ public class Java2STLangTypesHelper {
      * @param columnLabel
      * @return
      */
-	
-	public static String getPigType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig type according to the talendType
-            IMetadataColumn column = metadataTable.getColumn(columnLabel);
-            String talendType = column.getTalendType();
 
-            String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
-            // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
-            if (typeToGenerate.equals("int") || typeToGenerate.equals("short") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                return "int"; //$NON-NLS-1$
-            } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
-                return "long"; //$NON-NLS-1$
-            } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$
-                return "double"; //$NON-NLS-1$
-            }
+    public static String getPigType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig
+                                                                                        // type according to the
+                                                                                        // talendType
+        IMetadataColumn column = metadataTable.getColumn(columnLabel);
+        String talendType = column.getTalendType();
 
-            // Floats: float(Float)
-            if (typeToGenerate.equals("float")) { //$NON-NLS-1$ //$NON-NLS-2$
-                return "float"; //$NON-NLS-1$
-            }
+        String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
+        // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
+        if (typeToGenerate.equals("int") || typeToGenerate.equals("short") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+            return "int"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
+            return "long"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$
+            return "double"; //$NON-NLS-1$
+        }
 
-            // others treat as string
-            return "chararray"; //$NON-NLS-1$
-	
-       }
-	   
-	   	/**
+        // Floats: float(Float)
+        if (typeToGenerate.equals("float")) { //$NON-NLS-1$ 
+            return "float"; //$NON-NLS-1$
+        }
+
+        // others treat as string
+        return "chararray"; //$NON-NLS-1$
+
+    }
+
+    /**
      * DOC return as: int, long, double, float, string.
      * 
      * @author rdubois
@@ -102,34 +104,36 @@ public class Java2STLangTypesHelper {
      * @param columnLabel
      * @return
      */
-	
-	public static String getHCatType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig type according to the talendType
-            IMetadataColumn column = metadataTable.getColumn(columnLabel);
-            String talendType = column.getTalendType();
 
-            String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
-            // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
-            if (typeToGenerate.equals("int") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                return "int"; //$NON-NLS-1$
-            } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
-                return "bigint"; //$NON-NLS-1$
-			} else if (typeToGenerate.equals("short")) { //$NON-NLS-1$
-				return "smallint";
-			} else if (typeToGenerate.equals("boolean")) { //$NON-NLS-1$
-				return "boolean";
-            } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$
-                return "double"; //$NON-NLS-1$
-            }
+    public static String getHCatType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig
+                                                                                         // type according to the
+                                                                                         // talendType
+        IMetadataColumn column = metadataTable.getColumn(columnLabel);
+        String talendType = column.getTalendType();
 
-            // Floats: float(Float)
-            if (typeToGenerate.equals("float")) { //$NON-NLS-1$ //$NON-NLS-2$
-                return "float"; //$NON-NLS-1$
-            }
+        String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
+        // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
+        if (typeToGenerate.equals("int") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ 
+            return "int"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
+            return "bigint"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("short")) { //$NON-NLS-1$
+            return "smallint"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("boolean")) { //$NON-NLS-1$
+            return "boolean"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$ //$NON-NLS-2$
+            return "double"; //$NON-NLS-1$
+        }
 
-            // others treat as string
-            return "string"; //$NON-NLS-1$
-	
-       }
+        // Floats: float(Float)
+        if (typeToGenerate.equals("float")) { //$NON-NLS-1$ 
+            return "float"; //$NON-NLS-1$
+        }
+
+        // others treat as string
+        return "string"; //$NON-NLS-1$
+
+    }
 
     /**
      * DOC return as: %d, %f, %b, %s.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EParameterFieldType.java

@@ -98,6 +98,8 @@ public enum EParameterFieldType {
     NAME_SELECTION_AREA, // ycbai added for generic wizard
     NAME_SELECTION_REFERENCE, // ycbai added for generic wizard
     COMPONENT_REFERENCE, // ycbai added for generic wizard
+    HIDDEN_TEXT, // hcyi added for generic wizard/component properties
+    SCHEMA_REFERENCE, // hcyi added for component properties
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ElementParameterParser.java

@@ -317,7 +317,7 @@ public final class ElementParameterParser {
         return null;
     }
 
-    private static List<Map<String, String>> createTableValues(final List<Map<String, Object>> paramValues,
+    public static List<Map<String, String>> createTableValues(final List<Map<String, Object>> paramValues,
             final IElementParameter param) {
         List<Map<String, String>> tableValues = new ArrayList<Map<String, String>>();
         if (paramValues != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IGenericProvider.java

@@ -0,0 +1,26 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+import java.util.List;
+
+/**
+ * created by hcyi on Mar 25, 2016 Detailled comment
+ *
+ */
+public interface IGenericProvider {
+
+    public void loadComponentsFromExtensionPoint();
+
+    public List<?> addPaletteEntry();
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -64,7 +64,7 @@ import org.talend.repository.model.IProxyRepositoryService;
  */
 @SuppressWarnings("unchecked")
 public final class ProcessUtils {
-    
+
     private static boolean isHD;
 
     private static List<IProcess> fakeProcesses = new ArrayList<IProcess>();
@@ -751,6 +751,20 @@ public final class ProcessUtils {
         return false;
     }
 
+    public static boolean isSparkStreming(IProcess process) {
+        if (process instanceof IProcess2) {
+            return false;
+        }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerCoreService.class)) {
+            ITestContainerCoreService testContainerService = (ITestContainerCoreService) GlobalServiceRegister.getDefault()
+                    .getService(ITestContainerCoreService.class);
+            if (testContainerService != null) {
+                return testContainerService.isSparkStreaming(((IProcess2) process).getProperty().getItem());
+            }
+        }
+        return false;
+    }
+
     public static List<String> getTestData(IProcess process, String instanceName) {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService = (ITestContainerProviderService) GlobalServiceRegister
@@ -930,26 +944,26 @@ public final class ProcessUtils {
 
         return false;
     }
-    
+
     public static boolean isHDInsight() {
         return isHD;
     }
-    
+
     public static void setHDInsight(boolean isHD) {
         ProcessUtils.isHD = isHD;
     }
-    
+
     public static boolean isDistributionExist(ProcessItem processItem) {
         EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
         for (ElementParameterType pt : parameters) {
             if (pt.getName().equals("DISTRIBUTION")) { //$NON-NLS-1$
-                 String value = pt.getValue();
-                 if("MICROSOFT_HD_INSIGHT".equals(value)){ //$NON-NLS-1$
-                     return true;
-                 }
+                String value = pt.getValue();
+                if ("MICROSOFT_HD_INSIGHT".equals(value)) { //$NON-NLS-1$
+                    return true;
+                }
             }
         }
         return false;
     }
-    
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/relationship/RelationshipItemBuilder.java

@@ -434,6 +434,11 @@ public class RelationshipItemBuilder {
         }
     }
 
+    public void clearAllItemsRelations() {
+        currentProjectItemsRelations.clear();
+        referencesItemsRelations.clear();
+    }
+
     public void mergeRelationship(Map<Relation, Set<Relation>> itemRelations, Map<Relation, Set<Relation>> newRelations) {
         if (itemRelations != null && newRelations != null) {
             for (Relation relation : newRelations.keySet()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -87,15 +87,16 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static final String PROD_DQ = ProductUtils.PROD_DQ;
 
-    public static final String PROD_CAMEL = ProductUtils.PROD_CAMEL;
-
     public static final String PROD_MDM = ProductUtils.PROD_MDM;
 
+    public final static ERepositoryObjectType GENERAL_FILE = new ERepositoryObjectType("repository.generalfile", "",
+            "GENERAL_FILE", true, 1, new String[] { PROD_DI, PROD_DQ, PROD_MDM }, new String[] {}, false);
+
     public final static ERepositoryObjectType GIT_ROOT = new ERepositoryObjectType("repository.gitroot", "", "GIT_ROOT", true, 1,
-            new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
+            new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SVN_ROOT = new ERepositoryObjectType("repository.svnroot", "", "SVN_ROOT", true, 1,
-            new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
+            new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SVG_BUSINESS_PROCESS = new ERepositoryObjectType("repository.svgBusinessProcess",
             "businessProcessSVG", "SVG_BUSINESS_PROCESS", true, 3, new String[] { PROD_DI }, new String[] {});
@@ -107,7 +108,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
     // "documentations", "DOCUMENTATION", true, 11, new String[] { PROD_DI }, new String[] {});
 
     public final static ERepositoryObjectType METADATA = new ERepositoryObjectType("repository.metadata", "metadata", "METADATA",
-            true, 8, new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
+            true, 80, new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType METADATA_CON_TABLE = new ERepositoryObjectType("repository.metadataTable",
             "METADATA_CON_TABLE", 13, true, true, new String[] { PROD_DI }, new String[] {}, false);
@@ -149,7 +150,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             "MDM_ELEMENT_TYPE", 25, true, true, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType RECYCLE_BIN = new ERepositoryObjectType("repository.recyclebin", "", "RECYCLE_BIN",
-            true, 26, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false);
+            true, 900, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType METADATA_COLUMN = new ERepositoryObjectType("repository.column", "",
             "METADATA_COLUMN", true, 27, new String[] { PROD_DI }, new String[] {}, false);
@@ -184,22 +185,6 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             "REFERENCED_PROJECTS", 51, true, "repository.referencedProjects.alias", new String[] { PROD_DI, PROD_DQ },
             new String[] {}, false);
 
-    public final static ERepositoryObjectType GENERATED = new ERepositoryObjectType("repository.generated", //$NON-NLS-1$
-            "documentations/generated", "GENERATED", true, 52, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOBS = new ERepositoryObjectType(
-            "repository.jobs", "documentations/generated/jobs", //$NON-NLS-1$
-            "JOBS", true, 53, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOB_DOC = new ERepositoryObjectType("repository.jobdoc", //$NON-NLS-1$
-            "documentations/generated/jobs", "JOB_DOC", true, 54, new String[] { PROD_DI }, new String[] {});
-
-    public final static ERepositoryObjectType JOBLETS = new ERepositoryObjectType("repository.joblets", //$NON-NLS-1$
-            "documentations/generated/joblets", "JOBLETS", true, 58, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOBLET_DOC = new ERepositoryObjectType("repository.jobletdoc", //$NON-NLS-1$
-            "documentations/generated/joblets", "JOBLET_DOC", true, 59, new String[] { PROD_DI }, new String[] {});
-
     public final static ERepositoryObjectType COMPONENTS = new ERepositoryObjectType("repository.components", "components", //$NON-NLS-1$
             "COMPONENTS", true, 61, new String[] { PROD_DI }, new String[] {}, false);
 
@@ -361,10 +346,10 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             true, "repository.systemIndicators.phoneNumberStatistics.alias", new String[] { PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SERVICESOPERATION = new ERepositoryObjectType("repository.servicesOperation", //$NON-NLS-1$
-            "SERVICESOPERATION", 101, true, true, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false); //$NON-NLS-1$
+            "SERVICESOPERATION", 101, true, true, new String[] { PROD_DI }, new String[] {}, false); //$NON-NLS-1$
 
     public final static ERepositoryObjectType SERVICESPORT = new ERepositoryObjectType("repository.servicesPort", "SERVICESPORT", //$NON-NLS-1$ //$NON-NLS-2$
-            102, true, true, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false);
+            102, true, true, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType SYSTEM_INDICATORS_FRAUDDETECTION = new ERepositoryObjectType(
             "repository.systemIndicators.fraudDetection", "TDQ_Libraries/Indicators/System Indicators/Fraud Detection", //$NON-NLS-1$ //$NON-NLS-2$
@@ -436,8 +421,22 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType SQLPATTERNS = ERepositoryObjectType.valueOf("SQLPATTERNS"); //$NON-NLS-1$
 
+    /**
+     * Docs
+     * 
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
     public final static ERepositoryObjectType DOCUMENTATION = ERepositoryObjectType.valueOf("DOCUMENTATION"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType GENERATED = ERepositoryObjectType.valueOf("GENERATED");//$NON-NLS-1$
+
+    public final static ERepositoryObjectType JOB_DOC = ERepositoryObjectType.valueOf("JOB_DOC");//$NON-NLS-1$
+
+    public final static ERepositoryObjectType JOBLET_DOC = ERepositoryObjectType.valueOf("JOBLET_DOC");//$NON-NLS-1$
+
+    /**
+     * Metadatas
+     */
     public final static ERepositoryObjectType METADATA_FILE_POSITIONAL = ERepositoryObjectType
             .valueOf("METADATA_FILE_POSITIONAL"); //$NON-NLS-1$
 
@@ -460,11 +459,6 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType METADATA_MDMCONNECTION = ERepositoryObjectType.valueOf("METADATA_MDMCONNECTION"); //$NON-NLS-1$
 
-    /**
-     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
-     */
-    public final static ERepositoryObjectType PROCESS_MR = ERepositoryObjectType.valueOf("PROCESS_MR"); //$NON-NLS-1$
-
     public final static ERepositoryObjectType METADATA_CONNECTIONS = ERepositoryObjectType.valueOf("METADATA_CONNECTIONS");
 
     public final static ERepositoryObjectType METADATA_FILE_DELIMITED = ERepositoryObjectType.valueOf("METADATA_FILE_DELIMITED");
@@ -482,6 +476,11 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType METADATA_EDIFACT = ERepositoryObjectType.valueOf("METADATA_EDIFACT");
 
+    /**
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
+    public final static ERepositoryObjectType PROCESS_MR = ERepositoryObjectType.valueOf("PROCESS_MR"); //$NON-NLS-1$
+
     /**
      * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
      */
@@ -515,6 +514,18 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      */
     public final static ERepositoryObjectType PROCESS_SPARKSTREAMING = ERepositoryObjectType.valueOf("PROCESS_SPARKSTREAMING");
 
+    public final static ERepositoryObjectType METADATA_SAP_BW_DATASOURCE = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_DATASOURCE"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_DATASTOREOBJECT = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_DATASTOREOBJECT"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_INFOCUBE = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_INFOCUBE"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_INFOOBJECT = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_INFOOBJECT"); //$NON-NLS-1$
+
     private static Map<String, ERepositoryObjectType> typeCacheById = new HashMap<String, ERepositoryObjectType>();
 
     ERepositoryObjectType(String key, String folder, String type, boolean isStaticNode, int ordinal, String[] products,
@@ -610,8 +621,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             // limit with DI items only.
             List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
             for (ERepositoryObjectType currentType : values(ERepositoryObjectType.class)) {
-                if (ArrayUtils.contains(currentType.getProducts(), PROD_DI)
-                        || ArrayUtils.contains(currentType.getProducts(), PROD_CAMEL)) {
+                if (ArrayUtils.contains(currentType.getProducts(), PROD_DI)) {
                     toReturn.add(currentType);
                 }
             }
@@ -707,9 +717,9 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         return type;
     }
 
-    
     /**
      * Getter for typeCacheById.
+     * 
      * @return the typeCacheById
      */
     public static Map<String, ERepositoryObjectType> getTypeCacheById() {
@@ -730,12 +740,12 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static String getFolderName(ERepositoryObjectType type) {
 
-        if (type == GENERATED || type == JOBS || type == JOB_DOC) {
+        if (type == GENERATED || type == JOB_DOC) {
             if ((PluginChecker.isDocumentationPluginLoaded())) {
                 return type.getFolder();
             }
         }
-        if (type == JOBLETS || type == JOBLET_DOC) {
+        if (type == JOBLET_DOC) {
             if (PluginChecker.isJobLetPluginLoaded()) {
                 return type.getFolder();
             }
@@ -843,6 +853,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      * Please use instead the method : public static ERepositoryObjectType getType(Property property)<br>
      * 
      * This method might change to private in the future, since many calls will have low performance.
+     * 
      * @param item
      * @return
      */
@@ -1358,6 +1369,10 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         if (ERepositoryObjectType.PROCESS_ROUTELET != null) {
             allTypes.add(ERepositoryObjectType.PROCESS_ROUTELET);
         }
+        // testContainer is also instance of process
+        if (ERepositoryObjectType.TEST_CONTAINER != null) {
+            allTypes.add(ERepositoryObjectType.TEST_CONTAINER);
+        }
 
         return allTypes;
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/temp/ECodePart.java

@@ -34,9 +34,11 @@ public enum ECodePart {
     SPARKCODE("sparkcode"), //$NON-NLS-1$
     SPARKCONFIG("sparkconfig"), //$NON-NLS-1$
     SPARKJOBFOOTER("sparkjobfooter"), //$NON-NLS-1$
+    SPARKPOSTPROCESSING("sparkpostprocessing"), //$NON-NLS-1$
     SPARKSTREAMINGCODE("sparkstreamingcode"), //$NON-NLS-1$
     SPARKSTREAMINGCONFIG("sparkstreamingconfig"), //$NON-NLS-1$
-    SPARKSTREAMINGJOBFOOTER("sparkstreamingjobfooter"); //$NON-NLS-1$
+    SPARKSTREAMINGJOBFOOTER("sparkstreamingjobfooter"), //$NON-NLS-1$
+    SPARKSTREAMINGPOSTPROCESSING("sparkstreamingpostprocessing"); //$NON-NLS-1$
 
     private ECodePart(String name) {
         this.name = name;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/update/RepositoryUpdateManager.java

@@ -93,6 +93,7 @@ import org.talend.core.service.IMRProcessService;
 import org.talend.core.service.IMetadataManagmentService;
 import org.talend.core.service.IStormProcessService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.runprocess.ItemCacheManager;
 import org.talend.repository.model.IProxyRepositoryFactory;
@@ -1642,6 +1643,11 @@ public abstract class RepositoryUpdateManager {
 
     public static boolean updateMultiSchema(ConnectionItem connItem, List<IMetadataTable> oldMetadataTable,
             Map<String, String> oldTableMap) {
+        return updateMultiSchema(connItem, oldMetadataTable, oldTableMap, null);
+    }
+
+    public static boolean updateMultiSchema(ConnectionItem connItem, List<IMetadataTable> oldMetadataTable,
+            Map<String, String> oldTableMap, String bwTableType) {
         if (connItem == null) {
             return false;
         }
@@ -1653,8 +1659,8 @@ public abstract class RepositoryUpdateManager {
 
         if (!update) {
             if (oldMetadataTable != null) {
-                List<IMetadataTable> newMetadataTable = RepositoryUpdateManager.getConversionMetadataTables(connItem
-                        .getConnection());
+                List<IMetadataTable> newMetadataTable = RepositoryUpdateManager.getConversionMetadataTables(
+                        connItem.getConnection(), bwTableType);
                 update = !RepositoryUpdateManager.sameAsMetadatTable(newMetadataTable, oldMetadataTable, oldTableMap);
                 isAddColumn = isAddColumn(newMetadataTable, oldMetadataTable);
             }
@@ -1665,9 +1671,14 @@ public abstract class RepositoryUpdateManager {
 
             List<IMetadataTable> newMetadataTable = new ArrayList<IMetadataTable>();
             if (coreService != null
-                    && ((connection instanceof DatabaseConnection) || (connection instanceof GenericSchemaConnection))) {
-                Set<org.talend.core.model.metadata.builder.connection.MetadataTable> newTables = ConnectionHelper
-                        .getTables(connection);
+                    && ((connection instanceof DatabaseConnection) || (connection instanceof GenericSchemaConnection))
+                    || (connection instanceof SAPConnection)) {
+                Set<org.talend.core.model.metadata.builder.connection.MetadataTable> newTables = null;
+                if (bwTableType == null) {
+                    newTables = ConnectionHelper.getTables(connection);
+                } else {
+                    newTables = SAPBWTableHelper.getBWTables(connection, bwTableType);
+                }
                 if (newTables != null) {
                     for (org.talend.core.model.metadata.builder.connection.MetadataTable originalTable : newTables) {
                         IMetadataTable conversionTable = coreService.convert(originalTable);
@@ -1783,6 +1794,15 @@ public abstract class RepositoryUpdateManager {
                 String tableId = newTable.getId();
                 if (tableLabel.equals(oldtableLabel)) {
                     isDeleted = false;
+                    String newInnerIOType = newTable.getAdditionalProperties().get(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE);
+                    String oldInnerIOType = oldTable.getAdditionalProperties().get(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE);
+                    if (newInnerIOType != null) {
+                        if (newInnerIOType.equals(oldInnerIOType) && !tableId.equals(oldtableId)) {
+                            prefix = connItem.getProperty().getId() + UpdatesConstants.SEGMENT_LINE;
+                            deletedOrReselectTables.put(prefix + tableLabel, EUpdateResult.RELOAD);
+                        }
+                        continue;
+                    }
                     /* if table name is same but tableId is not same,means table has been deselect and reselect */
                     if (!tableId.equals(oldtableId)) {
                         prefix = connItem.getProperty().getId() + UpdatesConstants.SEGMENT_LINE;
@@ -2197,14 +2217,22 @@ public abstract class RepositoryUpdateManager {
         return repositoryUpdateManager.doWork();
     }
 
-    @SuppressWarnings("unchecked")
     public static List<IMetadataTable> getConversionMetadataTables(Connection conn) {
+        return getConversionMetadataTables(conn, null);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static List<IMetadataTable> getConversionMetadataTables(Connection conn, String bwTableType) {
         if (conn == null) {
             return Collections.emptyList();
         }
         List<IMetadataTable> tables = new ArrayList<IMetadataTable>();
-
-        Set tables2 = ConnectionHelper.getTables(conn);
+        Set tables2 = null;
+        if (bwTableType != null) {
+            tables2 = SAPBWTableHelper.getBWTables(conn, bwTableType);
+        } else {
+            tables2 = ConnectionHelper.getTables(conn);
+        }
         if (tables2 != null) {
             for (org.talend.core.model.metadata.builder.connection.MetadataTable originalTable : (Set<org.talend.core.model.metadata.builder.connection.MetadataTable>) tables2) {
                 if (GlobalServiceRegister.getDefault().isServiceRegistered(IMetadataManagmentService.class)) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/AbstractDragAndDropServiceHandler.java

@@ -12,6 +12,9 @@
 // ============================================================================
 package org.talend.core.model.utils;
 
+import java.util.List;
+
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.utils.TalendQuoteUtils;
@@ -56,4 +59,8 @@ public abstract class AbstractDragAndDropServiceHandler implements IDragAndDropS
         }
     }
 
+    @Override
+    public boolean isGenericRepositoryValue(List<ComponentProperties> componentProperties, String paramName) {
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/IDragAndDropServiceHandler.java

@@ -14,6 +14,7 @@ package org.talend.core.model.utils;
 
 import java.util.List;
 
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
@@ -109,4 +110,13 @@ public interface IDragAndDropServiceHandler {
      */
     public boolean isValidForDataViewer(Connection connection, IMetadataTable metadataTable);
 
+    /**
+     * 
+     * DOC hcyi Comment method "isGenericRepositoryValue".
+     * 
+     * @param connection
+     * @param paramName
+     * @return
+     */
+    public boolean isGenericRepositoryValue(List<ComponentProperties> componentProperties, String paramName);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusConstants.java

@@ -20,6 +20,8 @@ public class NexusConstants {
 
     public static final String SERVICES_SEARCH = "service/local/lucene/search?";//$NON-NLS-1$ 
 
+    public static final String SERVICES_RESOLVE = "service/local/artifact/maven/resolve?";//$NON-NLS-1$ 
+
     public static final String CONTENT_REPOSITORIES = "/content/repositories/";//$NON-NLS-1$ 
 
     public static final String SLASH = "/";//$NON-NLS-1$ 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusServerUtils.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.nexus;
 
+import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.net.Authenticator;
 import java.net.HttpURLConnection;
@@ -25,15 +26,19 @@ import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSocketFactory;
 
+import org.apache.commons.lang.StringUtils;
 import org.dom4j.Document;
 import org.dom4j.Node;
 import org.dom4j.io.SAXReader;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.service.IRemoteService;
 import org.talend.utils.json.JSONException;
@@ -46,6 +51,11 @@ import org.talend.utils.ssl.SSLUtils;
  */
 public class NexusServerUtils {
 
+    /**
+     * 
+     */
+    public static final String ORG_TALEND_DESIGNER_CORE = "org.talend.designer.core"; //$NON-NLS-1$
+
     public static final String KEY_NEXUS_RUL = "nexusUrl";//$NON-NLS-1$ 
 
     public static final String KEY_NEXUS_USER = "username";//$NON-NLS-1$ 
@@ -131,6 +141,9 @@ public class NexusServerUtils {
      * @return
      */
     public static boolean checkConnectionStatus(String nexusUrl, String repositoryId, final String userName, final String password) {
+        if (StringUtils.isEmpty(nexusUrl)) {
+            return false;
+        }
         final Authenticator defaultAuthenticator = NetworkUtil.getDefaultAuthenticator();
         if (userName != null && !"".equals(userName)) {
             Authenticator.setDefault(new Authenticator() {
@@ -155,10 +168,6 @@ public class NexusServerUtils {
 
             URL url = new URL(urlToCheck);
             HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
-            // if (userName != null && !"".equals(userName)) {
-            // urlConnection.setRequestProperty(
-            //                        "Authorization", "Basic " + Base64.encodeBase64((userName + ":" + password).getBytes()));//$NON-NLS-1$ //$NON-NLS-2$
-            // }
             if (urlConnection instanceof HttpsURLConnection) {
                 String userDir = Platform.getInstallLocation().getURL().getPath();
                 final SSLSocketFactory socketFactory = SSLUtils.getSSLContext(userDir).getSocketFactory();
@@ -173,14 +182,17 @@ public class NexusServerUtils {
 
                 });
             }
-            urlConnection.setConnectTimeout(10000);
-            urlConnection.setReadTimeout(10000);
+            IEclipsePreferences node = InstanceScope.INSTANCE.getNode(ORG_TALEND_DESIGNER_CORE);
+            int timeout = node.getInt(ITalendCorePrefConstants.NEXUS_TIMEOUT, 10000);
+
+            urlConnection.setConnectTimeout(timeout);
+            urlConnection.setReadTimeout(timeout);
             status = urlConnection.getResponseCode();
             if (status == CONNECTION_OK) {
                 return true;
             }
         } catch (Exception e) {
-            // do nothing
+            ExceptionHandler.process(e);
         } finally {
             Authenticator.setDefault(defaultAuthenticator);
         }
@@ -302,6 +314,47 @@ public class NexusServerUtils {
         }
     }
 
+    public static String resolveSha1(String nexusUrl, final String userName, final String password, String repositoryId,
+            String groupId, String artifactId, String version) throws Exception {
+        HttpURLConnection urlConnection = null;
+        final Authenticator defaultAuthenticator = NetworkUtil.getDefaultAuthenticator();
+        if (userName != null && !"".equals(userName)) {
+            Authenticator.setDefault(new Authenticator() {
+
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(userName, password.toCharArray());
+                }
+
+            });
+        }
+        try {
+            String service = NexusConstants.SERVICES_RESOLVE + "a=" + artifactId + "&g=" + groupId + "&r=" + repositoryId + "&v="
+                    + version;
+            urlConnection = getHttpURLConnection(nexusUrl, service, userName, password);
+            SAXReader saxReader = new SAXReader();
+
+            InputStream inputStream = urlConnection.getInputStream();
+            Document document = saxReader.read(inputStream);
+
+            Node sha1Node = document.selectSingleNode("/artifact-resolution/data/sha1");
+            String sha1 = null;
+            if (sha1Node != null) {
+                sha1 = sha1Node.getText();
+            }
+            return sha1;
+
+        } catch (FileNotFoundException e) {
+            // jar not existing on remote nexus
+            return null;
+        } finally {
+            Authenticator.setDefault(defaultAuthenticator);
+            if (null != urlConnection) {
+                urlConnection.disconnect();
+            }
+        }
+    }
+
     private static String getSearchQuery(String repositoryId, String groupId, String artifactId, String version, int from,
             int count) {
         String query = "";//$NON-NLS-1$ 
@@ -352,7 +405,9 @@ public class NexusServerUtils {
 
             });
         }
-        urlConnection.setConnectTimeout(10000);
+        IEclipsePreferences node = InstanceScope.INSTANCE.getNode(ORG_TALEND_DESIGNER_CORE);
+        int timeout = node.getInt(ITalendCorePrefConstants.NEXUS_TIMEOUT, 10000);
+        urlConnection.setConnectTimeout(timeout);
         return urlConnection;
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/ITalendCorePrefConstants.java

@@ -181,4 +181,8 @@ public interface ITalendCorePrefConstants {
     public static final String PERFORMANCE_TAC_CONNECTION_TIMEOUT = "PERFORMANCE_TAC_CONNECTION_TIMEOUT"; //$NON-NLS-1$
 
     public static final String PERFORMANCE_JAVA_PROCESS_CODE_FORMATE_TIMEOUT = "PERFORMANCE_JAVA_PROCESS_CODE_FORMATE_TIMEOUT"; //$NON-NLS-1$
+
+    public static final String NEXUS_TIMEOUT = "NEXUS_TIMEOUT"; //$NON-NLS-1$
+
+    public static final String NEXUS_REFRESH_FREQUENCY = "NEXUS_REFRESH_FREQUENCY"; //$NON-NLS-1$
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/PreferenceManipulator.java

@@ -21,6 +21,7 @@ import java.util.StringTokenizer;
 import org.eclipse.jface.preference.IPersistentPreferenceStore;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.ui.PlatformUI;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.Hex;
 import org.talend.core.model.general.ConnectionBean;
 import org.talend.utils.json.JSONArray;
@@ -39,9 +40,13 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
 
     private static final String PREF_DELIMITER = "|"; //$NON-NLS-1$
 
+    private static final String LOCAL_MODE = "local";
+
     /** The preference store manipulated. */
     private IPreferenceStore store;
 
+    private String PROJECT_BRANCH_MANAGEMENT = "localBranchManagement";
+
     /**
      * Constructs a new PreferenceManipulator.
      * 
@@ -246,6 +251,20 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
         save();
     }
 
+    @Deprecated
+    public int getLastLogonMode(String projectUrl, String projectName, String branch) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
+        int mode = store.getInt(hexKey);
+        return mode;
+    }
+
+    @Deprecated
+    public void setLastLogonMode(String projectUrl, String projectName, String branch, int mode) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
+        store.setValue(hexKey, mode);
+        save();
+    }
+
     public String getLastUser() {
         return store.getString(LAST_USED_USER);
     }
@@ -284,4 +303,37 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
             }
         }
     }
+
+    public void setNewCreateBranchObjectId(String projectUrl, String projectName, String branch, String id) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
+        store.setValue(hexKey, id);
+        save();
+    }
+
+    public String getNewCreateBranchObjectId(String projectUrl, String projectName, String branch) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
+        return store.getString(hexKey);
+    }
+
+    public JSONObject getLogonLocalBranchStatus(String projectUrl, String projectName) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + PROJECT_BRANCH_MANAGEMENT).getBytes());
+        String jsonString = store.getString(hexKey);
+        if (jsonString == null || "".equals(jsonString))
+            return null;
+        try {
+            JSONObject jsonObject = new JSONObject(jsonString);
+            return jsonObject;
+        } catch (JSONException e) {
+            // TODO Auto-generated catch block
+            ExceptionHandler.process(e);
+        }
+        return null;
+    }
+
+    public void setLogonLocalBranchStatus(String projectUrl, String projectName, JSONObject json) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + PROJECT_BRANCH_MANAGEMENT).getBytes());
+        store.setValue(hexKey, json.toString());
+        save();
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/conf/DefaultConfsManager.java

@@ -82,7 +82,7 @@ public abstract class DefaultConfsManager {
     }
 
     private static Object getValue(Object object, String... args) {
-        if (args.length > 0) {
+        if (args.length > 0 && object instanceof JSONObject) {
             String arg = args[0];
             Object obj = ((JSONObject) object).get(arg);
             String[] newArgs = (String[]) ArrayUtils.remove(args, 0);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IDistributionsManager.java

@@ -0,0 +1,37 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IDistributionsManager {
+
+    /**
+     * 
+     * Get all Distributions for this service manager.
+     */
+    IHDistribution[] getDistributions();
+
+    /**
+     * 
+     * Find the name of distribution.
+     */
+    IHDistribution getDistribution(String name, boolean byDisplay);
+
+    /**
+     * 
+     * Find the version of distribution.
+     */
+    IHDistributionVersion getDistributionVersion(String version, boolean byDisplay);
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDConstants.java

@@ -0,0 +1,49 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IHDConstants {
+
+    /**
+     * same as ICustomDistribution.DISTRIBUTION_NAME
+     */
+    static final String DISTRIBUTION_CUSTOM = "CUSTOM";
+
+    /**
+     * same as IHortonworksDistribution.DISTRIBUTION_NAME
+     */
+    static final String DISTRIBUTION_HORTONWORKS = "HORTONWORKS";
+
+    /**
+     * same as IAmazonEMRDistribution.DISTRIBUTION_NAME
+     */
+    static final String DISTRIBUTION_AMAZON_EMR = "AMAZON_EMR";
+
+    /**
+     * must be full class name of {@link HadoopComponent}
+     */
+    static final String SERVICE_HADOOP = "org.talend.hadoop.distribution.component.HadoopComponent";
+
+    /**
+     * must be full class name of {@link SparkComponent}
+     */
+    static final String SERVICE_SPARK = "org.talend.hadoop.distribution.component.SparkComponent";
+
+    /**
+     * must be full class name of {@link SparkStreamingComponent}
+     */
+    static final String SERVICE_SPARK_STREAMING = "org.talend.hadoop.distribution.component.SparkStreamingComponent";
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDistribution.java

@@ -0,0 +1,34 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IHDistribution {
+
+    String getName();
+
+    String getDisplayName();
+
+    IHDistributionVersion getDefaultVersion();
+
+    String[] getVersionsDisplay();
+
+    IHDistributionVersion[] getHDVersions();
+
+    IHDistributionVersion getHDVersion(String v, boolean byDisplay);
+
+    boolean useCustom();
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDistributionVersion.java

@@ -0,0 +1,25 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IHDistributionVersion {
+
+    IHDistribution getDistribution();
+
+    String getVersion();
+
+    String getDisplayVersion();
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/hive/HiveMetadataHelper.java

@@ -0,0 +1,225 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd.hive;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.talend.core.GlobalServiceRegister;
+import org.talend.core.database.EDatabaseTypeName;
+import org.talend.core.hadoop.IHadoopDistributionService;
+import org.talend.core.model.metadata.connection.hive.HiveModeInfo;
+import org.talend.core.model.metadata.connection.hive.HiveServerVersionInfo;
+import org.talend.core.runtime.hd.IDistributionsManager;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.IHDistributionVersion;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public class HiveMetadataHelper {
+
+    private static IHadoopDistributionService getHadoopDistributionService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
+            return (IHadoopDistributionService) GlobalServiceRegister.getDefault().getService(IHadoopDistributionService.class);
+        }
+        return null;
+    }
+
+    private static IDistributionsManager getHiveDistributionsManager() {
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            return hadoopDistributionService.getHiveDistributionManager();
+        }
+        return null;
+    }
+
+    public static String[] getDistributionsDisplay() {
+        List<String> distributionItems = new ArrayList<String>();
+        IDistributionsManager hiveDistributionManager = getHiveDistributionsManager();
+        if (hiveDistributionManager != null) {
+            IHDistribution[] distributions = hiveDistributionManager.getDistributions();
+            for (IHDistribution d : distributions) {
+                if (!d.useCustom()) {// custom should be add
+                    String[] distributionVersionsDisplay = getDistributionVersionsDisplay(d.getName(), false);
+                    if (distributionVersionsDisplay == null || distributionVersionsDisplay.length == 0) {
+                        continue; // if no version support, ignore it.
+                    }
+                }
+                distributionItems.add(d.getDisplayName());
+            }
+        }
+        return distributionItems.toArray(new String[distributionItems.size()]);
+    }
+
+    public static IHDistribution getDistribution(String hiveDistribution, boolean byDisplay) {
+        return getDistribution(hiveDistribution, byDisplay, false);
+    }
+
+    public static IHDistribution getDistribution(String hiveDistribution, boolean byDisplay, boolean withDefault) {
+        IDistributionsManager hiveDistributionManager = getHiveDistributionsManager();
+        if (hiveDistributionManager != null && hiveDistribution != null) {
+            IHDistribution distribution = hiveDistributionManager.getDistribution(hiveDistribution, byDisplay);
+            if (withDefault && distribution == null) {
+                IHDistribution[] distributions = hiveDistributionManager.getDistributions();
+                if (distributions.length > 0) {
+                    distribution = distributions[0];
+                }
+            }
+            return distribution;
+        }
+        return null;
+    }
+
+    public static String[] getDistributionVersionsDisplay(String hiveDistribution, boolean byDisplay) {
+        List<String> versionsItems = new ArrayList<String>();
+        IHDistribution distribution = getDistribution(hiveDistribution, byDisplay);
+        if (distribution != null) {
+            // IHDistributionVersion[] hdVersions = distribution.getHDVersions();
+            // for (IHDistributionVersion v : hdVersions) {
+            // String[] hiveModesDisplay = getHiveModesDisplay(distribution.getName(), v.getVersion(), null, false);
+            // if (hiveModesDisplay == null || hiveModesDisplay.length == 0) {
+            // continue; // if no hive mode to support, ignore this version?
+            // }
+            // String[] hiveServersDisplay = getHiveServersDisplay(distribution.getName(), v.getVersion(), false);
+            // if (hiveServersDisplay == null || hiveServersDisplay.length == 0) {
+            // continue; // if no hive server to support, ignore this version?
+            // }
+            // String displayVersion = v.getDisplayVersion();
+            // if (displayVersion != null) {
+            // versionsItems.add(displayVersion);
+            // }
+            // }
+            versionsItems.addAll(Arrays.asList(distribution.getVersionsDisplay()));
+        }
+        return versionsItems.toArray(new String[versionsItems.size()]);
+    }
+
+    public static String[] getHiveModesDisplay(String hiveDistribution, String hiveVersion, String hiveServer, boolean byDisplay) {
+        List<String> hiveModeItems = new ArrayList<String>();
+        if (doSupportEmbeddedMode(hiveDistribution, hiveVersion, byDisplay)) {
+            hiveModeItems.add(HiveModeInfo.EMBEDDED.getDisplayName());
+        }
+        if (doSupportStandaloneMode(hiveDistribution, hiveVersion, byDisplay)) {
+            HiveServerVersionInfo mode = byDisplay ? HiveServerVersionInfo.getByDisplay(hiveServer) : HiveServerVersionInfo
+                    .getByKey(hiveServer);
+            if (mode == null) {
+                String[] hiveServersDisplay = getHiveServersDisplay(hiveDistribution, hiveVersion, byDisplay);
+                if (hiveServersDisplay != null && hiveServersDisplay.length > 0) {
+                    mode = HiveServerVersionInfo.getByDisplay(hiveServersDisplay[0]);
+                }
+            }
+
+            /*
+             * According to the tHiveXXX component to set it
+             */
+            if (mode == null // add always?
+                    || mode == HiveServerVersionInfo.HIVE_SERVER_2 // server 2
+                    || (HiveServerVersionInfo.HIVE_SERVER_1 == mode && doSupportMethod(hiveDistribution, hiveVersion, byDisplay,
+                            "doSupportHive1Standalone"))) {//$NON-NLS-1$
+                hiveModeItems.add(HiveModeInfo.STANDALONE.getDisplayName());
+            }
+        }
+        return hiveModeItems.toArray(new String[0]);
+    }
+
+    public static boolean isHiveEmbeddedMode(String dbTypeDisplay, String hiveModeDisplay) {
+        if (dbTypeDisplay != null && hiveModeDisplay != null
+                && EDatabaseTypeName.HIVE.getDisplayName().equalsIgnoreCase(dbTypeDisplay)) {
+            return HiveModeInfo.getByDisplay(hiveModeDisplay) == HiveModeInfo.EMBEDDED;
+        }
+        return false;
+    }
+
+    public static String[] getHiveServersDisplay(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        List<String> hiveServerItems = new ArrayList<String>();
+        if (doSupportHive1(hiveDistribution, hiveVersion, byDisplay)) {
+            hiveServerItems.add(HiveServerVersionInfo.HIVE_SERVER_1.getDisplayName());
+        }
+        if (doSupportHive2(hiveDistribution, hiveVersion, byDisplay)) {
+            hiveServerItems.add(HiveServerVersionInfo.HIVE_SERVER_2.getDisplayName());
+        }
+        return hiveServerItems.toArray(new String[0]);
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportHive1}.
+     */
+    public static boolean doSupportHive1(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportHive1");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportHive2}.
+     */
+    public static boolean doSupportHive2(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportHive2");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportTezForHive}.
+     */
+    public static boolean doSupportTez(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportTezForHive");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportSecurity}. also, when hive service or embedded
+     */
+    public static boolean doSupportSecurity(String hiveDistribution, String hiveVersion, String hiveMode,
+            String hiveServerVersion, boolean byDisplay) {
+        HiveModeInfo hiveModeInfo = byDisplay ? HiveModeInfo.getByDisplay(hiveMode) : HiveModeInfo.get(hiveMode);
+        HiveServerVersionInfo hiveServerVersionInfo = byDisplay ? HiveServerVersionInfo.getByDisplay(hiveServerVersion)
+                : HiveServerVersionInfo.valueOf(hiveServerVersion);
+
+        boolean supportSecurity = doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportSecurity");//$NON-NLS-1$
+        if (supportSecurity
+                && (HiveServerVersionInfo.HIVE_SERVER_2 == hiveServerVersionInfo || hiveModeInfo == HiveModeInfo.EMBEDDED)) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportEmbeddedMode}.
+     */
+    public static boolean doSupportEmbeddedMode(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportEmbeddedMode");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportStandaloneMode}.
+     */
+    public static boolean doSupportStandaloneMode(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportStandaloneMode");//$NON-NLS-1$
+    }
+
+    public static boolean doSupportMethod(String hiveDistribution, String hiveVersion, boolean byDisplay, String supportMethodName) {
+        IHDistribution distribution = getDistribution(hiveDistribution, byDisplay);
+        if (distribution != null) {
+            IHDistributionVersion version = distribution.getHDVersion(hiveVersion, byDisplay);
+            IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+
+            if (version != null && hadoopDistributionService != null) {
+                try {
+                    return hadoopDistributionService.doSupportMethod(version, supportMethodName);
+                } catch (Exception e) {
+                    // ignore if NoSuchMethodException
+                }
+            }
+        }
+        return false;
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenUrlHelper.java

@@ -208,6 +208,9 @@ public class MavenUrlHelper {
         if (parseMvnUrl == null) {
             return null;
         }
+        if (!MavenConstants.DEFAULT_LIB_GROUP_ID.equals(parseMvnUrl.getGroupId())) {
+            return mavenUri; // snapshot url same as maven url
+        }
 
         return generateMvnUrl(parseMvnUrl.getGroupId(), parseMvnUrl.getArtifactId(), parseMvnUrl.getVersion()
                 + MavenConstants.SNAPSHOT, parseMvnUrl.getType(), parseMvnUrl.getClassifier());
@@ -219,6 +222,9 @@ public class MavenUrlHelper {
         if (parseMvnUrl == null) {
             return null;
         }
+        if (!MavenConstants.DEFAULT_LIB_GROUP_ID.equals(parseMvnUrl.getGroupId())) {
+            return snapshotMvnuri; // snapshot url same as maven url
+        }
         String snapshotVerstion = parseMvnUrl.getVersion();
         String verstion = snapshotVerstion.substring(0, snapshotVerstion.indexOf(MavenConstants.SNAPSHOT));
         return generateMvnUrl(parseMvnUrl.getGroupId(), parseMvnUrl.getArtifactId(), verstion, parseMvnUrl.getType(),

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/JobInfoProperties.java

@@ -47,6 +47,8 @@ public class JobInfoProperties extends Properties {
 
     public static final String JOB_NAME = "job"; //$NON-NLS-1$
 
+    public static final String JOB_TYPE = "jobType"; //$NON-NLS-1$
+
     public static final String JOB_VERSION = "jobVersion"; //$NON-NLS-1$
 
     public static final String DATE = "date"; //$NON-NLS-1$
@@ -94,6 +96,7 @@ public class JobInfoProperties extends Properties {
 
         setProperty(JOB_ID, jobInfo.getJobId());
         setProperty(JOB_NAME, jobInfo.getJobName());
+        setProperty(JOB_TYPE, processItem.getProcess().getJobType());
         setProperty(JOB_VERSION, jobInfo.getJobVersion());
         setProperty(CONTEXT_NAME, this.contextName);
         setProperty(DATE, DATAFORMAT.format(new Date()));

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/ComponentServiceWithValueEvaluator.java

@@ -23,10 +23,8 @@ import org.talend.components.api.service.ComponentService;
 import org.talend.components.api.wizard.ComponentWizard;
 import org.talend.components.api.wizard.ComponentWizardDefinition;
 import org.talend.components.api.wizard.WizardImageType;
-import org.talend.daikon.NamedThing;
 import org.talend.daikon.properties.PropertyValueEvaluator;
 import org.talend.daikon.properties.service.Repository;
-import org.talend.daikon.schema.Schema;
 
 /**
  * created by ycbai on 2016年2月4日 Detailled comment
@@ -44,8 +42,9 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
     }
 
     @Override
-    public String storeProperties(ComponentProperties properties, String name, String repositoryLocation, Schema schema) {
-        return service.storeProperties(properties, name, repositoryLocation, schema);
+    public String storeProperties(ComponentProperties properties, String name, String repositoryLocation,
+            String schemaPropertyName) {
+        return service.storeProperties(properties, name, repositoryLocation, schemaPropertyName);
     }
 
     @Override
@@ -90,11 +89,6 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
         return service.beforePropertyPresent(propName, properties);
     }
 
-    @Override
-    public ComponentProperties commitFormValues(ComponentProperties properties, String formName) {
-        return service.commitFormValues(properties, formName);
-    }
-
     @Override
     public Set<String> getAllComponentNames() {
         return service.getAllComponentNames();
@@ -136,7 +130,7 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
     }
 
     @Override
-    public List<ComponentDefinition> getPossibleComponents(ComponentProperties properties) throws Throwable {
+    public List<ComponentDefinition> getPossibleComponents(ComponentProperties... properties) throws Throwable {
         return service.getPossibleComponents(properties);
     }
 
@@ -166,16 +160,23 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
         return service.validateProperty(propName, properties);
     }
 
-    private void setupChildrenPropertiesContextValueEvaluator(ComponentProperties compProperties,
-            PropertyValueEvaluator valueEvaluator) {
-        List<NamedThing> properties = compProperties.getProperties();
-        for (NamedThing property : properties) {
-            if (property instanceof ComponentProperties) {
-                ComponentProperties childComponentProperties = (ComponentProperties) property;
-                childComponentProperties.setValueEvaluator(valueEvaluator);
-                setupChildrenPropertiesContextValueEvaluator(childComponentProperties, valueEvaluator);
-            }
-        }
+    @Deprecated
+    public ComponentProperties commitFormValues(ComponentProperties properties, String formName) {
+        // to delete
+        return null;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see
+     * org.talend.daikon.properties.service.PropertiesService#cancelFormValues(org.talend.daikon.properties.Properties,
+     * java.lang.String)
+     */
+    @Override
+    public ComponentProperties cancelFormValues(ComponentProperties arg0, String arg1) {
+        // to implement
+        return null;
     }
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericService.java

@@ -0,0 +1,42 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2015 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.services;
+
+import java.util.List;
+import java.util.Map;
+
+import org.talend.core.IService;
+import org.talend.core.model.process.IElementParameter;
+
+/**
+ * created by ycbai on 2016年3月24日 Detailled comment
+ *
+ */
+public interface IGenericService extends IService {
+
+    /**
+     * Call method <code>callBeforeActivate()</code> of <code>parameter</code>.
+     * 
+     * 
+     * @param parameter
+     * @return
+     */
+    public boolean callBeforeActivate(IElementParameter parameter);
+    
+    /**
+     * Get all installed generic components information.
+     * 
+     */
+    public List<Map<String, String>> getAllGenericComponentsInfo();
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericWizardService.java

@@ -16,7 +16,9 @@ import java.util.List;
 
 import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.widgets.Composite;
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.IService;
+import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.EComponentCategory;
@@ -93,4 +95,24 @@ public interface IGenericWizardService extends IService {
      */
     public Composite creatDynamicComposite(Composite composite, Element element, EComponentCategory sectionCategory,
             boolean isCompactView);
+
+    /**
+     * Update component schema for metadata table of repository.
+     * 
+     * @param metadataTable
+     */
+    public void updateComponentSchema(MetadataTable metadataTable);
+
+    /**
+     * Update component schema for node metadata table of node.
+     * 
+     * @param componentProperties
+     * @param schemaPropertyName
+     * @param metadataTable
+     */
+    public void updateComponentSchema(ComponentProperties componentProperties, String schemaPropertyName,
+            IMetadataTable metadataTable);
+
+    public List<ComponentProperties> getAllComponentProperties(Connection connection);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/IStormProcessService.java

@@ -17,9 +17,13 @@ import java.util.List;
 import org.eclipse.ui.IEditorPart;
 import org.talend.commons.ui.runtime.image.IImage;
 import org.talend.core.IService;
+import org.talend.core.model.components.IComponentsHandler;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.repository.IRepositoryEditorInput;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.repository.model.IRepositoryNode;
 import org.talend.repository.model.nodes.IProjectRepositoryNode;
@@ -51,4 +55,13 @@ public interface IStormProcessService extends IService {
 
     public IImage getStormWizardIcon(String framework);
 
+    public IComponentsHandler getSparkStreamingComponentsHandler();
+
+    public boolean isSparkStreaming(Item item);
+
+    public IRepositoryEditorInput createTestCaseStormProcessEditorInput(ProcessItem processItem, boolean load,
+            String originalJobID, List<INode> testNodes, Boolean lastVersion, Boolean readonly);
+
+    public IProcess2 createBigdataProcess(Item item);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/IJobletProviderService.java

@@ -20,6 +20,7 @@ import org.eclipse.ui.IEditorPart;
 import org.eclipse.ui.IWorkbenchPage;
 import org.eclipse.ui.IWorkbenchPart;
 import org.eclipse.ui.part.MultiPageEditorPart;
+import org.talend.commons.exception.PersistenceException;
 import org.talend.core.IService;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.metadata.IMetadataTable;
@@ -34,6 +35,7 @@ import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.IRepositoryEditorInput;
 import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 
@@ -132,4 +134,7 @@ public interface IJobletProviderService extends IService {
 
     public void clearJobletComponent();
 
+    public IRepositoryEditorInput getJobletProcessEditorInput(JobletProcessItem processItem, boolean load, boolean lastVersion,
+            boolean readOnly) throws PersistenceException;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/ITestContainerCoreService.java

@@ -22,4 +22,6 @@ import org.talend.core.model.properties.Item;
 public interface ITestContainerCoreService extends IService {
 
     public boolean isSpark(Item item);
+
+    public boolean isSparkStreaming(Item item);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/branding/IBrandingConfiguration.java

@@ -31,8 +31,6 @@ public interface IBrandingConfiguration extends IActionBarHelper {
 
     public static final String PERSPECTIVE_MDM_ID = ProductUtils.PERSPECTIVE_MDM_ID;
 
-    public static final String PERSPECTIVE_CAMEL_ID = ProductUtils.PERSPECTIVE_CAMEL_ID;
-
     /**
      * 
      * init perspective.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/images/RepositoryImageProvider.java

@@ -150,6 +150,8 @@ public class RepositoryImageProvider {
                 || type == ERepositoryObjectType.SYSTEM_INDICATORS_PHONENUMBER_STATISTICS
                 || type == ERepositoryObjectType.SYSTEM_INDICATORS_FRAUDDETECTION) {
             return ECoreImage.TDQ_INDICATOR_ICON;
+        } else if (type == ERepositoryObjectType.TDQ_DATA_PROFILING) {
+            return ECoreImage.TDQ_DATA_PROFILING_ICON;
         } else {
             IImage image = null;
             for (IRepositoryContentHandler handler : RepositoryContentManager.getHandlers()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/utils/ProductUtils.java

@@ -26,8 +26,6 @@ public final class ProductUtils {
      */
     public static final String PERSPECTIVE_DI_ID = "org.talend.rcp.perspective"; //$NON-NLS-1$
 
-    public static final String PERSPECTIVE_CAMEL_ID = "org.talend.camel.perspective"; //$NON-NLS-1$
-
     public static final String PERSPECTIVE_DQ_ID = "org.talend.dataprofiler.DataProfilingPerspective"; //$NON-NLS-1$
 
     public static final String PERSPECTIVE_MDM_ID = "org.talend.mdm.perspective"; //$NON-NLS-1$
@@ -54,7 +52,6 @@ public final class ProductUtils {
     static {
         productPerspectiveMap.put(PROD_DI, PERSPECTIVE_DI_ID);
         productPerspectiveMap.put(PROD_DQ, PERSPECTIVE_DQ_ID);
-        productPerspectiveMap.put(PROD_CAMEL, PERSPECTIVE_CAMEL_ID);
         productPerspectiveMap.put(PROD_MDM, PERSPECTIVE_MDM_ID);
         productPerspectiveMap.put(PROD_MAPPING, PERSPECTIVE_MAPPING_ID);
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -161,6 +161,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         syncRoutine(routineItem, copyToTemp, false);
     }
 
+    @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -43,6 +43,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException;
 
+    void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceToUpdate) throws SystemException;
+
     IFile getFile(Item item) throws SystemException;
 
     IFile getRoutinesFile(Item routineItem) throws SystemException;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -118,6 +118,8 @@ public interface IRunProcessService extends IService {
     public void setDelegateService(IRunProcessService delegateService);
 
     public void updateLibraries(Set<ModuleNeeded> jobModuleList, IProcess process);
+    
+    public void updateLibraries(Set<ModuleNeeded> jobModuleList, IProcess process, Set<ModuleNeeded> alreadyRetrievedModules);
 
     public void refreshView();
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -46,6 +46,9 @@ import org.talend.repository.model.IRepositoryNode;
 import org.talend.repository.model.nodes.IProjectRepositoryNode;
 import org.talend.repository.ui.views.IRepositoryView;
 
+import us.monoid.json.JSONException;
+import us.monoid.json.JSONObject;
+
 /**
  * ggu class global comment. Detailled comment
  */
@@ -576,4 +579,23 @@ public final class ProjectManager {
     public void clearFolderCache() {
         foldersMap.clear();
     }
+    
+    /**
+     * Returns the type of project (local / svn / git).
+     * @param project
+     * @return
+     */
+    public String getProjectType(Project project) {
+        String projectType = "local";
+        if (!currentProject.getEmfProject().isLocal()) {
+            JSONObject projectInfo;
+            try {
+                projectInfo = new JSONObject(currentProject.getEmfProject().getUrl());
+                projectType = projectInfo.getString("storage");
+            } catch (JSONException e) {
+                // nothing to do, consider local by default
+            }
+        }
+        return projectType;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryConstants.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.repository.model;
 
+import org.eclipse.core.runtime.Path;
 import org.talend.core.model.repository.ERepositoryObjectType;
 
 /**
@@ -143,14 +144,16 @@ public class RepositoryConstants {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.GENERATED.toString());
+        return ERepositoryObjectType.GENERATED != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.GENERATED.getFolder()).lastSegment());
     }
 
     public static boolean isJobsFolder(String label) {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.JOBS.toString());
+        return ERepositoryObjectType.JOB_DOC != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment());
     }
 
     /**
@@ -163,6 +166,7 @@ public class RepositoryConstants {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.JOBLETS.toString());
+        return ERepositoryObjectType.JOBLET_DOC != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment());
     }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryNodeUtilities.java

@@ -31,6 +31,8 @@ import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.hadoop.IHadoopClusterService;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.model.properties.ConnectionItem;
 import org.talend.core.model.properties.FolderItem;
 import org.talend.core.model.properties.FolderType;
@@ -41,11 +43,13 @@ import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.model.repository.ISubRepositoryObject;
 import org.talend.core.model.utils.RepositoryManagerHelper;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.i18n.Messages;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.ui.ITestContainerProviderService;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IRepositoryNode.ENodeType;
 import org.talend.repository.model.IRepositoryNode.EProperties;
@@ -75,8 +79,8 @@ public class RepositoryNodeUtilities {
         if (node.isBin()) {
             return new Path(""); //$NON-NLS-1$
         }
-        if ((node.getType() == ENodeType.STABLE_SYSTEM_FOLDER && node.getContentType() != ERepositoryObjectType.JOBS && node
-                .getContentType() != ERepositoryObjectType.JOBLETS) || node.getType() == ENodeType.SYSTEM_FOLDER) {
+        if ((node.getType() == ENodeType.STABLE_SYSTEM_FOLDER && node.getContentType() != ERepositoryObjectType.JOB_DOC && node
+                .getContentType() != ERepositoryObjectType.JOBLET_DOC) || node.getType() == ENodeType.SYSTEM_FOLDER) {
             return new Path(""); //$NON-NLS-1$
         }
         if (node.getType() == ENodeType.SIMPLE_FOLDER) {
@@ -87,11 +91,11 @@ public class RepositoryNodeUtilities {
         String label = node.getLabel();
         // checks if node is under Documentations/Generatated/Jobs
         if (node.getType() == ENodeType.STABLE_SYSTEM_FOLDER
-                && (node.getContentType() == ERepositoryObjectType.JOBS || node.getContentType() == ERepositoryObjectType.JOBLETS)) {
+                && (node.getContentType() == ERepositoryObjectType.JOB_DOC || node.getContentType() == ERepositoryObjectType.JOBLET_DOC)) {
 
             Object nodeLabel = node.getProperties(EProperties.LABEL);
 
-            if (nodeLabel == ERepositoryObjectType.JOBS) {
+            if (nodeLabel == ERepositoryObjectType.JOB_DOC) {
                 return new Path(""); //$NON-NLS-1$
             } else {
                 return getPath(node.getParent()).append(label);
@@ -463,7 +467,7 @@ public class RepositoryNodeUtilities {
         if (rootContextType == ERepositoryObjectType.REFERENCED_PROJECTS || rootNode.getType() == ENodeType.REFERENCED_PROJECT) {
             expandParentNode(view, rootNode);
         }
-        if (rootContextType != null) {
+        if (rootContextType != null && curType != null) {
 
             ERepositoryObjectType tmpType = null;
             if (curType == ERepositoryObjectType.METADATA_CON_TABLE || curType == ERepositoryObjectType.METADATA_CON_VIEW
@@ -490,8 +494,8 @@ public class RepositoryNodeUtilities {
                 tmpType = ERepositoryObjectType.METADATA;
             } else if (curType == ERepositoryObjectType.ROUTINES || curType == ERepositoryObjectType.SNIPPETS) {
                 tmpType = ERepositoryObjectType.ROUTINES;
-            } else if (curType == ERepositoryObjectType.DOCUMENTATION || curType == ERepositoryObjectType.JOB_DOC
-                    || curType == ERepositoryObjectType.JOBLET_DOC) {
+            } else if (curType.equals(ERepositoryObjectType.DOCUMENTATION) || curType.equals(ERepositoryObjectType.JOB_DOC)
+                    || curType.equals(ERepositoryObjectType.JOBLET_DOC)) {
                 tmpType = ERepositoryObjectType.DOCUMENTATION;
             }
 
@@ -570,15 +574,49 @@ public class RepositoryNodeUtilities {
 
     private static RepositoryNode getSAPSchemaFromConnection(RepositoryNode realNode, String name) {
         String[] values = name.split(" - "); //$NON-NLS-1$
-        if (values.length != 3) {
-            return null;
+        String metadataName = null;
+        String innerIOType = null;
+        if (values.length >= 2) {
+            metadataName = values[1];
         }
-        String metadataName = values[2];
-        String repositoryId = name.substring(0, name.lastIndexOf(" - ")); //$NON-NLS-1$
-        RepositoryNode functionNode = getSAPFunctionFromConnection(repositoryId);
-        for (IRepositoryNode node : functionNode.getChildren()) {
-            if (metadataName.equals(node.getProperties(EProperties.LABEL))) {
-                return (RepositoryNode) node;
+        if (values.length == 3) {
+            innerIOType = values[2];
+        }
+        if (innerIOType == null) {
+            RepositoryNode functionNode = getSAPFunctionFromConnection(realNode, name);
+            if (functionNode != null) {
+                return functionNode;
+            }
+        }
+
+        if (innerIOType == null || SAPBWTableHelper.isInnerIOType(innerIOType)) {
+            for (IRepositoryNode node : realNode.getChildren()) {
+                if (node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapTable")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapDataSource")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapDSO")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapInfoCube")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapInfoObject"))) { //$NON-NLS-1$
+                    for (IRepositoryNode metadataNode : node.getChildren()) {
+                        IRepositoryViewObject metadataObject = metadataNode.getObject();
+                        if (metadataObject instanceof ISubRepositoryObject) {
+                            AbstractMetadataObject metadataTable = ((ISubRepositoryObject) metadataObject)
+                                    .getAbstractMetadataObject();
+                            if (metadataName.equals(metadataTable.getLabel())) {
+                                String innerIOTypeInMetaTable = null;
+                                if (metadataTable instanceof SAPBWTable) {
+                                    innerIOTypeInMetaTable = ((SAPBWTable) metadataTable).getInnerIOType();
+                                }
+                                if (innerIOType == null) {
+                                    return (RepositoryNode) metadataNode;
+                                } else {
+                                    if (innerIOType.equals(innerIOTypeInMetaTable)) {
+                                        return (RepositoryNode) metadataNode;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
             }
         }
         return null;
@@ -599,19 +637,40 @@ public class RepositoryNodeUtilities {
     }
 
     public static RepositoryNode getSAPFunctionFromConnection(String id) {
+        return getSAPFunctionFromConnection(null, id);
+    }
+
+    public static RepositoryNode getSAPFunctionFromConnection(RepositoryNode realNode, String id) {
         String[] values = id.split(" - "); //$NON-NLS-1$
         String repositoryID = values[0];
         String functionName = values[1];
 
         try {
-            final RepositoryNode realNode = getRepositoryNode(repositoryID);
+            if (realNode == null) {
+                realNode = getRepositoryNode(repositoryID);
+            }
             if (realNode.getObject() != null) {
                 if (ERepositoryObjectType.METADATA_SAPCONNECTIONS.equals(realNode.getObject().getRepositoryObjectType())) {
                     for (IRepositoryNode node : realNode.getChildren()) {
-                        if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapFunction").equals(node.getLabel())) { //$NON-NLS-1$
+                        if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapBapi").equals(node.getLabel())) {
                             for (IRepositoryNode function : node.getChildren()) {
-                                if (functionName.equals(function.getProperties(EProperties.LABEL))) {
-                                    return (RepositoryNode) function;
+                                List<IRepositoryNode> inputAndOutputs = function.getChildren();
+                                for (IRepositoryNode inputAndOutput : inputAndOutputs) {
+                                    if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapBapi.input").equals(
+                                            inputAndOutput.getLabel())) {
+                                        for (IRepositoryNode input : inputAndOutput.getChildren()) {
+                                            if (input.getProperties(EProperties.LABEL).equals(functionName)) {
+                                                return (RepositoryNode) input;
+                                            }
+                                        }
+                                    } else if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapBapi.output")
+                                            .equals(inputAndOutput.getLabel())) {
+                                        for (IRepositoryNode output : inputAndOutput.getChildren()) {
+                                            if (output.getLabel().equals(functionName)) {
+                                                return (RepositoryNode) output;
+                                            }
+                                        }
+                                    }
                                 }
                             }
                         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/component/ComponentPaletteUtilities.java

@@ -108,7 +108,7 @@ public class ComponentPaletteUtilities {
                 }
 
                 if (extraPaletteEntry == null || extraPaletteEntry.size() == 0) {
-                    extraPaletteEntry = designerCoreUIService.createJobletEtnry();
+                    extraPaletteEntry = designerCoreUIService.createPaletteEtnry();
                 }
             }
         });
@@ -156,7 +156,7 @@ public class ComponentPaletteUtilities {
 
                 @Override
                 public void run() {
-                    extraPaletteEntry = designerCoreUIService.createJobletEtnry();
+                    extraPaletteEntry = designerCoreUIService.createPaletteEtnry();
                 }
             });
         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/composite/ElementsSelectionComposite.java

@@ -28,6 +28,7 @@ import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.layout.GridData;
 import org.eclipse.swt.layout.GridLayout;
 import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
 import org.talend.core.ui.advanced.composite.FilteredCheckboxTree;
 import org.talend.core.ui.advanced.composite.PatternFilter;
 
@@ -75,6 +76,22 @@ public class ElementsSelectionComposite<T> extends Composite {
         treeGridData.heightHint = 270;
     }
 
+    @Override
+    public void setEnabled(boolean enabled) {
+        super.setEnabled(enabled);
+        updateChildrenEnableStatus(this, enabled);
+    }
+
+    private void updateChildrenEnableStatus(Composite comp, boolean enabled) {
+        Control[] children = comp.getChildren();
+        for (Control control : children) {
+            control.setEnabled(enabled);
+            if (control instanceof Composite) {
+                updateChildrenEnableStatus(((Composite) control), enabled);
+            }
+        }
+    }
+
     protected void doSelectionChanged() {
         // Do nothing by default.
     }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/view/Contexts.java

@@ -42,7 +42,7 @@ public class Contexts {
             refreshView(cxtView);
         }
     }
-    
+
     public static void switchToCurContextsView(final IWorkbenchPart part) {
         final AbstractContextView cxtView = getViewWithPerspectiveIDs();
         Display.getDefault().asyncExec(new Runnable() {
@@ -60,7 +60,6 @@ public class Contexts {
     private static AbstractContextView getViewWithPerspectiveIDs() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         return getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
     }
 
@@ -91,7 +90,6 @@ public class Contexts {
     public static void refreshContextsView() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         AbstractContextView cxtView = getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
         if (cxtView != null) {
             refreshView(cxtView);
@@ -189,7 +187,6 @@ public class Contexts {
     public static void clearAll() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         AbstractContextView cxtView = getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
         refreshView(cxtView);
         if (cxtView != null) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/images/CoreImageProvider.java

@@ -29,6 +29,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.ui.component.ComponentsFactoryProvider;
 import org.talend.designer.core.IDesignerCoreService;
+import org.talend.utils.string.MD5;
 
 /**
  * amaumont class global comment. Detailled comment <br/>
@@ -80,17 +81,19 @@ public class CoreImageProvider {
         return getIcon(itemType);
     }
 
-    private static Map<ImageDescriptor, Image> componentCachedImages = new HashMap<ImageDescriptor, Image>();
+    private static Map<String, Image> componentCachedImages = new HashMap<String, Image>();
 
-    public static Image getComponentImageFromDesc(ImageDescriptor imageDescriptor) {
-        Image image = null;
-        image = componentCachedImages.get(imageDescriptor);
-        if (image == null || image.isDisposed()) {
-            image = imageDescriptor.createImage();
-            componentCachedImages.put(imageDescriptor, image);
-        }
-        return image;
-    }
+	public static Image getComponentImageFromDesc(
+			ImageDescriptor imageDescriptor) {
+		String md5Desc = MD5.getMD5(imageDescriptor.getImageData().data);
+		Image image = componentCachedImages.get(md5Desc);
+
+		if (image == null || image.isDisposed()) {
+			image = imageDescriptor.createImage();
+			componentCachedImages.put(md5Desc, image);
+		}
+		return image;
+	}
 
     public static Image getComponentIcon(IComponent component, ICON_SIZE iconSize) {
         if (component != null && iconSize != null) {
@@ -130,21 +133,24 @@ public class CoreImageProvider {
         if (name != null && !name.equals("")) { //$NON-NLS-1$
             for (IComponent component : ComponentsFactoryProvider.getInstance().getComponents()) {
                 if (name.equals(component.getName())) {
-                    Image image = componentCachedImages.get(component.getIcon16());
+                	String md5Desc16 = MD5.getMD5(component.getIcon16().getImageData().data);
+                    Image image = componentCachedImages.get(md5Desc16);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon16());
-                    image = componentCachedImages.get(component.getIcon24());
+                    componentCachedImages.remove(md5Desc16);
+                    String md5Desc24 = MD5.getMD5(component.getIcon24().getImageData().data);
+                    image = componentCachedImages.get(md5Desc24);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon24());
-                    image = componentCachedImages.get(component.getIcon32());
+                    componentCachedImages.remove(md5Desc24);
+                    String md5Desc32 = MD5.getMD5(component.getIcon32().getImageData().data);
+                    image = componentCachedImages.get(md5Desc32);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon32());
+                    componentCachedImages.remove(md5Desc32);
                 }
             }
         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/metadata/dialog/MetadataDialog.java

@@ -314,7 +314,7 @@ public class MetadataDialog extends Dialog {
         // TDI-29264:improve here for adapt two or more schemas with different mode,such as tHiveCreateTable etc
         boolean nodeModeFlag = false;
         for (IElementParameter param : node.getElementParameters()) {
-            if (param.getFieldType() == EParameterFieldType.SCHEMA_TYPE
+            if ((param.getFieldType() == EParameterFieldType.SCHEMA_TYPE || param.getFieldType() == EParameterFieldType.SCHEMA_REFERENCE)
                     && (param.getContext() == null || param.getContext().equals(currentTable.getAttachedConnector()))) {
                 IElementParameter schemaParam = param.getChildParameters().get("SCHEMA_TYPE");
                 if (schemaParam.getValue() != null) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/services/IDesignerCoreUIService.java

@@ -51,7 +51,7 @@ public interface IDesignerCoreUIService extends IService {
 
     void removePreferenceStorePropertyChangeListener(IPropertyChangeListener listener);
 
-    List<PaletteEntry> createJobletEtnry();
+    List<PaletteEntry> createPaletteEtnry();
 
     boolean executeCommand(IGEFProcess process, Command cmd);
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/AbstractTokenCollector.java

@@ -19,9 +19,7 @@ import us.monoid.json.JSONObject;
  */
 public abstract class AbstractTokenCollector implements ITokenCollector {
 
-    protected static final TokenKey TOKEN_STUDIO = new TokenKey("tokenStudio"); //$NON-NLS-1$
-
-    protected static final TokenKey PROPERTIES = new TokenKey("properties"); //$NON-NLS-1$
+    protected static final TokenKey PROJECTS_REPOSITORY = new TokenKey("projects.repository"); //$NON-NLS-1$
 
     public AbstractTokenCollector() {
         //
@@ -33,25 +31,12 @@ public abstract class AbstractTokenCollector implements ITokenCollector {
 
     public JSONObject collect() throws Exception {
         JSONObject result = new JSONObject();
-
-        // tokenStudio
-        JSONObject tokenStudioObject = new JSONObject();
-        result.put(TOKEN_STUDIO.getKey(), tokenStudioObject);
-        collectTokenStudio(tokenStudioObject);
-
-        // properties
-        JSONObject propertiesObject = new JSONObject();
-        tokenStudioObject.put(PROPERTIES.getKey(), propertiesObject);
-        collectProperties(propertiesObject);
+        // default is empty
 
         return result;
     }
-
-    protected void collectTokenStudio(JSONObject tokenStudioObject) throws Exception {
-        // will do something
-    }
-
+    
+    @Deprecated
     protected void collectProperties(JSONObject propertiesObject) throws Exception {
-        // will do something
     }
 }