TDI-35231: Performance of tfileOutputMSXML component

https://jira.talendforge.org/browse/TDI-35231

main/features/org.talend.commons.feature/feature.xml

@@ -109,4 +109,18 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="jackson-core-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="jackson-mapper-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

main/features/org.talend.utils.feature/feature.xml

@@ -114,4 +114,38 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="jackson-core-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="jackson-mapper-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.apache.servicemix.bundles.avro"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"/>
+
+   <plugin
+         id="com.thoughtworks.paranamer"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.apache.commons.compress"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

main/plugins/org.talend.commons.runtime/src/org/talend/commons/runtime/model/components/IComponentConstants.java

@@ -24,4 +24,9 @@ public interface IComponentConstants {
     public static final String JOBLET_NAME_CHANGED = "joblet name changed"; //$NON-NLS-1$
 
     public static final String JOBLET_SCHEMA_CHANGED = "joblet schema changed"; //$NON-NLS-1$
+
+    public static final String COMPONENT_PROPERTIES_TAG = "component.json.serialized"; //$NON-NLS-1$
+
+    public static final String COMPONENT_SCHEMA_TAG = "component.property.schema"; //$NON-NLS-1$
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/NetworkUtil.java

@@ -39,70 +39,6 @@ public class NetworkUtil {
 
     private static final String TALEND_DISABLE_INTERNET = "talend.disable.internet";//$NON-NLS-1$
 
-    private final static Set<String> getMacAddressList() throws IOException {
-        final Set<String> macAddressList = new HashSet<String>();
-
-        final String os = System.getProperty("os.name"); //$NON-NLS-1$
-
-        final String[] command;
-        if (os.startsWith("Windows")) { //$NON-NLS-1$
-            command = windowsCommand;
-        } else if (os.startsWith("Linux")) { //$NON-NLS-1$
-            command = linuxCommand;
-        } else {
-            throw new IOException("Unknown operating system: " + os); //$NON-NLS-1$
-        }
-
-        final Process process = Runtime.getRuntime().exec(command);
-        // Discard the stderr
-        new Thread() {
-
-            @Override
-            public void run() {
-                try {
-                    InputStream errorStream = process.getErrorStream();
-                    while (errorStream.read() != -1) {
-                    }
-                    errorStream.close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }.start();
-
-        // Extract the MAC addresses from stdout
-        BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
-        for (String line = null; (line = reader.readLine()) != null;) {
-            Matcher matcher = macPattern.matcher(line);
-            if (matcher.matches()) {
-                // macAddressList.add(matcher.group(1));
-                macAddressList.add(matcher.group(1).replaceAll("[-:]", "")); //$NON-NLS-1$ //$NON-NLS-2$
-            }
-        }
-        reader.close();
-        return macAddressList;
-    }
-
-    /**
-     * Note： should rename as "getFullMacAddresses" later since it takes in fact the concatenation of all mac address.
-     * 
-     * @return
-     */
-    public static String getMacAddress() {
-        try {
-            Set<String> addressList = getMacAddressList();
-            StringBuffer sb = new StringBuffer();
-            // remove empty mac address
-            addressList.remove("0000000000E0"); //$NON-NLS-1$
-            for (String curMacAddress : addressList) {
-                sb.append(curMacAddress);
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            return null;
-        }
-    }
-
     public static boolean isNetworkValid() {
         String disableInternet = System.getProperty(TALEND_DISABLE_INTERNET);
         if ("true".equals(disableInternet)) { //$NON-NLS-1$
@@ -143,12 +79,4 @@ public class NetworkUtil {
         return null;
     }
 
-    public final static void main(String[] args) {
-        try {
-            System.out.println("  MAC Address: " + getMacAddress()); //$NON-NLS-1$
-        } catch (Throwable t) {
-            t.printStackTrace();
-        }
-    }
-
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/i18n/messages.properties

@@ -163,6 +163,10 @@ ProjectRepositoryNode.sapFunctions.inputSchema=Input
 ProjectRepositoryNode.sapFunctions.outputSchema=Output
 ProjectRepositoryNode.sapIDocs=SAP iDocs
 ProjectRepositoryNode.sapTables=SAP Tables
+ProjectRepositoryNode.sapBWDataSource=SAP DataSource
+ProjectRepositoryNode.sapBWDataStoreObject=SAP DSO
+ProjectRepositoryNode.sapBWInfoCube=SAP InfoCube
+ProjectRepositoryNode.sapBWInfoObject=SAP InfoObject
 
 RepositoryDropAdapter_copyingItems=Copying items...
 RepositoryDropAdapter_errorMsg=This directory contains the locked item,it can not be moved now

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/IRepositoryFactory.java

@@ -38,6 +38,7 @@ import org.talend.core.model.repository.IRepositoryWorkUnitListener;
 import org.talend.core.model.repository.LockInfo;
 import org.talend.core.repository.utils.XmiResourceManager;
 import org.talend.repository.RepositoryWorkUnit;
+import org.talend.utils.json.JSONArray;
 
 /**
  * Defines all methods that a repository provider plug-in must provides to client.<br/>
@@ -414,4 +415,6 @@ public interface IRepositoryFactory {
     public void setStorage(String storage);
 
     public String getStorage();
+
+    public JSONArray getAllRemoteLocks();
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProjectRepositoryNode.java

@@ -39,7 +39,6 @@ import org.talend.commons.utils.data.container.Container;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.database.EDatabaseTypeName;
-import org.talend.core.model.genhtml.IHTMLDocConstants;
 import org.talend.core.model.metadata.MetadataManager;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
@@ -47,6 +46,7 @@ import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.metadata.builder.connection.QueriesConnection;
 import org.talend.core.model.metadata.builder.connection.Query;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.model.metadata.builder.connection.SAPConnection;
 import org.talend.core.model.metadata.builder.connection.SAPFunctionUnit;
 import org.talend.core.model.metadata.builder.connection.SAPIDocUnit;
@@ -92,9 +92,9 @@ import org.talend.core.ui.ICDCProviderService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.core.ui.branding.IBrandingService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.TableHelper;
-import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.BinRepositoryNode;
@@ -980,51 +980,6 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         return false;
     }
 
-    @SuppressWarnings("rawtypes")
-    private void convertDocumentation(org.talend.core.model.general.Project newProject, Container generatedContainer,
-            RepositoryNode parent, ERepositoryObjectType type) {
-        // for folder Documentation/generated
-        // RepositoryNode generatedFolder = getRootRepositoryNode(ERepositoryObjectType.GENERATED);
-
-        // for folder Documentation/generated/jobs
-        convertDocumentation(newProject, generatedContainer, parent, type, ERepositoryObjectType.JOBS,
-                ERepositoryObjectType.JOB_DOC);
-
-        // for folder Documentation/generated/joblets
-        convertDocumentation(newProject, generatedContainer, parent, type, ERepositoryObjectType.JOBLETS,
-                ERepositoryObjectType.JOBLET_DOC);
-
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ICamelDesignerCoreService.class)) {
-            ICamelDesignerCoreService service = (ICamelDesignerCoreService) GlobalServiceRegister.getDefault().getService(
-                    ICamelDesignerCoreService.class);
-            if (service.getRouteDocsType() != null && service.getRouteDocType() != null) {
-                convertDocumentation(newProject, generatedContainer, parent, type, service.getRouteDocsType(),
-                        service.getRouteDocType());
-            }
-        }
-
-    }
-
-    @SuppressWarnings("rawtypes")
-    private void convertDocumentation(org.talend.core.model.general.Project newProject, Container generatedContainer,
-            RepositoryNode parent, ERepositoryObjectType type, ERepositoryObjectType parentDocType, ERepositoryObjectType docType) {
-        RepositoryNode docsFolder = getRootRepositoryNode(parentDocType);
-
-        Container docsNode = null;
-        for (Object object : generatedContainer.getSubContainer()) {
-            if (((Container) object).getLabel().equalsIgnoreCase(parentDocType.name().toLowerCase())) {
-                docsNode = (Container) object;
-                break;
-            }
-        }
-
-        // get the files under generated/nodes.
-        if (docsNode != null) {
-            convert(newProject, docsNode, docsFolder, docType);
-        }
-
-    }
-
     private RepositoryNode getSQLPatternNode(String parentLabel, String label) {
         if (getMergeRefProject()) {
             List<IRepositoryNode> sqlChildren = getRootRepositoryNode(ERepositoryObjectType.SQLPATTERNS).getChildren();
@@ -1093,7 +1048,7 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             Folder oFolder = new Folder((Property) container.getProperty(), type);
             boolean found = false;
             // add for bug TDI-26084, whether or not hide folders under job_doc/joblet_doc.
-            if (ERepositoryObjectType.JOB_DOC.equals(type) || ERepositoryObjectType.JOBLET_DOC.equals(type)) {
+            if (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC)) {
                 for (FolderItem delFolder : delFolderItems) {
                     String parentName = ((FolderItem) delFolder.getParent()).getProperty().getLabel();
                     String oFolderPath = oFolder.getPath();
@@ -1126,19 +1081,6 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             if (label.equals("bin") || label.startsWith(".")) { //$NON-NLS-1$ //$NON-NLS-2$
                 continue;
             }
-            // currently, only process the docs for job, joblet and route.
-            if (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC)
-                    || type.equals(ERepositoryObjectType.JOBS) || type.equals(ERepositoryObjectType.JOBLETS)
-                    || type.equals(ERepositoryObjectType.valueOf("ROUTE_DOCS"))//$NON-NLS-1$ 
-                    || type.equals(ERepositoryObjectType.valueOf("ROUTE_DOC"))) {//$NON-NLS-1$ 
-                boolean isJobDocRootFolder = ((label.indexOf("_") != -1) && (label.indexOf(".") != -1)); //$NON-NLS-1$ //$NON-NLS-2$
-                boolean isPicFolderName = label.equals(IHTMLDocConstants.PIC_FOLDER_NAME);
-
-                // Do not show job documentation root folder and Foder "pictures" on the repository view.
-                if (isJobDocRootFolder || isPicFolderName) {
-                    continue;
-                }
-            }
             // for system folder
             if (RepositoryConstants.SYSTEM_DIRECTORY.equals(label)) {
                 if (getMergeRefProject()) {
@@ -1166,10 +1108,13 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
                     parent.getChildren().add(folder);
                 }
 
-            } else if (ERepositoryObjectType.GENERATED.name().equalsIgnoreCase(label)) {
-                if (PluginChecker.isDocumentationPluginLoaded()) {
-                    convertDocumentation(newProject, container, parent, type);
-                }
+            } else
+            // ERepositoryObjectType.GENERATED
+            if (type.equals(ERepositoryObjectType.DOCUMENTATION) && label.equalsIgnoreCase("generated")) {//$NON-NLS-1$ 
+                // if (PluginChecker.isDocumentationPluginLoaded()) {
+                // use CNF content provider instead
+                // convertDocumentation(newProject, container, parent, type);
+                // }
                 continue;
             } else {
                 if (getMergeRefProject()) {
@@ -1462,6 +1407,12 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         if (vo != null) {
             String schema = vo.getId();
             schema = schema + " - " + metadataTable.getLabel(); //$NON-NLS-1$
+            if (metadataTable instanceof SAPBWTable) {
+                String innerIOType = ((SAPBWTable) metadataTable).getInnerIOType();
+                if (innerIOType != null) {
+                    schema = schema + " - " + innerIOType; //$NON-NLS-1$
+                }
+            }
             List<IRepositoryViewObject> objs = getValidationRuleObjsFromSchema(validationRules, schema);
             if (objs.size() > 0) {
                 int num = objs.size();
@@ -1632,6 +1583,18 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             iDocNode.setChildrenObjectType(ERepositoryObjectType.METADATA_SAP_IDOC);
             node.getChildren().add(iDocNode);
             createSAPIDocNodes(repObj, metadataConnection, iDocNode);
+
+            // 4. BW DataSource:
+            createSAPBWDataSourceNodes(repObj, metadataConnection, node, validationRules);
+
+            // 5. BW DataStoreObject:
+            createSAPBWDataStoreObjectNodes(repObj, metadataConnection, node, validationRules);
+
+            // 6. BW InfoCube:
+            createSAPBWInfoCubeNodes(repObj, metadataConnection, node, validationRules);
+
+            // 7. BW InfoObject:
+            createSAPBWInfoObjectNodes(repObj, metadataConnection, node, validationRules);
         } else if (metadataConnection instanceof SalesforceSchemaConnection) {
             createSalesforceModuleNodes(repObj, metadataConnection, node, validationRules);
         } else {
@@ -1658,6 +1621,66 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
 
     }
 
+    private void createSAPBWDataSourceNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWDataSource"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_DATASOURCE);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> datasources = ((SAPConnection) metadataConnection).getBWDataSources();
+        EList tables = new BasicEList();
+        tables.addAll(datasources);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWDataStoreObjectNodes(IRepositoryViewObject repObj, Connection metadataConnection,
+            RepositoryNode node, List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWDataStoreObject"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_DATASTOREOBJECT);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> dataStoreObjects = ((SAPConnection) metadataConnection).getBWDataStoreObjects();
+        EList tables = new BasicEList();
+        tables.addAll(dataStoreObjects);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWInfoCubeNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWInfoCube"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_INFOCUBE);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> infoCubes = ((SAPConnection) metadataConnection).getBWInfoCubes();
+        EList tables = new BasicEList();
+        tables.addAll(infoCubes);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWInfoObjectNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWInfoObject"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_INFOOBJECT);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> infoObjects = ((SAPConnection) metadataConnection).getBWInfoObjects();
+        EList<SAPBWTable> tables = new BasicEList<SAPBWTable>();
+        for (SAPBWTable bwTable : infoObjects) {
+            if (!SAPBWTableHelper.IO_INNERTYPE_BASIC.equals(bwTable.getInnerIOType())) {
+                tables.add(bwTable);
+            }
+        }
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
     private void createSalesforceModuleNodes(IRepositoryViewObject rebObj, Connection metadataConnection,
             RepositoryNode connectionNode, List<IRepositoryViewObject> validationRules) {
         EList modules = ((SalesforceSchemaConnection) metadataConnection).getModules();

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -126,6 +126,7 @@ import org.talend.repository.documentation.ERepositoryActionName;
 import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.repository.model.RepositoryConstants;
 import org.talend.utils.io.FilesUtils;
+import org.talend.utils.json.JSONArray;
 import orgomg.cwm.objectmodel.core.ModelElement;
 
 /**
@@ -2227,4 +2228,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         }
     }
 
+    public JSONArray getAllRemoteLocks() {
+        return repositoryFactoryFromProvider.getAllRemoteLocks();
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/repositoryObject/MetadataTableRepositoryObject.java

@@ -27,6 +27,8 @@ import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
 import org.talend.core.model.metadata.builder.connection.Concept;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.MDMConnection;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
+import org.talend.core.model.metadata.builder.connection.SAPConnection;
 import org.talend.core.model.metadata.builder.connection.SAPFunctionUnit;
 import org.talend.core.model.metadata.builder.connection.SalesforceModuleUnit;
 import org.talend.core.model.properties.ConnectionItem;
@@ -38,6 +40,7 @@ import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.repository.ISubRepositoryObject;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.SwitchHelpers;
 import org.talend.cwm.relational.TdTable;
@@ -121,6 +124,12 @@ public class MetadataTableRepositoryObject extends MetadataTable implements ISub
             moduleUnit.getTables().remove(table);
             return;
         }
+        
+        if (table.eContainer() instanceof SAPConnection && table instanceof SAPBWTable) {
+            SAPBWTableHelper.removeBWTable((Connection) table.eContainer(), ((SAPBWTable) table).getModelType(),
+                    (SAPBWTable) table);
+        }
+
         if (table.getNamespace() instanceof Package) {
             Package pkg = (Package) table.getNamespace();
             if (pkg.getOwnedElement().contains(table)) {
@@ -184,6 +193,10 @@ public class MetadataTableRepositoryObject extends MetadataTable implements ISub
         } else if (table.eContainer() instanceof SAPFunctionUnit) {
             SAPFunctionUnit funUnit = (SAPFunctionUnit) table.eContainer();
             tables = ConnectionHelper.getTables(connection, funUnit);
+        } else if (table instanceof SAPBWTable) {
+            String bwTableType = ((SAPBWTable) table).getModelType();
+            String innerIOType = ((SAPBWTable) table).getInnerIOType();
+            tables = SAPBWTableHelper.getBWTables(connection, bwTableType, innerIOType, true);
         } else {
             tables = ConnectionHelper.getTables(connection);
         }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/recyclebin/RecycleBinManager.java

@@ -20,6 +20,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.eclipse.core.resources.IProject;
+import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.common.util.URI;
 import org.eclipse.emf.ecore.resource.Resource;
@@ -75,6 +76,11 @@ public class RecycleBinManager {
         projectRecyclebins.clear();
     }
 
+    public void clearCache(Project project) {
+        String projectTechnicalLabel = project.getTechnicalLabel();
+        projectRecyclebins.remove(projectTechnicalLabel);
+    }
+
     public void clearIndex(Project project) {
         loadRecycleBin(project);
         projectRecyclebins.get(project.getTechnicalLabel()).getDeletedItems().clear();
@@ -88,8 +94,15 @@ public class RecycleBinManager {
         List<TalendItem> notDeletedItems = new ArrayList<TalendItem>();
         for (TalendItem deletedItem : deletedItems) {
             try {
+                final ERepositoryObjectType type = ERepositoryObjectType.getType(deletedItem.getType());
+                // ignore the generated doc in recycle bin
+                if (type != null
+                        && (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC) || type
+                                .equals(ERepositoryObjectType.valueOf("ROUTE_DOC")))) { //$NON-NLS-1$
+                    continue;
+                }
                 IRepositoryViewObject object = ProxyRepositoryFactory.getInstance().getLastVersion(project, deletedItem.getId(),
-                        deletedItem.getPath(), ERepositoryObjectType.getType(deletedItem.getType()));
+                        deletedItem.getPath(), type);
                 if (object == null) {
                     object = ProxyRepositoryFactory.getInstance().getLastVersion(project, deletedItem.getId());
                 }
@@ -164,6 +177,11 @@ public class RecycleBinManager {
         }
     }
 
+    public RecycleBin getRecycleBin(Project project) {
+        loadRecycleBin(project);
+        return projectRecyclebins.get(project.getTechnicalLabel());
+    }
+
     private void loadRecycleBin(Project project) {
         if (projectRecyclebins.get(project.getTechnicalLabel()) != null) {
             // already loaded, nothing to do. Don't do any force reload
@@ -186,6 +204,16 @@ public class RecycleBinManager {
         }
     }
 
+    public RecycleBin loadRecycleBin(IPath recycleBinIndexPath) throws Exception {
+        Resource resource = createRecycleBinResource(recycleBinIndexPath);
+        resource.load(null);
+        return loadRecycleBin(resource);
+    }
+
+    public RecycleBin loadRecycleBin(Resource resource) {
+        return (RecycleBin) EcoreUtil.getObjectByType(resource.getContents(), RecycleBinPackage.eINSTANCE.getRecycleBin());
+    }
+
     public void saveRecycleBin(Project project) {
         if (projectRecyclebins.get(project.getTechnicalLabel()) == null) {
             loadRecycleBin(project);
@@ -219,7 +247,12 @@ public class RecycleBinManager {
 
     private Resource createRecycleBinResource(Project project) {
         IProject eclipseProject = ProjectManager.getInstance().getResourceProject(project.getEmfProject());
-        URI uri = URIHelper.convert(eclipseProject.getFullPath().append(TALEND_RECYCLE_BIN_INDEX));
+
+        return createRecycleBinResource(eclipseProject.getFullPath().append(TALEND_RECYCLE_BIN_INDEX));
+    }
+
+    public Resource createRecycleBinResource(IPath recycleBinIndexPath) {
+        URI uri = URIHelper.convert(recycleBinIndexPath);
 
         XMLResourceFactoryImpl resourceFact = new XMLResourceFactoryImpl();
         XMLResource resource = (XMLResource) resourceFact.createResource(uri);
@@ -234,5 +267,4 @@ public class RecycleBinManager {
         resource.getDefaultLoadOptions().put(XMLResource.OPTION_USE_LEXICAL_HANDLER, Boolean.TRUE);
         return resource;
     }
-
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CreateFolderAction.java

@@ -107,8 +107,9 @@ public class CreateFolderAction extends AContextualAction {
                 canWork = false;
                 break;
             case SYSTEM_FOLDER:
-                if (ERepositoryObjectType.GENERATED.equals(property) || ERepositoryObjectType.JOBS.equals(property)
-                        || ERepositoryObjectType.JOBLETS.equals(property) || ERepositoryObjectType.SQLPATTERNS.equals(property)
+                if (property == null || property.equals(ERepositoryObjectType.GENERATED)
+                        || property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
+                        || ERepositoryObjectType.SQLPATTERNS.equals(property)
                         || ERepositoryObjectType.REFERENCED_PROJECTS.equals(property)
                         || ERepositoryObjectType.SVN_ROOT.equals(property)) {
                     canWork = false;
@@ -122,7 +123,7 @@ public class CreateFolderAction extends AContextualAction {
                 }
                 break;
             case SIMPLE_FOLDER:
-                if (ERepositoryObjectType.JOB_DOC.equals(property) || ERepositoryObjectType.JOBLET_DOC.equals(property)
+                if (property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
                         || (ERepositoryObjectType.SQLPATTERNS.equals(property) && !isUnderUserDefined(node))) {
                     canWork = false;
                 } else if (property != null

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DeleteAction.java

@@ -1197,9 +1197,6 @@ public class DeleteAction extends AContextualAction {
             return true;
         }
 
-        if (node.getContentType() == ERepositoryObjectType.JOBS) {
-            return true;
-        }
         if (node.getContentType() == ERepositoryObjectType.GENERATED) {
             return true;
         }
@@ -1495,8 +1492,8 @@ public class DeleteAction extends AContextualAction {
                         label = (String) obj;
                     }
                     boolean isGointoCondition = false;
-                    if (node.getContentType() == ERepositoryObjectType.JOB_DOC
-                            || node.getContentType() == ERepositoryObjectType.JOBLET_DOC
+                    if (node.getContentType().equals(ERepositoryObjectType.JOB_DOC)
+                            || node.getContentType().equals(ERepositoryObjectType.JOBLET_DOC)
                             || RepositoryConstants.USER_DEFINED.equals(label)) {
                         visible = false;
                         isGointoCondition = true;

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DuplicateAction.java

@@ -152,8 +152,8 @@ public class DuplicateAction extends AContextualAction {
                     } else if (node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC) {
                         canWork = false;
-                    } else if (node.getContentType() == ERepositoryObjectType.JOBS
-                            || node.getContentType() == ERepositoryObjectType.JOBLETS
+                    } else if (node.getContentType() == ERepositoryObjectType.JOB_DOC
+                            || node.getContentType() == ERepositoryObjectType.JOBLET_DOC
                             || node.getContentType() == ERepositoryObjectType.GENERATED
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC) {
@@ -511,7 +511,7 @@ public class DuplicateAction extends AContextualAction {
     private boolean isKeyword(String itemName) {
         ERepositoryObjectType itemType = sourceNode.getObjectType();
         ERepositoryObjectType[] types = { ERepositoryObjectType.PROCESS, ERepositoryObjectType.ROUTINES,
-                ERepositoryObjectType.JOBS, ERepositoryObjectType.JOBLET, ERepositoryObjectType.JOBLETS,
+                ERepositoryObjectType.JOB_DOC, ERepositoryObjectType.JOBLET, ERepositoryObjectType.JOBLET_DOC,
                 ERepositoryObjectType.JOB_SCRIPT };
         List<ERepositoryObjectType> arraysList = Arrays.asList(types);
         List<ERepositoryObjectType> typeList = new ArrayList<ERepositoryObjectType>();

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/PasteAction.java

@@ -166,8 +166,8 @@ public class PasteAction extends AContextualAction {
         IProxyRepositoryFactory proxyFactory = ProxyRepositoryFactory.getInstance();
         IRepositoryViewObject object = target.getObject();
 
-        if (target.getContentType() == ERepositoryObjectType.JOBS
-                || target.getContentType() == ERepositoryObjectType.JOBLETS
+        if (target.getContentType() == ERepositoryObjectType.JOB_DOC
+                || target.getContentType() == ERepositoryObjectType.JOBLET_DOC
                 || target.getContentType() == ERepositoryObjectType.GENERATED
                 || target.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                 || target.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ConvertJobsUtil.java

@@ -33,7 +33,6 @@ import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.hadoop.HadoopConstants;
 import org.talend.core.hadoop.IHadoopDistributionService;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.properties.Item;
@@ -43,6 +42,8 @@ import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.repository.i18n.Messages;
 import org.talend.core.runtime.CoreRuntimePlugin;
+import org.talend.core.runtime.hd.IHDConstants;
+import org.talend.core.runtime.hd.IHDistributionVersion;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.convert.IProcessConvertService;
 import org.talend.designer.core.convert.IProcessConvertToAllTypeService;
@@ -317,8 +318,8 @@ public class ConvertJobsUtil {
         ERepositoryObjectType oldRepType = (oldJobType == null ? null : oldJobType.getERepositoryObjectType());
         ERepositoryObjectType newRepType = (newJobType == null ? null : newJobType.getERepositoryObjectType());
 
-        boolean isNeedConvert = ProcessConvertManager.getInstance().CheckConvertProcess(oldRepType, oldFrameworkValue, newRepType,
-                newFrameworkValue);
+        boolean isNeedConvert = ProcessConvertManager.getInstance().CheckConvertProcess(oldRepType, oldFrameworkValue,
+                newRepType, newFrameworkValue);
 
         // if need popup warning, then do this check
         if (isNeedConvert && needPopupWarning) {
@@ -327,29 +328,29 @@ public class ConvertJobsUtil {
                 boolean isSparkStreaming = SPARKSTREAMING_FRAMEWORK.equals(newFrameworkValue);
                 if (isSpark || isSparkStreaming) {
                     try {
-                        IProcessConvertService converter = ProcessConvertManager.getInstance()
-                                .extractConvertService(ProcessConverterType.CONVERTER_FOR_MAPREDUCE);
+                        IProcessConvertService converter = ProcessConvertManager.getInstance().extractConvertService(
+                                ProcessConverterType.CONVERTER_FOR_MAPREDUCE);
 
                         IProcess process = converter.getProcessFromItem(originalItem, false);
                         IElementParameter mrVersion = process.getElementParameter(MR_VERSION);
-                        if (mrVersion != null) {
-                            EHadoopVersion4Drivers hadoopVersion = EHadoopVersion4Drivers
-                                    .indexOfByVersion((String) mrVersion.getValue());
-                            if (hadoopVersion != null) {
-                                if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
-                                    IHadoopDistributionService hadoopService = (IHadoopDistributionService) GlobalServiceRegister
-                                            .getDefault().getService(IHadoopDistributionService.class);
-                                    boolean isSupport = false;
-                                    if (isSpark) {
-                                        isSupport = hadoopService.isSupportSpark(hadoopVersion);
-                                    } else if (isSparkStreaming) {
-                                        isSupport = hadoopService.isSupportSparkStreaming(hadoopVersion);
-                                    }
-                                    if (!isSupport) {
-                                        MessageDialog.openWarning(Display.getDefault().getActiveShell(),
-                                                Messages.getString("ConvertJobsUtil.warning.title"), //$NON-NLS-1$
-                                                Messages.getString("ConvertJobsUtil.warning.message")); //$NON-NLS-1$
-                                    }
+                        if (mrVersion != null
+                                && GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
+                            IHadoopDistributionService hadoopService = (IHadoopDistributionService) GlobalServiceRegister
+                                    .getDefault().getService(IHadoopDistributionService.class);
+                            IHDistributionVersion distributionVersion = hadoopService.getHadoopDistributionVersion(
+                                    (String) mrVersion.getValue(), false);
+                            if (distributionVersion != null) {
+                                boolean isSupport = false;
+                                if (isSpark) {
+                                    isSupport = hadoopService.doSupportService(distributionVersion, IHDConstants.SERVICE_SPARK);
+                                } else if (isSparkStreaming) {
+                                    isSupport = hadoopService.doSupportService(distributionVersion,
+                                            IHDConstants.SERVICE_SPARK_STREAMING);
+                                }
+                                if (!isSupport) {
+                                    MessageDialog.openWarning(Display.getDefault().getActiveShell(),
+                                            Messages.getString("ConvertJobsUtil.warning.title"), //$NON-NLS-1$
+                                            Messages.getString("ConvertJobsUtil.warning.message")); //$NON-NLS-1$
                                 }
                             }
                         }

main/plugins/org.talend.core.runtime/META-INF/MANIFEST.MF

@@ -15,7 +15,6 @@ Export-Package: org.talend.commons.utils.generation,
  org.talend.core.database.conn,
  org.talend.core.database.conn.template,
  org.talend.core.database.conn.version,
- org.talend.core.database.hbase.conn.version,
  org.talend.core.database.utils,
  org.talend.core.download,
  org.talend.core.exception,
@@ -54,6 +53,8 @@ Export-Package: org.talend.commons.utils.generation,
  org.talend.core.repository,
  org.talend.core.runtime,
  org.talend.core.runtime.conf,
+ org.talend.core.runtime.hd,
+ org.talend.core.runtime.hd.hive,
  org.talend.core.runtime.maven,
  org.talend.core.runtime.preference,
  org.talend.core.runtime.process,
@@ -105,14 +106,14 @@ Require-Bundle: org.eclipse.jdt.core,
  org.apache.log4j,
  org.talend.model,
  org.talend.utils;visibility:=reexport,
- org.talend.libraries.apache,
  org.talend.libraries.dom4j-jaxen,
  org.talend.libraries.csv,
  org.talend.commons.runtime;visibility:=reexport,
  org.talend.commons.ui,
  org.ops4j.pax.url.mvn,
  org.talend.components.api,
- org.talend.libraries.jackson
+ org.apache.servicemix.bundles.avro,
+ jackson-core-asl
 Bundle-Activator: org.talend.core.runtime.CoreRuntimePlugin
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,

main/plugins/org.talend.core.runtime/mappings/mapping_Cassandra22_datastax.xml

@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<mapping>
+    <dbms product="Cassandra" id="cassandra22_datastax_id" label="Mapping Cassandra"
+        default="true">
+        <dbTypes>
+            <dbType type="ascii" ignorePre="true" />
+            <dbType type="bigint" ignorePre="true" />
+            <dbType type="blob" ignorePre="true" />
+            <dbType type="boolean" ignorePre="true" />
+            <dbType type="counter" ignorePre="true" />
+            <dbType type="date" ignorePre="true" />
+            <dbType type="decimal" ignorePre="true" />
+            <dbType type="double" ignorePre="true" />
+            <dbType type="float" ignorePre="true" />
+            <dbType type="inet" ignorePre="true" />
+            <dbType type="int" ignorePre="true" />
+            <dbType type="list" ignorePre="true" />
+            <dbType type="map" ignorePre="true" />
+            <dbType type="set" ignorePre="true" />
+            <dbType type="smallint" ignorePre="true" />
+            <dbType type="text" ignorePre="true" />
+            <dbType type="time" ignorePre="true" />
+            <dbType type="timestamp" ignorePre="true" />
+            <dbType type="timeuuid" ignorePre="true" />
+            <dbType type="tinyint" ignorePre="true" />
+            <dbType type="uuid" ignorePre="true" />
+            <dbType type="varchar" ignorePre="true" />
+            <dbType type="varint" ignorePre="true" />
+        </dbTypes>
+
+        <language name="java">
+            <talendToDbTypes>
+                <talendType type="id_List">
+                    <dbType type="list" default="true"/>
+                </talendType>
+                <talendType type="id_Boolean">
+                    <dbType type="boolean" default="true" />
+                </talendType>
+                <talendType type="id_Byte">
+                    <dbType type="tinyint" default="true" />
+                </talendType>
+                <talendType type="id_byte[]">
+                    <dbType type="blob" default="true" />
+                </talendType>
+                <talendType type="id_Character">
+                    <dbType type="ascii" default="true" />
+                    <dbType type="text" />
+                    <dbType type="varchar" />
+                </talendType>
+                <talendType type="id_Date">
+                    <dbType type="timestamp" default="true" />
+                    <dbType type="date" />
+                </talendType>
+                <talendType type="id_BigDecimal">
+                    <dbType type="decimal" default="true" />
+                </talendType>
+                <talendType type="id_Double">
+                    <dbType type="double" default="true" />
+                </talendType>
+                <talendType type="id_Float">
+                    <dbType type="float" default="true" />
+                </talendType>
+                <talendType type="id_Integer">
+                    <dbType type="int" default="true" />
+                </talendType>
+                <talendType type="id_Long">
+                    <dbType type="bigint" default="true" />
+                    <dbType type="counter" />
+                    <dbType type="time" />
+                </talendType>
+                <talendType type="id_Object">
+                    <dbType type="varint" default="true" />
+                    <dbType type="inet" />
+                    <dbType type="map" />
+                    <dbType type="set" />
+                </talendType>    
+                <talendType type="id_Short">
+                    <dbType type="smallint" default="true" />
+                </talendType>
+                <talendType type="id_String">
+                    <dbType type="ascii" default="true" />
+                    <dbType type="text" />
+                    <dbType type="timeuuid" />
+                    <dbType type="uuid" />
+                    <dbType type="varchar" />
+                </talendType>
+            </talendToDbTypes>
+            <dbToTalendTypes>
+                <dbType type="ascii">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="bigint">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="blob">
+                    <talendType type="id_byte[]" default="true" />
+                </dbType>
+                <dbType type="boolean">
+                    <talendType type="id_Boolean" default="true" />
+                </dbType>
+                <dbType type="counter">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="date">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="decimal">
+                    <talendType type="id_BigDecimal" default="true" />
+                </dbType>
+                <dbType type="double">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="float">
+                    <talendType type="id_Float" default="true" />
+                </dbType>
+                <dbType type="inet">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="int">
+                    <talendType type="id_Integer" default="true" />
+                </dbType>
+                <dbType type="list">
+                    <talendType type="id_List" default="true" />
+                </dbType>
+                <dbType type="map">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="set">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="smallint">
+                    <talendType type="id_Short" default="true" />
+                </dbType>
+                <dbType type="text">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="time">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="timestamp">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="timeuuid">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="tinyint">
+                    <talendType type="id_Byte" default="true" />
+                </dbType>
+                <dbType type="uuid">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="varchar">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="varint">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+            </dbToTalendTypes>
+        </language>
+    </dbms>
+
+</mapping>

main/plugins/org.talend.core.runtime/mappings/mapping_Hive.xml

@@ -9,11 +9,13 @@
 			<dbType type="INT" ignoreLen="true" ignorePre="true"/>
 			<dbType type="TINYINT" ignoreLen="true" ignorePre="true"/>
 			<dbType type="STRING" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DECIMAL" ignoreLen="false" ignorePre="false"/>
 			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true"/>
 			<dbType type="STRUCT" ignoreLen="true" ignorePre="true"/>
 			<dbType type="MAP" ignoreLen="true" ignorePre="true"/>
 			<dbType type="ARRAY" ignoreLen="true" ignorePre="true"/>
 			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DATE" ignoreLen="true" ignorePre="true"/>
 		</dbTypes>
 		
 		<language name="java">
@@ -34,9 +36,11 @@
 				</talendType>
 				<talendType type="id_Date">
 				    <dbType type="TIMESTAMP" default="true"/>
+				    <dbType type="DATE"/>
 				</talendType>
 				<talendType type="id_BigDecimal">
-					<dbType type="BIGINT" default="true"/>
+				    <dbType type="DECIMAL" default="true"/>
+					<dbType type="BIGINT"/>
 					<dbType type="FLOAT"/>
 					<dbType type="DOUBLE"/>
 				</talendType>
@@ -80,6 +84,9 @@
 					<talendType type="id_BigDecimal"/>
 					<talendType type="id_Float"/>
 				</dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true"/>
+                </dbType>
 				<dbType type="BIGINT">
 					<talendType type="id_BigDecimal" default="true"/>
 					<talendType type="id_Byte"/>
@@ -111,7 +118,10 @@
 				<dbType type="ARRAY">
 					<talendType type="id_Object" default="true"/>
 				</dbType>
-				<dbType type="TIMESTAMP">
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true"/>
+                </dbType>
+                <dbType type="DATE">
                     <talendType type="id_Date" default="true"/>
                 </dbType>
 			</dbToTalendTypes>

main/plugins/org.talend.core.runtime/mappings/mapping_Redshift_new.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0"?>
+<mapping>
+	<dbms product="Redshift" id="redshift_new_id" label="Mapping Redshift"
+		default="true">
+		<dbTypes>
+            <dbType type="SMALLINT" ignoreLen="true" ignorePre="true" />
+            <dbType type="INTEGER" ignoreLen="true" ignorePre="true" />
+            <dbType type="BIGINT" ignoreLen="true" ignorePre="true" />
+            <dbType type="DECIMAL" ignoreLen="true" />
+            <dbType type="REAL" ignoreLen="true" />
+            <dbType type="DOUBLE PRECISION" ignoreLen="true" />
+            <dbType type="BOOLEAN" ignoreLen="true" ignorePre="true" />
+            <dbType type="CHAR" ignoreLen="true" ignorePre="true" />
+            <dbType type="VARCHAR" ignorePre="true" />
+            <dbType type="DATE" ignoreLen="true" ignorePre="true" />
+            <dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true" />
+		</dbTypes>
+		<language name="java">
+            <talendToDbTypes>
+                <talendType type="id_List">
+                </talendType>
+                <talendType type="id_Boolean">
+                    <dbType type="BOOLEAN" default="true" />
+                </talendType>
+                <talendType type="id_Byte">
+                </talendType>
+                <talendType type="id_byte[]">
+                </talendType>
+                <talendType type="id_Character">
+                </talendType>
+                <talendType type="id_Date">
+                    <dbType type="DATE" default="true" />
+                    <dbType type="TIMESTAMP" />
+                </talendType>
+                <talendType type="id_BigDecimal">
+                    <dbType type="DECIMAL" default="true" />
+                    <dbType type="BIGINT" />
+                </talendType>
+                <talendType type="id_Double">
+                    <dbType type="DOUBLE PRECISION" default="true" />
+                    <dbType type="REAL" />
+                </talendType>
+                <talendType type="id_Float">
+                </talendType>
+                <talendType type="id_Integer">
+                    <dbType type="INTEGER" default="true" />
+                    <dbType type="SMALLINT" />
+                </talendType>
+                <talendType type="id_Long">
+                    <dbType type="BIGINT" default="true" />
+                    <dbType type="INTEGER" />
+                </talendType>
+                <talendType type="id_Object">
+                </talendType>    
+                <talendType type="id_Short">
+                </talendType>
+                <talendType type="id_String">
+                    <dbType type="VARCHAR" default="true" />
+                    <dbType type="CHAR" />
+                </talendType>
+            </talendToDbTypes>
+            <dbToTalendTypes>
+                <dbType type="SMALLINT">
+                    <talendType type="id_Integer" default="true" />
+                </dbType>
+                <dbType type="INTEGER">
+                    <talendType type="id_Integer" default="true" />
+                    <talendType type="id_Long" />
+                </dbType>
+                <dbType type="BIGINT">
+                    <talendType type="id_Long" default="true" />
+                    <talendType type="id_BigDecimal" />
+                </dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true" />
+                </dbType>
+                <dbType type="REAL">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="DOUBLE PRECISION">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="BOOLEAN">
+                    <talendType type="id_Boolean" default="true" />
+                </dbType>
+                <dbType type="CHAR">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="VARCHAR">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="DATE">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+            </dbToTalendTypes>		  
+		</language>
+	</dbms>
+</mapping>
+        

main/plugins/org.talend.core.runtime/resources/hadoop/hadoop_default_confs.json

@@ -10,6 +10,11 @@
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
         "JOBHISTORY_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -35,6 +40,11 @@
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
         "JOBHISTORY_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -56,6 +66,11 @@
         "RESOURCEMANAGER_SCHEDULER_ADDRESS": "localhost:8030",
         "JOBHISTORY_ADDRESS": "0.0.0.0:10020",
         "STAGING_DIRECTORY": "/user",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -79,6 +94,11 @@
         "NAMENODE_PRINCIPAL": "nn/_HOST@EXAMPLE.COM",
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -150,6 +170,11 @@
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
         "JOBHISTORY_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083
@@ -174,6 +199,11 @@
         "NAMENODE_PRINCIPAL": "nn/_HOST@EXAMPLE.COM",
         "JOBTRACKER_PRINCIPAL" : "mapred/_HOST@EXAMPLE.COM",
         "RESOURCE_MANAGER_PRINCIPAL" : "yarn/_HOST@EXAMPLE.COM",
+        "CLOUDERA_NAVIGATOR_USERNAME": "username",
+        "CLOUDERA_NAVIGATOR_PASSWORD": "",
+        "CLOUDERA_NAVIGATOR_URL": "http://localhost:7187/api/v8/",
+        "CLOUDERA_NAVIGATOR_METADATA_URL": "http://localhost:7187/api/v8/metadata/plugin",
+        "CLOUDERA_NAVIGATOR_CLIENT_URL": "http://localhost",
         "HIVE": {
             "EMBEDDED": {
                 "PORT": 9083

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/classloader/ClassLoaderFactory.java

@@ -34,7 +34,8 @@ import org.talend.core.ILibraryManagerService;
 import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.IMetadataConnection;
-import org.talend.core.model.metadata.connection.hive.HiveConnUtils;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.hive.HiveMetadataHelper;
 import org.talend.repository.ProjectManager;
 import org.talend.utils.io.FilesUtils;
 
@@ -43,21 +44,21 @@ import org.talend.utils.io.FilesUtils;
  */
 public class ClassLoaderFactory {
 
-    private final static String EXTENSION_POINT_ID = "org.talend.core.runtime.classloader_provider"; //$NON-NLS-1$
+    public final static String EXTENSION_POINT_ID = "org.talend.core.runtime.classloader_provider"; //$NON-NLS-1$
 
     private static IConfigurationElement[] configurationElements = null;
 
     private static Map<String, DynamicClassLoader> classLoadersMap = null;
 
-    private final static String SEPARATOR = ";"; //$NON-NLS-1$
+    public final static String SEPARATOR = ";"; //$NON-NLS-1$
 
     private final static String PATH_SEPARATOR = "/"; //$NON-NLS-1$
 
-    private final static String INDEX_ATTR = "index"; //$NON-NLS-1$
+    public final static String INDEX_ATTR = "index"; //$NON-NLS-1$
 
-    private final static String LIB_ATTR = "libraries"; //$NON-NLS-1$
+    public final static String LIB_ATTR = "libraries"; //$NON-NLS-1$
 
-    private final static String PARENT_ATTR = "parent"; //$NON-NLS-1$
+    public final static String PARENT_ATTR = "parent"; //$NON-NLS-1$
 
     static {
         IExtensionRegistry registry = Platform.getExtensionRegistry();
@@ -166,44 +167,67 @@ public class ClassLoaderFactory {
         classLoadersMap = new ConcurrentHashMap<String, DynamicClassLoader>();
     }
 
-    private static synchronized DynamicClassLoader findLoader(String index, ClassLoader parentLoader,
-            boolean showDownloadIfNotExist) {
-        if (index != null && configurationElements != null) {
+    public static IConfigurationElement findIndex(String index) {
+        if (StringUtils.isNotEmpty(index) && configurationElements != null) {
             for (IConfigurationElement current : configurationElements) {
                 String key = current.getAttribute(INDEX_ATTR);
                 if (index.equals(key)) {
-                    String libraries = current.getAttribute(LIB_ATTR);
-                    if (StringUtils.isNotEmpty(index)) {
-                        DynamicClassLoader classLoader = null;
-                        ClassLoader parentClassLoader = null;
-                        String parentKey = current.getAttribute(PARENT_ATTR);
-                        // take parent classlaoder in extensions first
-                        if (StringUtils.isNotEmpty(parentKey)) {
-                            parentClassLoader = getClassLoader(parentKey, showDownloadIfNotExist);
-                        }
-                        if (parentClassLoader == null) {
-                            parentClassLoader = parentLoader;
-                        }
-
-                        if (parentClassLoader == null) {
-                            classLoader = new DynamicClassLoader();
-                        } else {
-                            classLoader = new DynamicClassLoader(parentClassLoader);
-                        }
-                        boolean putInCache = true;
-                        if (StringUtils.isNotEmpty(libraries)) {
-                            String[] librariesArray = libraries.split(SEPARATOR);
-                            putInCache = loadLibraries(classLoader, librariesArray, showDownloadIfNotExist);
-                        }
-                        if (putInCache) {
-                            // if any libraries can't be retreived , do not put it in cache
-                            classLoadersMap.put(index, classLoader);
-                        }
-                        return classLoader;
-                    }
+                    return current;
                 }
             }
         }
+        return null;
+    }
+
+    public static String[] getLibs(String index) {
+        IConfigurationElement current = findIndex(index);
+        return getLibs(current);
+    }
+
+    public static String[] getLibs(IConfigurationElement current) {
+        if (current != null) {
+            String libraries = current.getAttribute(LIB_ATTR);
+            if (StringUtils.isNotEmpty(libraries)) {
+                return libraries.split(SEPARATOR);
+            }
+        }
+        return new String[0];
+    }
+
+    private static synchronized DynamicClassLoader findLoader(String index, ClassLoader parentLoader,
+            boolean showDownloadIfNotExist) {
+        IConfigurationElement current = findIndex(index);
+        if (current != null) {
+            // String key = current.getAttribute(INDEX_ATTR);
+            // String libraries = current.getAttribute(LIB_ATTR);
+            String parentKey = current.getAttribute(PARENT_ATTR);
+
+            ClassLoader parentClassLoader = null;
+            // take parent classlaoder in extensions first
+            if (StringUtils.isNotEmpty(parentKey)) {
+                parentClassLoader = getClassLoader(parentKey, showDownloadIfNotExist);
+            }
+            if (parentClassLoader == null) {
+                parentClassLoader = parentLoader;
+            }
+
+            DynamicClassLoader classLoader = null;
+            if (parentClassLoader == null) {
+                classLoader = new DynamicClassLoader();
+            } else {
+                classLoader = new DynamicClassLoader(parentClassLoader);
+            }
+            boolean putInCache = true;
+            String[] librariesArray = getLibs(current);
+            if (librariesArray.length > 0) {
+                putInCache = loadLibraries(classLoader, librariesArray, showDownloadIfNotExist);
+            }
+            if (putInCache) {
+                // if any libraries can't be retreived , do not put it in cache
+                classLoadersMap.put(index, classLoader);
+            }
+            return classLoader;
+        }
 
         return null;
     }
@@ -275,7 +299,9 @@ public class ClassLoaderFactory {
         String distroKey = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_DISTRIBUTION);
         String distroVersion = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_VERSION);
         String hiveModel = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_MODE);
-        if (HiveConnUtils.isCustomDistro(distroKey)) {
+
+        IHDistribution distribution = HiveMetadataHelper.getDistribution(distroKey, false);
+        if (distribution != null && distribution.useCustom()) {
             String jarsStr = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HADOOP_CUSTOM_JARS);
             moduleList = jarsStr.split(";"); //$NON-NLS-1$
         } else {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -23,7 +23,7 @@ import org.talend.core.database.conn.template.DbConnStrForHive;
 import org.talend.core.database.conn.template.EDatabaseConnTemplate;
 import org.talend.core.database.conn.version.EDatabaseVersion4Drivers;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
-import org.talend.core.model.metadata.connection.hive.HiveConnVersionInfo;
+import org.talend.core.model.metadata.connection.hive.HiveModeInfo;
 import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.utils.TalendQuoteUtils;
 
@@ -191,15 +191,16 @@ public class DatabaseConnStrUtil {
         }
         // DbConnStrForHive.URL_HIVE_2_TEMPLATE or DbConnStrForHive.URL_HIVE_1_TEMPLATE
         // set a default
+        boolean isEmbedded = (HiveModeInfo.get(hiveModel) == HiveModeInfo.EMBEDDED);
         String url = null;
         if (template.startsWith(DbConnStrForHive.URL_HIVE_2_TEMPLATE)) {
-            if (HiveConnVersionInfo.MODE_EMBEDDED.getKey().equalsIgnoreCase(hiveModel)) {
+            if (isEmbedded) {
                 url = getHive2EmbeddedURLString();
             } else {
                 url = getHive2StandaloneURLString(false, server, port, sidOrDatabase, hivePrincipal);
             }
         } else if (template.startsWith(DbConnStrForHive.URL_HIVE_1_TEMPLATE)) {
-            if (HiveConnVersionInfo.MODE_EMBEDDED.getKey().equalsIgnoreCase(hiveModel)) {
+            if (isEmbedded) {
                 url = getHive1EmbeddedURLString();
             } else {
                 url = getHive1StandaloneURLString(false, server, port, sidOrDatabase);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/EDatabaseConnTemplate.java

@@ -81,7 +81,7 @@ public enum EDatabaseConnTemplate {
             "jdbc:odbc:<datasource>")), //$NON-NLS-1$
 
     ACCESS(new DbConnStr(EDatabaseTypeName.ACCESS, //
-            "jdbc:ucanaccess://<filename>")), //$NON-NLS-1$
+            "jdbc:ucanaccess://<filename>;<property>",null,"memory=false")), //$NON-NLS-1$
 
     SYBASEASE(new DbConnStr(EDatabaseTypeName.SYBASEASE, //
             "jdbc:sybase:Tds:<host>:<port>/<sid>?<property>", //$NON-NLS-1$ 
@@ -357,6 +357,7 @@ public enum EDatabaseConnTemplate {
             case TERADATA: // for feature 0013719
             case VERTICA:
             case EXASOL:
+            case ACCESS:
                 return true;
             default:
             }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/hbase/conn/version/EHBaseDistribution4Versions.java

@@ -1,208 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.database.hbase.conn.version;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
-
-/**
- * DOC ycbai class global comment. Detailled comment
- */
-public enum EHBaseDistribution4Versions {
-    HDP_2_3(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_3.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_3
-            .getVersionValue()),
-
-    HDP_2_2(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_2.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_2
-            .getVersionValue()),
-
-    HDP_2_1(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_1.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_1
-            .getVersionValue()),
-
-    HDP_2_0(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_2_0.getVersionDisplay(), EHadoopVersion4Drivers.HDP_2_0
-            .getVersionValue()),
-
-    HDP_1_3(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_1_3.getVersionDisplay(), EHadoopVersion4Drivers.HDP_1_3
-            .getVersionValue()),
-
-    HDP_1_2(EHBaseDistributions.HORTONWORKS, EHadoopVersion4Drivers.HDP_1_2.getVersionDisplay(), EHadoopVersion4Drivers.HDP_1_2
-            .getVersionValue()),
-
-    APACHE_2_4_0_EMR(
-                     EHBaseDistributions.AMAZON_EMR,
-                     EHadoopVersion4Drivers.APACHE_2_4_0_EMR.getVersionDisplay(),
-                     EHadoopVersion4Drivers.APACHE_2_4_0_EMR.getVersionValue()),
-
-    APACHE_1_0_3_EMR(
-                     EHBaseDistributions.AMAZON_EMR,
-                     EHadoopVersion4Drivers.APACHE_1_0_3_EMR.getVersionDisplay(),
-                     EHadoopVersion4Drivers.APACHE_1_0_3_EMR.getVersionValue()),
-
-    APACHE_1_0_0(
-                 EHBaseDistributions.APACHE,
-                 EHadoopVersion4Drivers.APACHE_1_0_0.getVersionDisplay(),
-                 EHadoopVersion4Drivers.APACHE_1_0_0.getVersionValue()),
-
-    CLOUDERA_CDH5_5(
-                    EHBaseDistributions.CLOUDERA,
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_5.getVersionDisplay(),
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_5.getVersionValue()),
-
-    CLOUDERA_CDH5_4(
-                    EHBaseDistributions.CLOUDERA,
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_4.getVersionDisplay(),
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_4.getVersionValue()),
-
-    CLOUDERA_CDH5_1(
-                    EHBaseDistributions.CLOUDERA,
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_1.getVersionDisplay(),
-                    EHadoopVersion4Drivers.CLOUDERA_CDH5_1.getVersionValue()),
-
-    CLOUDERA_CDH5_1_MR1(
-                        EHBaseDistributions.CLOUDERA,
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionDisplay(),
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionValue()),
-
-    CLOUDERA_CDH5(
-                  EHBaseDistributions.CLOUDERA,
-                  EHadoopVersion4Drivers.CLOUDERA_CDH5.getVersionDisplay(),
-                  EHadoopVersion4Drivers.CLOUDERA_CDH5.getVersionValue()),
-
-    CLOUDERA_CDH4_YARN(
-                       EHBaseDistributions.CLOUDERA,
-                       EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN.getVersionDisplay(),
-                       EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN.getVersionValue()),
-
-    CLOUDERA_CDH4(
-                  EHBaseDistributions.CLOUDERA,
-                  EHadoopVersion4Drivers.CLOUDERA_CDH4.getVersionDisplay(),
-                  EHadoopVersion4Drivers.CLOUDERA_CDH4.getVersionValue()),
-
-    MAPR_5_0_0(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR500.getVersionDisplay(), EHadoopVersion4Drivers.MAPR500
-            .getVersionValue()),
-
-    MAPR_4_1_0(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR410.getVersionDisplay(), EHadoopVersion4Drivers.MAPR410
-            .getVersionValue()),
-
-    MAPR_4_0_1(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR401.getVersionDisplay(), EHadoopVersion4Drivers.MAPR401
-            .getVersionValue()),
-
-    MAPR_3_1_0(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR310.getVersionDisplay(), EHadoopVersion4Drivers.MAPR310
-            .getVersionValue()),
-
-    MAPR_3_0_1(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR301.getVersionDisplay(), EHadoopVersion4Drivers.MAPR301
-            .getVersionValue()),
-
-    MAPR_2_1_3(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR213.getVersionDisplay(), EHadoopVersion4Drivers.MAPR213
-            .getVersionValue()),
-
-    MAPR_2_1_2(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR212.getVersionDisplay(), EHadoopVersion4Drivers.MAPR212
-            .getVersionValue()),
-
-    MAPR(EHBaseDistributions.MAPR, EHadoopVersion4Drivers.MAPR2.getVersionDisplay(), EHadoopVersion4Drivers.MAPR2
-            .getVersionValue()),
-
-    PIVOTAL_HD_2_0(
-                   EHBaseDistributions.PIVOTAL_HD,
-                   EHadoopVersion4Drivers.PIVOTAL_HD_2_0.getVersionDisplay(),
-                   EHadoopVersion4Drivers.PIVOTAL_HD_2_0.getVersionValue()),
-
-    PIVOTAL_HD_1_0_1(
-                     EHBaseDistributions.PIVOTAL_HD,
-                     EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1.getVersionDisplay(),
-                     EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1.getVersionValue()),
-
-    CUSTOM(EHBaseDistributions.CUSTOM, EHadoopVersion4Drivers.CUSTOM.getVersionDisplay(), EHadoopVersion4Drivers.CUSTOM
-            .getVersionValue());
-
-    private EHBaseDistributions distribution;
-
-    private String versionDisplayName;
-
-    private String versionValue;
-
-    EHBaseDistribution4Versions(EHBaseDistributions distribution, String versionDisplayName, String versionValue) {
-        this.distribution = distribution;
-        this.versionDisplayName = versionDisplayName;
-        this.versionValue = versionValue;
-    }
-
-    public static List<EHBaseDistribution4Versions> indexOfByDistribution(String distribution) {
-        List<EHBaseDistribution4Versions> distribution4Versions = new ArrayList<EHBaseDistribution4Versions>();
-        if (distribution != null) {
-            for (EHBaseDistribution4Versions d4v : EHBaseDistribution4Versions.values()) {
-                if (d4v.getDistribution().getName().equals(distribution)
-                        || d4v.getDistribution().getDisplayName().equals(distribution)) {
-                    distribution4Versions.add(d4v);
-                }
-            }
-        }
-        return distribution4Versions;
-    }
-
-    public static EHBaseDistribution4Versions indexOfByVersionDisplay(String displayName) {
-        return indexOf(displayName, true);
-    }
-
-    public static EHBaseDistribution4Versions indexOfByVersion(String value) {
-        return indexOf(value, false);
-    }
-
-    private static EHBaseDistribution4Versions indexOf(String name, boolean display) {
-        if (name != null) {
-            for (EHBaseDistribution4Versions version : EHBaseDistribution4Versions.values()) {
-                if (display) {
-                    if (name.equalsIgnoreCase(version.getVersionDisplayName())) {
-                        return version;
-                    }
-                } else {
-                    if (name.equalsIgnoreCase(version.getVersionValue())) {
-                        return version;
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    public static List<String> getHadoopDistributionVersions(String distribution) {
-        return getHadoopDistributionVersions(distribution, true);
-    }
-
-    public static List<String> getHadoopDistributionVersions(String distribution, boolean display) {
-        List<String> result = new ArrayList<String>();
-        List<EHBaseDistribution4Versions> d4vList = EHBaseDistribution4Versions.indexOfByDistribution(distribution);
-        for (EHBaseDistribution4Versions d4v : d4vList) {
-            if (display) {
-                result.add(d4v.getVersionDisplayName());
-            } else {
-                result.add(d4v.getVersionValue());
-            }
-        }
-        return result;
-    }
-
-    public EHBaseDistributions getDistribution() {
-        return this.distribution;
-    }
-
-    public String getVersionDisplayName() {
-        return this.versionDisplayName;
-    }
-
-    public String getVersionValue() {
-        return this.versionValue;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/hbase/conn/version/EHBaseDistributions.java

@@ -1,92 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.database.hbase.conn.version;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.talend.core.hadoop.version.EHadoopDistributions;
-
-/**
- * DOC ycbai class global comment. Detailled comment
- */
-public enum EHBaseDistributions {
-
-    HORTONWORKS(EHadoopDistributions.HORTONWORKS.getDisplayName()),
-
-    CLOUDERA(EHadoopDistributions.CLOUDERA.getDisplayName()),
-
-    MAPR(EHadoopDistributions.MAPR.getDisplayName()),
-
-    APACHE(EHadoopDistributions.APACHE.getDisplayName()),
-
-    AMAZON_EMR(EHadoopDistributions.AMAZON_EMR.getDisplayName()),
-
-    PIVOTAL_HD(EHadoopDistributions.PIVOTAL_HD.getDisplayName()),
-
-    CUSTOM(EHadoopDistributions.CUSTOM.getDisplayName());
-
-    private String displayName;
-
-    EHBaseDistributions(String displayName) {
-        this.displayName = displayName;
-    }
-
-    public String getName() {
-        return name();
-    }
-
-    public String getDisplayName() {
-        return this.displayName;
-    }
-
-    public static List<String> getAllDistributionDisplayNames() {
-        return getAllDistributionNames(true);
-    }
-
-    public static List<String> getAllDistributionNames(boolean display) {
-        List<String> names = new ArrayList<String>();
-        EHBaseDistributions[] values = values();
-        for (EHBaseDistributions distribution : values) {
-            if (display) {
-                names.add(distribution.getDisplayName());
-            } else {
-                names.add(distribution.getName());
-            }
-        }
-
-        return names;
-    }
-
-    public static EHBaseDistributions getDistributionByDisplayName(String name) {
-        return getDistributionByName(name, true);
-    }
-
-    public static EHBaseDistributions getDistributionByName(String name, boolean display) {
-        if (name != null) {
-            for (EHBaseDistributions distribution : values()) {
-                if (display) {
-                    if (name.equalsIgnoreCase(distribution.getDisplayName())) {
-                        return distribution;
-                    }
-                } else {
-                    if (name.equalsIgnoreCase(distribution.getName())) {
-                        return distribution;
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopClusterService.java

@@ -216,6 +216,8 @@ public interface IHadoopClusterService extends IService {
      */
     public boolean hasDiffsFromClusterToProcess(Item item, IProcess process);
 
+    public Connection getHadoopClusterConnectionBySubConnection(Connection hadoopSubConnection);
+
     public String getHadoopClusterProperties(Connection hadoopSubConnection);
 
     /**

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopDistributionService.java

@@ -12,18 +12,85 @@
 // ============================================================================
 package org.talend.core.hadoop;
 
-import org.talend.core.IService;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
+import java.util.Map;
 
+import org.talend.core.IService;
+import org.talend.core.runtime.hd.IDistributionsManager;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.IHDistributionVersion;
 
 /**
- * created by cmeng on Jan 15, 2016
- * Detailled comment
+ * created by cmeng on Jan 15, 2016 Detailled comment
  *
  */
 public interface IHadoopDistributionService extends IService {
 
-    public boolean isSupportSpark(EHadoopVersion4Drivers version);
+    /**
+     * 
+     * According to the service, find the distributions. The service is name of HadoopComponent service.
+     * {@link org.talend.hadoop.distribution.component.HadoopComponent}.
+     */
+    IHDistribution[] getDistributions(String service);
+
+    /**
+     * Check the distribution version support the service or not.
+     * 
+     * The service must be full name, and be sub class of
+     * {@link org.talend.hadoop.distribution.component.HadoopComponent)
+     */
+    boolean doSupportService(IHDistributionVersion distributionVersion, String service);
+
+    /**
+     * Check the boolean methods for sub class of {@link org.talend.hadoop.distribution.component.HadoopComponent)
+     * 
+     * If no the method in the HadoopComponent object, won't contain in the result map.
+     */
+    Map<String, Boolean> doSupportMethods(IHDistributionVersion distributionVersion, String... methods) throws Exception;
+
+    boolean doSupportMethod(IHDistributionVersion distributionVersion, String method) throws Exception;
+
+    /**
+     * Find the matched distribution via name.
+     * 
+     * If the display is true, the name match for display one of distribution.
+     */
+    IHDistribution getHadoopDistribution(String name, boolean byDisplay);
+
+    /**
+     * Find the matched distribution via version .
+     * 
+     * If the display is true, the version match for display one of distribution.
+     */
+    IHDistributionVersion getHadoopDistributionVersion(String version, boolean byDisplay);
+
+    /**
+     * 
+     * for Hadoop Distribution {@link HadoopComponent}.
+     */
+    IDistributionsManager getHadoopDistributionManager();
+
+    /**
+     * 
+     * for HBase in Database wizard.{@link HBaseComponent}.
+     */
+    IDistributionsManager getHBaseDistributionManager();
+
+    /**
+     * 
+     * for Hive Distribution {@link HiveComponent}.
+     */
+    IDistributionsManager getSparkDistributionManager();
+
+    /**
+     * 
+     * for Spark Distribution {@link SparkComponent}.
+     */
+    IDistributionsManager getHiveDistributionManager();
+
+    /**
+     * 
+     * Temp for Oozie.
+     */
+    IHDistribution[] getOozieDistributions();
 
-    public boolean isSupportSparkStreaming(EHadoopVersion4Drivers version);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/conf/EHadoopProperties.java

@@ -45,6 +45,16 @@ public enum EHadoopProperties {
     DATABASE,
 
     PORT,
+    
+    CLOUDERA_NAVIGATOR_USERNAME,
+    
+    CLOUDERA_NAVIGATOR_PASSWORD,
+    
+    CLOUDERA_NAVIGATOR_URL,
+    
+    CLOUDERA_NAVIGATOR_METADATA_URL,
+    
+    CLOUDERA_NAVIGATOR_CLIENT_URL,
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/repository/HadoopRepositoryUtil.java

@@ -139,8 +139,11 @@ public class HadoopRepositoryUtil {
         }
         List<Map<String, Object>> parentProperties = null;
         if (hadoopClusterService != null) {
-            parentProperties = getHadoopPropertiesList(hadoopClusterService.getHadoopClusterProperties(connection),
-                    connection.isContextMode(), true);
+            Connection hadoopClusterConnection = hadoopClusterService.getHadoopClusterConnectionBySubConnection(connection);
+            if (hadoopClusterConnection != null) {
+                parentProperties = getHadoopPropertiesList(hadoopClusterService.getHadoopClusterProperties(connection),
+                        hadoopClusterConnection.isContextMode(), true);
+            }
         }
         List<Map<String, Object>> properties = getHadoopPropertiesList(propertiesJsonStr, connection.isContextMode(), true);
         Map<String, Map<String, Object>> propertiesMap = new HashMap<String, Map<String, Object>>();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/EHadoopDistributions.java

@@ -12,11 +12,10 @@
 // ============================================================================
 package org.talend.core.hadoop.version;
 
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * DOC ycbai class global comment. Detailled comment
+ * 
+ * @deprecated @link HadoopComponent via osgi service instead.
  */
 public enum EHadoopDistributions {
 
@@ -50,28 +49,6 @@ public enum EHadoopDistributions {
         return this.displayName;
     }
 
-    public static List<String> getAllDistributionDisplayNames() {
-        return getAllDistributionNames(true);
-    }
-
-    public static List<String> getAllDistributionNames(boolean display) {
-        List<String> names = new ArrayList<String>();
-        EHadoopDistributions[] values = values();
-        for (EHadoopDistributions distribution : values) {
-            if (display) {
-                names.add(distribution.getDisplayName());
-            } else {
-                names.add(distribution.getName());
-            }
-        }
-
-        return names;
-    }
-
-    public static EHadoopDistributions getDistributionByDisplayName(String name) {
-        return getDistributionByName(name, true);
-    }
-
     public static EHadoopDistributions getDistributionByName(String name, boolean display) {
         if (name != null) {
             for (EHadoopDistributions distribution : values()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/EHadoopVersion4Drivers.java

@@ -16,15 +16,20 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.ArrayUtils;
-import org.talend.commons.exception.ExceptionHandler;
-import org.talend.core.GlobalServiceRegister;
-import org.talend.core.hadoop.IHadoopDistributionService;
 
 /**
  * DOC ycbai class global comment. Detailled comment
+ * 
+ * @deprecated @link HadoopComponent via osgi service instead.
  */
 public enum EHadoopVersion4Drivers {
 
+    HDP_2_4(EHadoopDistributions.HORTONWORKS, "Hortonworks Data Platform V2.4.0", //$NON-NLS-1$
+            "HDP_2_4", //$NON-NLS-1$
+            true,
+            false,
+            new EMRVersion[] { EMRVersion.YARN }),
+
     HDP_2_3(EHadoopDistributions.HORTONWORKS, "Hortonworks Data Platform V2.3.0", //$NON-NLS-1$
             "HDP_2_3", //$NON-NLS-1$
             true,
@@ -93,6 +98,8 @@ public enum EHadoopVersion4Drivers {
 
     CLOUDERA_CDH4(EHadoopDistributions.CLOUDERA, "Cloudera CDH4.X(MR1 mode)", "Cloudera_CDH4", true, false), //$NON-NLS-1$ //$NON-NLS-2$
 
+    MAPR510(EHadoopDistributions.MAPR, "MapR 5.1.0(YARN mode)", "MAPR510", true, true, new EMRVersion[] { EMRVersion.YARN }), //$NON-NLS-1$ //$NON-NLS-2$
+
     MAPR500(EHadoopDistributions.MAPR, "MapR 5.0.0(YARN mode)", "MAPR500", true, true, new EMRVersion[] { EMRVersion.YARN }), //$NON-NLS-1$ //$NON-NLS-2$
 
     MAPR410(EHadoopDistributions.MAPR, "MapR 4.1.0(YARN mode)", "MAPR410", false, true, new EMRVersion[] { EMRVersion.YARN }), //$NON-NLS-1$ //$NON-NLS-2$
@@ -115,6 +122,12 @@ public enum EHadoopVersion4Drivers {
               false,
               new EMRVersion[] { EMRVersion.YARN }),
 
+    EMR_4_5_0(EHadoopDistributions.AMAZON_EMR, "EMR 4.5.0 (Apache 2.7.2)", //$NON-NLS-1$
+              "EMR_4_5_0", //$NON-NLS-1$
+              false,
+              false,
+              new EMRVersion[] { EMRVersion.YARN }),
+
     APACHE_2_4_0_EMR(EHadoopDistributions.AMAZON_EMR, "Apache 2.4.0", //$NON-NLS-1$
                      "APACHE_2_4_0_EMR", //$NON-NLS-1$
                      false,
@@ -245,15 +258,4 @@ public enum EHadoopVersion4Drivers {
         return ArrayUtils.contains(getMrVersions(), EMRVersion.YARN);
     }
 
-    public boolean isSupportSpark() {
-        boolean isSupport = false;
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
-            IHadoopDistributionService hadoopDistributionService = (IHadoopDistributionService) GlobalServiceRegister.getDefault()
-                    .getService(IHadoopDistributionService.class);
-            isSupport = hadoopDistributionService.isSupportSpark(this);
-        } else {
-            ExceptionHandler.process(new Exception(IHadoopDistributionService.class.getName() + " is not registed.")); //$NON-NLS-1$
-        }
-        return isSupport;
-    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/EMRVersion.java

@@ -18,12 +18,13 @@ import java.util.List;
 /**
  * created by ycbai on 2013-9-16 Detailled comment
  * 
+ * @deprecated @link EHadoopVersion instead.
  */
 public enum EMRVersion {
 
-    MR1,
+    MR1, // EHadoopVersion.HADOOP_1
 
-    YARN,
+    YARN, // EHadoopVersion.HADOOP_2
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/HadoopVersionDialog.java

@@ -32,14 +32,17 @@ import org.talend.commons.ui.runtime.image.ImageProvider;
 import org.talend.commons.ui.swt.formtools.LabelledCombo;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.hadoop.HadoopConstants;
+import org.talend.core.hadoop.IHadoopDistributionService;
 import org.talend.core.hadoop.IHadoopService;
-import org.talend.core.hadoop.version.EHadoopDistributions;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.runtime.CoreRuntimePlugin;
+import org.talend.core.runtime.hd.IDistributionsManager;
+import org.talend.core.runtime.hd.IHDConstants;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.IHDistributionVersion;
 import org.talend.core.runtime.i18n.Messages;
 
 /**
@@ -99,6 +102,13 @@ public class HadoopVersionDialog extends TitleAreaDialog {
         this.types = types;
     }
 
+    private IHadoopDistributionService getHadoopDistributionService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
+            return (IHadoopDistributionService) GlobalServiceRegister.getDefault().getService(IHadoopDistributionService.class);
+        }
+        return null;
+    }
+
     @Override
     protected void configureShell(Shell newShell) {
         super.configureShell(newShell);
@@ -267,16 +277,24 @@ public class HadoopVersionDialog extends TitleAreaDialog {
         }
     }
 
+    private IHDistribution getHadoopDistribution() {
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            final IHDistribution distributionByDisplay = hadoopDistributionService.getHadoopDistribution(
+                    distributionCombo.getText(), true);
+            return distributionByDisplay;
+        }
+        return null;
+    }
+
     private void addListener() {
         distributionCombo.addModifyListener(new ModifyListener() {
 
             @Override
             public void modifyText(final ModifyEvent e) {
-                String newDistributionDisplayName = distributionCombo.getText();
-                EHadoopDistributions newDistribution = EHadoopDistributions
-                        .getDistributionByDisplayName(newDistributionDisplayName);
-                if (newDistribution != null) {
-                    distribution = newDistribution.getName();
+                final IHDistribution distributionByDisplay = getHadoopDistribution();
+                if (distributionByDisplay != null) {
+                    distribution = distributionByDisplay.getName();
                     updateVersionPart();
                 }
             }
@@ -286,10 +304,14 @@ public class HadoopVersionDialog extends TitleAreaDialog {
 
             @Override
             public void modifyText(final ModifyEvent e) {
+                final IHDistribution distributionByDisplay = getHadoopDistribution();
                 String newVersionDisplayName = versionCombo.getText();
-                EHadoopVersion4Drivers newVersion4Drivers = EHadoopVersion4Drivers.indexOfByVersionDisplay(newVersionDisplayName);
-                if (newVersion4Drivers != null) {
-                    version = newVersion4Drivers.getVersionValue();
+
+                if (distributionByDisplay != null) {
+                    final IHDistributionVersion hdVersion = distributionByDisplay.getHDVersion(newVersionDisplayName, true);
+                    if (hdVersion != null) {
+                        version = hdVersion.getVersion();
+                    }
                 }
             }
         });
@@ -336,48 +358,27 @@ public class HadoopVersionDialog extends TitleAreaDialog {
     }
 
     private void init() {
-        distributionCombo.getCombo().setItems(getDistributions().toArray(new String[0]));
-        distributionCombo.select(0);
-    }
-
-    protected List<String> getDistributions() {
-        List<String> distributions = null;
-        if (isSparkJob()) {
-            EHadoopVersion4Drivers hadoopVersions[] = EHadoopVersion4Drivers.values();
-            Set<EHadoopDistributions> hadoopDistributions = new HashSet<EHadoopDistributions>();
-            for (EHadoopVersion4Drivers hadoopVersion : hadoopVersions) {
-                if (hadoopVersion.isSupportSpark()) {
-                    hadoopDistributions.add(hadoopVersion.getDistribution());
+        List<String> distributionsDisplay = new ArrayList<String>();
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            IDistributionsManager distributionManager = null;
+            if (isSparkJob()) {
+                distributionManager = hadoopDistributionService.getSparkDistributionManager();
+            } else {
+                distributionManager = hadoopDistributionService.getHadoopDistributionManager();
+            }
+            IHDistribution[] distributions = distributionManager.getDistributions();
+            if (distributions != null) {
+                for (IHDistribution d : distributions) {
+                    if (!d.useCustom()) {
+                        distributionsDisplay.add(d.getDisplayName());
+                    }
                 }
             }
-            distributions = new ArrayList<String>(hadoopDistributions.size());
-            for (EHadoopDistributions hadoopDistribution : hadoopDistributions) {
-                distributions.add(hadoopDistribution.getDisplayName());
-            }
-
-        } else {
-            distributions = EHadoopDistributions.getAllDistributionDisplayNames();
         }
-        distributions.remove(EHadoopDistributions.CUSTOM.getDisplayName());
 
-        return distributions;
-    }
-
-    protected List<String> getVersions(EHadoopDistributions dis) {
-        List<String> result = new ArrayList<String>();
-        boolean isSparkJob = isSparkJob();
-        List<EHadoopVersion4Drivers> v4dList = EHadoopVersion4Drivers.indexOfByDistribution(dis);
-        for (EHadoopVersion4Drivers v4d : v4dList) {
-            boolean needToAdd = true;
-            if (isSparkJob && !v4d.isSupportSpark()) {
-                needToAdd = false;
-            }
-
-            if (needToAdd) {
-                result.add(v4d.getVersionDisplay());
-            }
-        }
-        return result;
+        distributionCombo.getCombo().setItems(distributionsDisplay.toArray(new String[0]));
+        distributionCombo.select(0);
     }
 
     protected boolean isSparkJob() {
@@ -390,13 +391,34 @@ public class HadoopVersionDialog extends TitleAreaDialog {
     }
 
     private void updateVersionPart() {
-        EHadoopDistributions dis = EHadoopDistributions.getDistributionByName(distribution, false);
-        List<String> items = getVersions(dis);
-        String[] versions = new String[items.size()];
-        items.toArray(versions);
-        versionCombo.getCombo().setItems(versions);
-        if (versions.length > 0) {
-            versionCombo.getCombo().select(0);
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            final IHDistribution hDistribution = hadoopDistributionService.getHadoopDistribution(distribution, false);
+            if (hDistribution != null) {
+                boolean isSparkJob = isSparkJob();
+                final IHDistributionVersion[] hdVersions = hDistribution.getHDVersions();
+                List<String> versionsDisplay = new ArrayList<String>();
+                for (IHDistributionVersion hdv : hdVersions) {
+                    final String displayVersion = hdv.getDisplayVersion();
+                    if (displayVersion == null) {
+                        continue;
+                    }
+                    if (isSparkJob) {
+                        if (hadoopDistributionService.doSupportService(hdv, IHDConstants.SERVICE_SPARK)) {
+                            versionsDisplay.add(displayVersion);
+                        }
+                    } else {
+                        versionsDisplay.add(displayVersion);
+                    }
+                }
+                versionCombo.getCombo().setItems(versionsDisplay.toArray(new String[0]));
+                IHDistributionVersion defaultVersion = hDistribution.getDefaultVersion();
+                if (defaultVersion != null) {
+                    versionCombo.getCombo().setText(defaultVersion.getDisplayVersion());
+                } else if (!versionsDisplay.isEmpty()) {
+                    versionCombo.getCombo().select(0);
+                }
+            }
         }
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleNeeded.java

@@ -162,7 +162,12 @@ public class ModuleNeeded {
     public ModuleNeeded(String context, String informationMsg, boolean required, String mvnUri) {
         this(context, null, informationMsg, required, null, null, mvnUri, ELibraryInstallStatus.UNKNOWN);
         MavenArtifact mavenArtifact = MavenUrlHelper.parseMvnUrl(mvnUri);
-        setModuleName(mavenArtifact.getArtifactId() + "." + mavenArtifact.getType());
+        if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(mavenArtifact.getGroupId())
+                || StringUtils.isEmpty(mavenArtifact.getVersion())) {
+            setModuleName(mavenArtifact.getArtifactId() + "." + mavenArtifact.getType()); //$NON-NLS-1$
+        } else {
+            setModuleName(mavenArtifact.getArtifactId() + "-" + mavenArtifact.getVersion() + "." + mavenArtifact.getType());  //$NON-NLS-1$//$NON-NLS-2$
+        }
 
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -253,6 +253,7 @@ public class MetadataTable implements IMetadataTable, Cloneable {
                                 if (!otherColumn.sameMetacolumnAs(myColumn, options)) {
                                     return false;
                                 }
+                                break;
                             }
                         }
                         if (!exist) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolAvroHelper.java

@@ -0,0 +1,579 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.metadata;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.SchemaBuilder.BaseFieldTypeBuilder;
+import org.apache.avro.SchemaBuilder.FieldAssembler;
+import org.apache.avro.SchemaBuilder.FieldBuilder;
+import org.apache.avro.SchemaBuilder.PropBuilder;
+import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.talend.core.model.metadata.builder.connection.ConnectionFactory;
+import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.types.JavaTypesManager;
+import org.talend.core.utils.TalendQuoteUtils;
+import org.talend.cwm.helper.TaggedValueHelper;
+import org.talend.daikon.avro.util.AvroTypes;
+import org.talend.daikon.avro.util.AvroUtils;
+import org.talend.daikon.talend6.Talend6SchemaConstants;
+import orgomg.cwm.objectmodel.core.Expression;
+import orgomg.cwm.objectmodel.core.TaggedValue;
+
+/**
+ * Utility classes for conversion between a {@link MetadataTable} and an Avro {@link Schema}.
+ */
+public final class MetadataToolAvroHelper {
+
+    /**
+     * @return An Avro schema with enriched properties from the incoming metadata table.
+     */
+    public static org.apache.avro.Schema convertToAvro(MetadataTable in) {
+        RecordBuilder<Schema> builder = SchemaBuilder.builder().record(in.getLabel());
+        copyTableProperties(builder, in);
+
+        FieldAssembler<Schema> fa = builder.fields();
+        int dynamicPosition = -1;
+        org.talend.core.model.metadata.builder.connection.MetadataColumn dynColumn = null;
+        int i = 0;
+        for (org.talend.core.model.metadata.builder.connection.MetadataColumn column : in.getColumns()) {
+            if ("id_Dynamic".equals(column.getTalendType())) { //$NON-NLS-1$
+                dynamicPosition = i;
+                dynColumn = column;
+            } else {
+                fa = convertToAvro(fa, column);
+            }
+            i++;
+        }
+
+        Schema schema = fa.endRecord();
+
+        if (dynColumn != null) {
+            // store all the dynamic column's properties
+            schema = copyDynamicColumnProperties(schema, dynColumn);
+            // store dynamic position
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION,
+                    String.valueOf(dynamicPosition));
+            // tag avro schema with include-all-columns
+            schema = AvroUtils.setIncludeAllFields(schema, true);
+        }
+        return schema;
+    }
+
+    /**
+     * Copy all of the information from the MetadataTable in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyTableProperties(PropBuilder<T> builder, MetadataTable in) {
+
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Table-specific properties.
+        if (in.getName() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_NAME, in.getName());
+        }
+        if (in.getTableType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_TYPE, in.getTableType());
+        }
+
+        return builder;
+    }
+
+    /**
+     * Build a field into a schema using enriched properties from the incoming column.
+     */
+    private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        FieldBuilder<Schema> fb = fa.name(in.getLabel());
+        copyColumnProperties(fb, in);
+        BaseFieldTypeBuilder<Schema> ftb = in.isNullable() ? fb.type().nullable() : fb.type();
+
+        Object defaultValue = null;
+        Expression initialValue = in.getInitialValue();
+        if (initialValue != null) {
+            defaultValue = initialValue.getBody();
+        }
+
+        String tt = in.getTalendType();
+
+        Schema type = null;
+        // Numeric types.
+        if (JavaTypesManager.LONG.getId().equals(tt)) {
+            type = AvroTypes._long();
+            defaultValue = defaultValue == null ? null : Long.parseLong(defaultValue.toString());
+        } else if (JavaTypesManager.INTEGER.getId().equals(tt)) {
+            type = AvroTypes._int();
+            defaultValue = defaultValue == null ? null : Integer.parseInt(defaultValue.toString());
+        } else if (JavaTypesManager.SHORT.getId().equals(tt)) {
+            type = AvroTypes._short();
+            defaultValue = defaultValue == null ? null : Integer.parseInt(defaultValue.toString());
+        } else if (JavaTypesManager.BYTE.getId().equals(tt)) {
+            type = AvroTypes._byte();
+            defaultValue = defaultValue == null ? null : Integer.parseInt(defaultValue.toString());
+        } else if (JavaTypesManager.DOUBLE.getId().equals(tt)) {
+            type = AvroTypes._double();
+            defaultValue = defaultValue == null ? null : Double.parseDouble(defaultValue.toString());
+        } else if (JavaTypesManager.FLOAT.getId().equals(tt)) {
+            type = AvroTypes._float();
+            defaultValue = defaultValue == null ? null : Float.parseFloat(defaultValue.toString());
+        } else if (JavaTypesManager.BIGDECIMAL.getId().equals(tt)) {
+            // decimal(precision, scale) == column length and precision?
+            type = AvroTypes._decimal();
+        }
+
+        // Other primitive types that map directly to Avro.
+        else if (JavaTypesManager.BOOLEAN.getId().equals(tt)) {
+            type = AvroTypes._boolean();
+            defaultValue = defaultValue == null ? null : Boolean.parseBoolean(defaultValue.toString());
+        } else if (JavaTypesManager.BYTE_ARRAY.getId().equals(tt)) {
+            type = AvroTypes._bytes();
+        } else if (JavaTypesManager.DATE.getId().equals(tt)) {
+            type = AvroTypes._date();
+        }
+        // String-ish types.
+        else if (JavaTypesManager.STRING.getId().equals(tt) || JavaTypesManager.FILE.getId().equals(tt)
+                || JavaTypesManager.DIRECTORY.getId().equals(tt) || JavaTypesManager.VALUE_LIST.getId().equals(tt)
+                || JavaTypesManager.CHARACTER.getId().equals(tt) || JavaTypesManager.PASSWORD.getId().equals(tt)) {
+            type = AvroTypes._string();
+        }
+
+        // Types with unknown elements, store as binary
+        if (JavaTypesManager.OBJECT.getId().equals(tt)) {
+            // FIXME it's not right, as it don't store all the information about the object
+        }
+
+        if (JavaTypesManager.LIST.getId().equals(tt)) {
+            // FIXME it's not right, as it don't store all the information about the object
+        }
+        // Can this occur?
+        if (type == null) {
+            throw new UnsupportedOperationException("Unrecognized type " + tt); //$NON-NLS-1$
+        }
+
+        type = in.isNullable() ? AvroUtils.wrapAsNullable(type) : type;
+        return defaultValue == null ? fb.type(type).noDefault() : fb.type(type).withDefault(defaultValue);
+    }
+
+    private static Schema copyDynamicColumnProperties(Schema schema,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        Map<String, String> props = new HashMap<String, String>();
+        if (in.getId() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_NAME, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_DYNAMIC_ADDITIONAL_PROPERTIES
+                        + additionalTag, tv.getValue());
+            }
+        }
+
+        // Column-specific properties.
+        if (in.isKey()) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
+        }
+        if (in.getType() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
+        }
+        if (in.getTalendType() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
+        }
+        if (in.getPattern() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_PATTERN,
+                    TalendQuoteUtils.removeQuotesIfExist(in.getPattern()));
+        }
+        if (in.getLength() >= 0) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_LENGTH,
+                    String.valueOf((int) in.getLength()));
+        }
+        if (in.getOriginalLength() >= 0) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH,
+                    String.valueOf(in.getOriginalLength()));
+        }
+        if (in.isNullable()) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
+        }
+        if (in.getPrecision() >= 0) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_PRECISION,
+                    String.valueOf(in.getPrecision()));
+        }
+        if (in.getInitialValue() != null && in.getInitialValue().getBody() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
+        }
+        if (in.getName() != null) {
+            // keyword fixes?
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
+        }
+        if (in.getRelatedEntity() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
+        }
+        if (in.getRelationshipType() != null) {
+            schema = AvroUtils.setProperty(schema, Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE,
+                    in.getRelationshipType());
+        }
+        return schema;
+    }
+
+    /**
+     * Copy all of the information from the IMetadataColumn in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyColumnProperties(PropBuilder<T> builder,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Column-specific properties.
+        if (in.isKey()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
+        }
+        if (in.getType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
+        }
+        if (in.getTalendType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
+        }
+        if (in.getPattern() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN, TalendQuoteUtils.removeQuotesIfExist(in.getPattern()));
+        }
+        if (in.getLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH, String.valueOf((int) in.getLength()));
+        }
+        if (in.getOriginalLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH, String.valueOf(in.getOriginalLength()));
+        }
+        if (in.isNullable()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
+        }
+        if (in.getPrecision() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION, String.valueOf(in.getPrecision()));
+        }
+        if (in.getInitialValue() != null && in.getInitialValue().getBody() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
+        }
+        if (in.getName() != null) {
+            // keyword fixes?
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
+        }
+        if (in.getRelatedEntity() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
+        }
+        if (in.getRelationshipType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE, in.getRelationshipType());
+        }
+        return builder;
+    }
+
+    /**
+     * @param in A schema which must be of type {@link Schema.Type#RECORD}.
+     * @return A MetadataTable containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static MetadataTable convertFromAvro(org.apache.avro.Schema in) {
+        MetadataTable table = ConnectionFactory.eINSTANCE.createMetadataTable();
+
+        // Set the defaults values to the name (the only information guaranteed to be present in the schema.)
+        table.setId(in.getName());
+        table.setLabel(in.getName());
+        table.setName(in.getName());
+        table.setSourceName(in.getName());
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            table.setId(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            table.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            table.setLabel(null);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            table.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : in.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
+                table.getTaggedValue().add(tv);
+            }
+        }
+
+        // Table-specific properties.
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_NAME))) {
+            table.setName(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_TYPE))) {
+            table.setTableType(prop);
+        }
+
+        // Add the columns.
+        List<org.talend.core.model.metadata.builder.connection.MetadataColumn> columns = new ArrayList<>(in.getFields().size());
+        for (Schema.Field f : in.getFields()) {
+            columns.add(convertFromAvro(f));
+        }
+        boolean isDynamic = AvroUtils.isIncludeAllFields(in);
+        if (isDynamic) {
+            org.talend.core.model.metadata.builder.connection.MetadataColumn col = convertFromAvroForDynamic(in);
+            // get dynamic position
+            int dynPosition = Integer.valueOf(in.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION));
+            columns.add(dynPosition, col);
+        }
+        table.getColumns().addAll(columns);
+        return table;
+    }
+
+    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvroForDynamic(Schema schema) {
+        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
+        String prop;
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_ID))) {
+            col.setId(prop);
+        }
+
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_COMMENT))) {
+            col.setComment(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_COLUMN_NAME))) {
+            col.setLabel(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_DYNAMIC_IS_READ_ONLY))) {
+            col.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : schema.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, schema.getProp(key));
+                col.getTaggedValue().add(tv);
+            }
+        }
+
+        // Column-specific properties.
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
+            col.setKey(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
+            col.setSourceType(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
+            col.setTalendType(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
+            col.setPattern(TalendQuoteUtils.addQuotesIfNotExist(prop));
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setOriginalLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
+            col.setNullable(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
+            Long value = Long.parseLong(prop);
+            col.setPrecision(value > 0 ? value : -1);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
+            col.setDefaultValue(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
+            col.setName(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
+            col.setRelatedEntity(prop);
+        }
+        if (null != (prop = schema.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
+            col.setRelationshipType(prop);
+        }
+
+        col.setTalendType("id_Dynamic"); //$NON-NLS-1$
+        return col;
+    }
+
+    /**
+     * @param in A field from an incoming schema
+     * @return A MetadataColumn containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvro(Schema.Field field) {
+        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
+        Schema in = field.schema();
+
+        // Set the defaults values to the name (the only information guaranteed to be available in every field).
+        col.setId(field.name());
+        col.setLabel(field.name());
+        col.setName(field.name());
+        Schema nonnullable = AvroUtils.unwrapIfNullable(in);
+        if (AvroTypes.isSameType(nonnullable, AvroTypes._boolean())) {
+            col.setTalendType(JavaTypesManager.BOOLEAN.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._byte())) {
+            col.setTalendType(JavaTypesManager.BYTE.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._bytes())) {
+            col.setTalendType(JavaTypesManager.BYTE_ARRAY.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._character())) {
+            col.setTalendType(JavaTypesManager.CHARACTER.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._date())) {
+            col.setTalendType(JavaTypesManager.DATE.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._decimal())) {
+            col.setTalendType(JavaTypesManager.BIGDECIMAL.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._double())) {
+            col.setTalendType(JavaTypesManager.DOUBLE.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._float())) {
+            col.setTalendType(JavaTypesManager.FLOAT.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._int())) {
+            col.setTalendType(JavaTypesManager.INTEGER.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._long())) {
+            col.setTalendType(JavaTypesManager.LONG.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._short())) {
+            col.setTalendType(JavaTypesManager.SHORT.getId());
+        } else if (AvroTypes.isSameType(nonnullable, AvroTypes._string())) {
+            col.setTalendType(JavaTypesManager.STRING.getId());
+        }
+        // FIXME missing List and Object here
+
+        // TODO setSourceType from the field Schema type.
+        col.setNullable(AvroUtils.isNullable(in));
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            col.setId(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            col.setComment(field.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            col.setLabel(null);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            col.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : field.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, field.getProp(key));
+                col.getTaggedValue().add(tv);
+            }
+        }
+
+        // Column-specific properties.
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
+            col.setKey(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
+            col.setSourceType(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
+            col.setTalendType(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
+            col.setPattern(TalendQuoteUtils.addQuotesIfNotExist(prop));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setOriginalLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
+            col.setNullable(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
+            Long value = Long.parseLong(prop);
+            col.setPrecision(value > 0 ? value : -1);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
+            col.setDefaultValue(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
+            col.setName(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
+            col.setRelatedEntity(prop);
+        }
+        if (null != (prop = field.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
+            col.setRelationshipType(prop);
+        }
+
+        // If the source type wasn't set, there is an issue. Can this occur in the studio.
+        if (col.getTalendType() == null) {
+            throw new UnsupportedOperationException("Unrecognized type " + in); //$NON-NLS-1$
+        }
+
+        return col;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -24,16 +24,6 @@ import java.util.Set;
 import java.util.Vector;
 import java.util.regex.Pattern;
 
-import org.apache.avro.LogicalTypes;
-import org.apache.avro.LogicalTypes.Decimal;
-
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaBuilder;
-import org.apache.avro.SchemaBuilder.BaseFieldTypeBuilder;
-import org.apache.avro.SchemaBuilder.FieldAssembler;
-import org.apache.avro.SchemaBuilder.FieldBuilder;
-import org.apache.avro.SchemaBuilder.PropBuilder;
-import org.apache.avro.SchemaBuilder.RecordBuilder;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.emf.common.util.BasicEList;
@@ -57,7 +47,6 @@ import org.talend.core.language.ECodeLanguage;
 import org.talend.core.language.LanguageManager;
 import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
 import org.talend.core.model.metadata.builder.connection.Connection;
-import org.talend.core.model.metadata.builder.connection.ConnectionFactory;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.metadata.builder.connection.QueriesConnection;
 import org.talend.core.model.metadata.builder.connection.Query;
@@ -82,9 +71,6 @@ import org.talend.core.runtime.i18n.Messages;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.utils.KeywordsValidator;
 import org.talend.cwm.helper.ConnectionHelper;
-import org.talend.cwm.helper.TaggedValueHelper;
-import org.talend.daikon.avro.util.AvroUtils;
-import org.talend.daikon.talend6.Talend6SchemaConstants;
 import org.talend.designer.core.model.utils.emf.talendfile.ColumnType;
 import org.talend.designer.core.model.utils.emf.talendfile.MetadataType;
 import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
@@ -618,6 +604,7 @@ public final class MetadataToolHelper {
         target.getListColumns().addAll(columnsTAdd);
         target.sortCustomColumns();
         target.setLabel(source.getLabel());
+        target.getAdditionalProperties().putAll(source.getAdditionalProperties());
     }
 
     public static void copyTable(List<IMetadataColumn> sourceColumns, IMetadataTable target, List<IMetadataColumn> targetColumns) {
@@ -1078,7 +1065,8 @@ public final class MetadataToolHelper {
         String uniqueName = null;
         for (int i = 0; i < elementParameters.size(); i++) {
             IElementParameter param = elementParameters.get(i);
-            if (param.getFieldType().equals(EParameterFieldType.SCHEMA_TYPE)
+            if ((param.getFieldType().equals(EParameterFieldType.SCHEMA_TYPE) || param.getFieldType().equals(
+                    EParameterFieldType.SCHEMA_REFERENCE))
                     && param.getContext().equals(metadataTable.getAttachedConnector())) {
                 if (param.getValue() instanceof IMetadataTable) {
                     param.setValueToDefault(elementParameters);
@@ -1411,387 +1399,4 @@ public final class MetadataToolHelper {
         }
         return result;
     }
-
-    /**
-     * @return An Avro schema with enriched properties from the incoming metadata table.
-     */
-    public static org.apache.avro.Schema convertToAvro(MetadataTable in) {
-        RecordBuilder<Schema> builder = SchemaBuilder.builder().record(in.getLabel());
-        copyTableProperties(builder, in);
-
-        FieldAssembler<Schema> fa = builder.fields();
-        for (org.talend.core.model.metadata.builder.connection.MetadataColumn column : in.getColumns()) {
-            fa = convertToAvro(fa, column);
-        }
-        return fa.endRecord();
-    }
-
-    /**
-     * Copy all of the information from the MetadataTable in the form of key/value properties into an Avro object.
-     * 
-     * @param builder Any Avro builder capable of taking key/value in the form of strings.
-     * @param in The element to copy information from.
-     * @return the instance of the builder passed in.
-     */
-    private static <T extends PropBuilder<T>> PropBuilder<T> copyTableProperties(PropBuilder<T> builder, MetadataTable in) {
-
-        // Properties common to tables and columns.
-        if (in.getId() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
-        }
-        if (in.getComment() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
-        }
-        if (in.getLabel() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
-        }
-        if (in.isReadOnly()) {
-            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
-        }
-        for (TaggedValue tv : in.getTaggedValue()) {
-            String additionalTag = tv.getTag();
-            if (tv.getValue() != null) {
-                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
-            }
-        }
-
-        // Table-specific properties.
-        if (in.getName() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_NAME, in.getName());
-        }
-        if (in.getTableType() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_TYPE, in.getTableType());
-        }
-
-        return builder;
-    }
-
-    /**
-     * Build a field into a schema using enriched properties from the incoming column.
-     */
-    private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
-            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
-        FieldBuilder<Schema> fb = fa.name(in.getLabel());
-        copyColumnProperties(fb, in);
-        BaseFieldTypeBuilder<Schema> ftb = in.isNullable() ? fb.type() : fb.type().nullable();
-
-        String defaultValue = in.getInitialValue().getBody();
-
-        String tt = in.getTalendType();
-
-        // Numeric types.
-        if (JavaTypesManager.LONG.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.longType().noDefault()
-                    : ftb.longType().longDefault(Long.parseLong(defaultValue));
-        }
-        if (JavaTypesManager.INTEGER.getId().equals(tt) || JavaTypesManager.SHORT.getId().equals(tt)
-                || JavaTypesManager.BYTE.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.intType().noDefault()
-                    : ftb.intType().intDefault(Integer.parseInt(defaultValue));
-        }
-        if (JavaTypesManager.DOUBLE.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.doubleType().noDefault()
-                    : ftb.doubleType().doubleDefault(Double.parseDouble(defaultValue));
-        }
-        if (JavaTypesManager.FLOAT.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.floatType().noDefault()
-                    : ftb.floatType().floatDefault(Float.parseFloat(defaultValue));
-        }
-        if (JavaTypesManager.BIGDECIMAL.getId().equals(tt)) {
-            // decimal(precision, scale) == column length and precision?
-            Decimal d = LogicalTypes.decimal((int) in.getLength(), (int) in.getPrecision());
-            Schema bigdecimal = d.addToSchema(Schema.create(Schema.Type.BYTES));
-            return fb.type(bigdecimal).withDefault(defaultValue);
-        }
-
-        // Other primitive types that map directly to Avro.
-        if (JavaTypesManager.BOOLEAN.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.booleanType().noDefault()
-                    : ftb.booleanType().booleanDefault(Boolean.parseBoolean(defaultValue));
-        }
-        if (JavaTypesManager.BYTE_ARRAY.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.bytesType().noDefault()
-                    : ftb.bytesType().bytesDefault(defaultValue);
-        }
-        if (JavaTypesManager.DATE.getId().equals(tt)) {
-            // Date is saved as a long, like System.currentTimeMillis()
-            return ftb.longType().noDefault();
-        }
-        // String-ish types.
-        if (JavaTypesManager.STRING.getId().equals(tt) || JavaTypesManager.FILE.getId().equals(tt)
-                || JavaTypesManager.DIRECTORY.getId().equals(tt) || JavaTypesManager.VALUE_LIST.getId().equals(tt)
-                || JavaTypesManager.CHARACTER.getId().equals(tt) || JavaTypesManager.PASSWORD.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.stringType().noDefault()
-                    : ftb.stringType().stringDefault(defaultValue);
-        }
-
-        // Types with unknown elements, store as binary
-        if (JavaTypesManager.OBJECT.getId().equals(tt) || JavaTypesManager.DYNAMIC.getId().equals(tt)) {
-            return defaultValue == null //
-            ? ftb.bytesType().noDefault()
-                    : ftb.bytesType().bytesDefault(defaultValue);
-        }
-        if (JavaTypesManager.LIST.getId().equals(tt)) {
-            return ftb.array().items().bytesType().noDefault();
-        }
-
-        // Can this occur?
-        throw new UnsupportedOperationException("Unrecognized type " + tt); //$NON-NLS-1$
-    }
-
-    /**
-     * Copy all of the information from the IMetadataColumn in the form of key/value properties into an Avro object.
-     * 
-     * @param builder Any Avro builder capable of taking key/value in the form of strings.
-     * @param in The element to copy information from.
-     * @return the instance of the builder passed in.
-     */
-    private static <T extends PropBuilder<T>> PropBuilder<T> copyColumnProperties(PropBuilder<T> builder,
-            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
-        // Properties common to tables and columns.
-        if (in.getId() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
-        }
-        if (in.getComment() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
-        }
-        if (in.getLabel() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
-        }
-        if (in.isReadOnly()) {
-            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
-        }
-        for (TaggedValue tv : in.getTaggedValue()) {
-            String additionalTag = tv.getTag();
-            if (tv.getValue() != null) {
-                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
-            }
-        }
-
-        // Column-specific properties.
-        if (in.isKey()) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
-        }
-        if (in.getType() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
-        }
-        if (in.getTalendType() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
-        }
-        if (in.getPattern() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN, in.getPattern());
-        }
-        if (in.getLength() >= 0) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH, String.valueOf((int) in.getLength()));
-        }
-        if (in.getOriginalLength() >= 0) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH, String.valueOf(in.getOriginalLength()));
-        }
-        if (in.isNullable()) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
-        }
-        if (in.getPrecision() >= 0) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION, String.valueOf(in.getPrecision()));
-        }
-        if (in.getInitialValue().getBody() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
-        }
-        if (in.getName() != null) {
-            // keyword fixes?
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
-        }
-        if (in.getRelatedEntity() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
-        }
-        if (in.getRelationshipType() != null) {
-            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE, in.getRelationshipType());
-        }
-        return builder;
-    }
-
-    /**
-     * @param in A schema which must be of type {@link Schema.Type#RECORD}.
-     * @return A MetadataTable containing all the information from the Schema, including any information included the
-     * schema as JSON property annotations for Talend 6 generated schemas.
-     */
-    public static MetadataTable convertFromAvro(org.apache.avro.Schema in) {
-        MetadataTable table = ConnectionFactory.eINSTANCE.createMetadataTable();
-
-        // Set the defaults values to the name (the only information guaranteed to be present in the schema.)
-        table.setId(in.getName());
-        table.setLabel(in.getName());
-        table.setName(in.getName());
-        table.setSourceName(in.getName());
-
-        // Extract any properties that have been saved in the enriched schema.
-
-        // Properties common to tables and columns.
-        String prop;
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
-            table.setId(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
-            table.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
-            table.setLabel(null);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
-            table.setReadOnly(Boolean.parseBoolean(prop));
-        }
-        for (String key : in.getJsonProps().keySet()) {
-            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
-                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
-                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
-                table.getTaggedValue().add(tv);
-            }
-        }
-
-        // Table-specific properties.
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_NAME))) {
-            table.setName(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_TYPE))) {
-            table.setTableType(prop);
-        }
-
-        // Add the columns.
-        List<org.talend.core.model.metadata.builder.connection.MetadataColumn> columns = new ArrayList<>(in.getFields().size());
-        for (Schema.Field f : in.getFields()) {
-            columns.add(convertFromAvro(f));
-        }
-        table.getColumns().addAll(columns);
-        return table;
-    }
-
-    /**
-     * @param in A field from an incoming schema
-     * @return A MetadataColumn containing all the information from the Schema, including any information included the
-     * schema as JSON property annotations for Talend 6 generated schemas.
-     */
-    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvro(Schema.Field field) {
-        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
-        Schema in = field.schema();
-
-        // Set the defaults values to the name (the only information guaranteed to be available in every field).
-        col.setId(field.name());
-        col.setLabel(field.name());
-        col.setName(field.name());
-        switch (AvroUtils.unwrapIfNullable(in).getType()) {
-        case ARRAY:
-            col.setSourceType(JavaTypesManager.LIST.getId());
-            break;
-        case BOOLEAN:
-            col.setSourceType(JavaTypesManager.BOOLEAN.getId());
-            break;
-        case BYTES:
-        case FIXED:
-            col.setSourceType(JavaTypesManager.BYTE_ARRAY.getId());
-            break;
-        case DOUBLE:
-            col.setSourceType(JavaTypesManager.DOUBLE.getId());
-            break;
-        case FLOAT:
-            col.setSourceType(JavaTypesManager.FLOAT.getId());
-            break;
-        case INT:
-            col.setSourceType(JavaTypesManager.INTEGER.getId());
-            break;
-        case LONG:
-            col.setSourceType(JavaTypesManager.LONG.getId());
-            break;
-        case ENUM:
-        case STRING:
-            col.setSourceType(JavaTypesManager.STRING.getId());
-            break;
-        case RECORD:
-        case NULL:
-        case MAP:
-        case UNION:
-        default:
-            // Can this occur in the studio?
-            break;
-        }
-        // TODO setSourceType from the field Schema type.
-        col.setNullable(AvroUtils.isNullable(in));
-
-        // Extract any properties that have been saved in the enriched schema.
-
-        // Properties common to tables and columns.
-        String prop;
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
-            col.setId(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
-            col.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
-            col.setLabel(null);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
-            col.setReadOnly(Boolean.parseBoolean(prop));
-        }
-        for (String key : in.getJsonProps().keySet()) {
-            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
-                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
-                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
-                col.getTaggedValue().add(tv);
-            }
-        }
-
-        // Column-specific properties.
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
-            col.setKey(Boolean.parseBoolean(prop));
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
-            col.setSourceType(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
-            col.setTalendType(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
-            col.setPattern(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
-            Long value = Long.parseLong(prop);
-            col.setLength(value > 0 ? value : -1);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
-            Long value = Long.parseLong(prop);
-            col.setOriginalLength(value > 0 ? value : -1);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
-            col.setNullable(Boolean.parseBoolean(prop));
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
-            Long value = Long.parseLong(prop);
-            col.setPrecision(value > 0 ? value : -1);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
-            col.setDefaultValue(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
-            col.setName(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
-            col.setRelatedEntity(prop);
-        }
-        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
-            col.setRelationshipType(prop);
-        }
-
-        // If the source type wasn't set, there is an issue. Can this occur in the studio.
-        if (col.getSourceType() == null) {
-            throw new UnsupportedOperationException("Unrecognized type " + in); //$NON-NLS-1$
-        }
-
-        return col;
-    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/builder/ConvertionHelper.java

@@ -40,9 +40,11 @@ import org.talend.core.model.metadata.builder.connection.DelimitedFileConnection
 import org.talend.core.model.metadata.builder.connection.MDMConnection;
 import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.utils.KeywordsValidator;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.model.bridge.ReponsitoryContextBridge;
 import orgomg.cwm.objectmodel.core.TaggedValue;
 
@@ -337,6 +339,16 @@ public final class ConvertionHelper {
         result.setId(old.getId());
         result.setLabel(old.getLabel());
         result.setTableType(old.getTableType());
+        if (old instanceof SAPBWTable) {
+            String innerType = ((SAPBWTable) old).getInnerIOType();
+            String sourceSysName = ((SAPBWTable) old).getSourceSystemName();
+            if (innerType != null) {
+                result.getAdditionalProperties().put(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE, innerType);
+            }
+            if (sourceSysName != null) {
+                result.getAdditionalProperties().put(SAPBWTableHelper.SAP_DATASOURCE_SOURCESYSNAME, sourceSysName);
+            }
+        }
         String sourceName = old.getName();
         if (sourceName == null) {
             sourceName = old.getLabel();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveConnUtils.java

@@ -1,503 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.model.metadata.connection.hive;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang.ArrayUtils;
-
-/**
- * @author Marvin Wang
- * @version 1.0 jdk1.6
- * @date Aug 9, 2012
- */
-public class HiveConnUtils {
-
-    /**
-     * Returns the distribution names that can be used to show in UI. If no items, return <code>new String[0]</code>.
-     * Added by Marvin Wang on Aug. 9, 2012.
-     * 
-     * @see #getDistributionNameList()
-     * @return
-     */
-    public static String[] getDistributionNames() {
-        List<String> list = getDistributionNameList();
-        if (list == null) {
-            return new String[0];
-        } else {
-            return list.toArray(new String[list.size()]);
-        }
-    }
-
-    /**
-     * Returns the name list of distribution. If no items, return <code>null</code>. These names are from
-     * {@link HiveConnVersionInfo} that defines the level <code>0</code>. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @see #extractVersionInfo(int)
-     * @return
-     */
-    public static List<String> getDistributionNameList() {
-        return getNameListByLevel(0);
-    }
-
-    /**
-     * Returns the children of the given object, refer to the method {@link #getFollowersByKeyAndLevel(String, int)}.
-     * Added by Marvin Wang on Aug 10, 2012.
-     * 
-     * @param obj
-     * @return
-     */
-    protected static List<HiveConnVersionInfo> getFollowersOfObject(HiveConnVersionInfo obj) {
-        String key = obj.getKey();
-        int level = obj.getLevel();
-        List<HiveConnVersionInfo> beans = getObjectsByLevel(level + 1);
-        if (beans != null && beans.size() > 0) {
-            List<HiveConnVersionInfo> listFollowingKey = new ArrayList<HiveConnVersionInfo>();
-            for (HiveConnVersionInfo bean : beans) {
-                extractObjectFollowsKey(key, bean, listFollowingKey);
-            }
-            return listFollowingKey;
-        }
-        return null;
-    }
-
-    /**
-     * If the object from the given array <code>HiveConnVersionInfo[]</code> includes the key, then put the object into
-     * the given list to store. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param key
-     * @param beans
-     * @param listFollowingKey
-     */
-    protected static void extractObjectFollowsKey(String key, HiveConnVersionInfo bean, List<HiveConnVersionInfo> listFollowingKey) {
-        HiveConnVersionInfo[] followsBeans = bean.getFollows();
-        if (followsBeans != null && followsBeans.length > 0) {
-            for (HiveConnVersionInfo followsBean : followsBeans) {
-                if (followsBean.getKey().equals(key)) {
-                    listFollowingKey.add(bean);
-                }
-            }
-        }
-    }
-
-    /**
-     * Returns all names with a specified level. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param level
-     * @return
-     */
-    protected static List<String> getNameListByLevel(int level) {
-        List<HiveConnVersionInfo> beans = getObjectsByLevel(level);
-        if (beans != null && beans.size() > 0) {
-            List<String> nameList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : beans) {
-                nameList.add(bean.getDisplayName());
-            }
-            return nameList;
-        }
-        return null;
-    }
-
-    /**
-     * Returns all {@link HiveConnVersionInfo} objects with the given level. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param level
-     * @return
-     */
-    protected static List<HiveConnVersionInfo> getObjectsByLevel(int level) {
-        HiveConnVersionInfo[] beans = HiveConnVersionInfo.values();
-        if (beans != null && beans.length > 0) {
-            List<HiveConnVersionInfo> list = new ArrayList<HiveConnVersionInfo>();
-            for (HiveConnVersionInfo bean : beans) {
-                int beanLevel = bean.getLevel();
-                if (level == beanLevel) {
-                    list.add(bean);
-                }
-            }
-            // ADD msjian TDQ-6407 2012-11-26:for top not support hive embedded mode
-            // if (PluginChecker.isOnlyTopLoaded() && (level == 0 || level == 2)) {
-            // list.remove(0);
-            // }
-            // TDQ-6407~
-            return list;
-        }
-        return null;
-    }
-
-    /**
-     * Returns the index of distribution that is level 0 by display name. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param displayName
-     * @return
-     */
-    public static int getDistributionIndex(String displayName) {
-        String[] names = getDistributionNames();
-        if (names != null && names.length > 0) {
-            for (int i = 0; i < names.length; i++) {
-                if (displayName != null && displayName.equals(names[i])) {
-                    return i;
-                }
-            }
-        }
-        return -1;
-    }
-
-    /**
-     * Returns the key by display name and level. Added by Marvin Wang on Aug 9, 2012.
-     * 
-     * @param displayName
-     * @param level
-     * @return
-     */
-    protected static String getKeyByDisplayName(String displayName, int level) {
-        List<HiveConnVersionInfo> beans = getObjectsByLevel(level);
-        if (beans != null && beans.size() > 0) {
-            for (HiveConnVersionInfo bean : beans) {
-                if (displayName != null && displayName.equals(bean.getDisplayName())) {
-                    return bean.getKey();
-                }
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Returns all hive versions by distribution index, it means these hive versions follow the distribution. Added by
-     * Marvin Wang on Aug 10, 2012.
-     * 
-     * @param level0Index
-     * @return
-     */
-    protected static List<HiveConnVersionInfo> getHiveVersions(int distributionIndex) {
-        List<HiveConnVersionInfo> level0Objects = getObjectsByLevel(0);
-        HiveConnVersionInfo objOfLevel0ByIndex = level0Objects.get(distributionIndex);
-        return getFollowersOfObject(objOfLevel0ByIndex);
-    }
-
-    protected static List<String> getHiveVersionNameList(int distributionIndex) {
-        List<HiveConnVersionInfo> hiveVersionObjs = getHiveVersions(distributionIndex);
-        if (hiveVersionObjs != null && hiveVersionObjs.size() > 0) {
-            List<String> hiveVersionNameList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : hiveVersionObjs) {
-                hiveVersionNameList.add(bean.getDisplayName());
-            }
-            return hiveVersionNameList;
-        }
-        return null;
-    }
-
-    public static String[] getHiveVersionNames(int distributionIndex) {
-        List<String> hiveVersionNameList = getHiveVersionNameList(distributionIndex);
-        if (hiveVersionNameList != null && hiveVersionNameList.size() > 0) {
-            return hiveVersionNameList.toArray(new String[hiveVersionNameList.size()]);
-        } else {
-            return new String[0];
-        }
-    }
-
-    protected static List<HiveConnVersionInfo> getHiveModes(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> supportedModes = new ArrayList<HiveConnVersionInfo>();
-        HiveConnVersionInfo hiveVersionObj = getHiveVersionObj(distributionIndex, versionIndex);
-        if (hiveVersionObj != null) {
-            boolean supportStandalone = isSupportStandalone(hiveVersionObj, hiveServerIndex);
-            List<HiveConnVersionInfo> modes = getFollowersOfObject(hiveVersionObj);
-            if (modes != null && modes.size() > 0) {
-                for (HiveConnVersionInfo mode : modes) {
-                    if (HiveConnVersionInfo.MODE_EMBEDDED.equals(mode) || HiveConnVersionInfo.MODE_STANDALONE.equals(mode)
-                            && supportStandalone) {
-                        supportedModes.add(mode);
-                    }
-                }
-            }
-        }
-
-        return supportedModes;
-    }
-
-    private static boolean isSupportStandalone(HiveConnVersionInfo hiveVersionObj, int hiveServerIndex) {
-        boolean isHiveServer1 = "HIVE".equals(HiveServerVersionUtils.extractKey(hiveServerIndex)); //$NON-NLS-1$
-        return !(isHiveServer1 && (HiveConnVersionInfo.HDP_1_2.equals(hiveVersionObj)
-                || HiveConnVersionInfo.HDP_1_3.equals(hiveVersionObj) || HiveConnVersionInfo.HDP_2_0.equals(hiveVersionObj)));
-    }
-
-    protected static List<String> getHiveModeNameList(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> hiveModeObjs = getHiveModes(distributionIndex, versionIndex, hiveServerIndex);
-        // ADD msjian TDQ-6407 2012-11-26: for top, not support hive embeded mode,hide this menu
-        // if (PluginChecker.isOnlyTopLoaded() && hiveModeObjs.size() > 1) {
-        // hiveModeObjs.remove(0);
-        // }
-        // TDQ-6407~
-        if (hiveModeObjs != null && hiveModeObjs.size() > 0) {
-            List<String> hiveModeNameList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : hiveModeObjs) {
-                hiveModeNameList.add(bean.getDisplayName());
-            }
-            return hiveModeNameList;
-        }
-        return null;
-    }
-
-    public static String[] getHiveModeNames(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<String> hiveModeNameList = getHiveModeNameList(distributionIndex, versionIndex, hiveServerIndex);
-        if (hiveModeNameList != null && hiveModeNameList.size() > 0) {
-            return hiveModeNameList.toArray(new String[hiveModeNameList.size()]);
-        } else {
-            return new String[0];
-        }
-    }
-
-    protected static List<String> getHiveModeKeyList(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> hiveModeObjs = getHiveModes(distributionIndex, versionIndex, hiveServerIndex);
-        // ADD msjian TDQ-6407 2012-11-26: for top, not support hive embeded mode,hide this menu
-        // if (PluginChecker.isOnlyTopLoaded() && hiveModeObjs.size() > 1) {
-        // hiveModeObjs.remove(0);
-        // }
-        // TDQ-6407~
-        if (hiveModeObjs != null && hiveModeObjs.size() > 0) {
-            List<String> hiveModeKeyList = new ArrayList<String>();
-            for (HiveConnVersionInfo bean : hiveModeObjs) {
-                hiveModeKeyList.add(bean.getKey());
-            }
-            return hiveModeKeyList;
-        }
-        return null;
-    }
-
-    public static String[] getHiveModeKeys(int distributionIndex, int versionIndex, int hiveServerIndex) {
-        List<String> hiveModeKeyList = getHiveModeKeyList(distributionIndex, versionIndex, hiveServerIndex);
-        if (hiveModeKeyList != null && hiveModeKeyList.size() > 0) {
-            return hiveModeKeyList.toArray(new String[hiveModeKeyList.size()]);
-        } else {
-            return new String[0];
-        }
-    }
-
-    public static int getIndexOfDistribution(String distributionKey) {
-        int level = 0;
-        List<HiveConnVersionInfo> distributions = getObjectsByLevel(level);
-        if (distributions != null && distributions.size() > 0) {
-            for (int i = 0; i < distributions.size(); i++) {
-                if (distributions.get(i).getKey().equals(distributionKey)) {
-                    return i;
-                }
-            }
-        }
-        return -1;
-    }
-
-    protected static int getIndexOfDistribution(HiveConnVersionInfo distribution) {
-        String key = distribution.getKey();
-        return getIndexOfDistribution(key);
-    }
-
-    public static int getIndexOfHiveVersion(String distributionKey, String hiveVersionKey) {
-        int distributionIndex = getIndexOfDistribution(distributionKey);
-        List<HiveConnVersionInfo> hiveVersions = getHiveVersions(distributionIndex);
-        if (hiveVersions != null && hiveVersions.size() > 0) {
-            for (int i = 0; i < hiveVersions.size(); i++) {
-                if (hiveVersions.get(i).getKey().equals(hiveVersionKey)) {
-                    return i;
-                }
-            }
-        }
-        return 0;
-    }
-
-    protected static int getIndexOfHiveVersion(HiveConnVersionInfo distribution, HiveConnVersionInfo hiveVersion) {
-        String distributionKey = distribution.getKey();
-        String hiveVersionKey = hiveVersion.getKey();
-        return getIndexOfHiveVersion(distributionKey, hiveVersionKey);
-    }
-
-    public static int getIndexOfHiveMode(String distributionKey, String hiveVersionKey, String hiveModeKey, String hiveServerKey) {
-        int distributionIndex = getIndexOfDistribution(distributionKey);
-        int hiveVersionIndex = getIndexOfHiveVersion(distributionKey, hiveVersionKey);
-        int hiveServerIndex = getIndexOfHiveServer(hiveServerKey);
-        List<HiveConnVersionInfo> hiveModes = getHiveModes(distributionIndex, hiveVersionIndex, hiveServerIndex);
-        if (hiveModes != null && hiveModes.size() > 0) {
-            for (int i = 0; i < hiveModes.size(); i++) {
-                if (hiveModes.get(i).getKey().equals(hiveModeKey)) {
-                    return i;
-                }
-            }
-        }
-        return 0;
-    }
-
-    protected static int getIndexOfHiveMode(HiveConnVersionInfo distribution, HiveConnVersionInfo hiveVersion,
-            HiveConnVersionInfo hiveMode, String hiveServerKey) {
-        String distributionKey = distribution.getKey();
-        String hiveModeKey = hiveMode.getKey();
-        String hiveVersionkey = hiveVersion.getKey();
-        return getIndexOfHiveMode(distributionKey, hiveVersionkey, hiveModeKey, hiveServerKey);
-    }
-
-    public static int getIndexOfHiveServer(String hiveServerKey) {
-        List<String> hiveServers = HiveServerVersionUtils.extractListKeys();
-        if (hiveServers != null && hiveServers.size() > 0) {
-            for (int i = 0; i < hiveServers.size(); i++) {
-                if (hiveServers.get(i).equals(hiveServerKey)) {
-                    return i;
-                }
-            }
-        }
-
-        return 0;
-    }
-
-    public static int getIndexOfHiveServerByDisplayName(String hiveServerDisplayName) {
-        List<String> hiveServers = HiveServerVersionUtils.extractListDisplayNames();
-        if (hiveServers != null && hiveServers.size() > 0) {
-            for (int i = 0; i < hiveServers.size(); i++) {
-                if (hiveServers.get(i).equals(hiveServerDisplayName)) {
-                    return i;
-                }
-            }
-        }
-
-        return 0;
-    }
-
-    public static boolean isEmbeddedMode(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, int hiveServerIndex) {
-        List<HiveConnVersionInfo> hiveModes = getHiveModes(distributionIndex, hiveVersionIndex, hiveServerIndex);
-        if (hiveModes != null && hiveModes.size() > 0) {
-            HiveConnVersionInfo hiveMode = hiveModes.get(hiveModeIndex);
-            if (HiveConnVersionInfo.MODE_EMBEDDED.getKey().equals(hiveMode.getKey())) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Returns the distribution <code>HiveConnVersionInfo</code> object by the given <code>index</code> from level 0.
-     * Added by Marvin Wang on Aug 13, 2012.
-     * 
-     * @param index
-     * @return
-     */
-    public static HiveConnVersionInfo getDistributionObj(int index) {
-        List<HiveConnVersionInfo> distributions = getObjectsByLevel(0);
-        if (distributions != null && distributions.size() > 0 && index != -1) {
-            return distributions.get(index);
-        }
-        return null;
-    }
-
-    /**
-     * Returns the hive version object by the given index of distribution and hive version. Added by Marvin Wang on Aug
-     * 13, 2012.
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @return
-     */
-    public static HiveConnVersionInfo getHiveVersionObj(int distributionIndex, int hiveVersionIndex) {
-        List<HiveConnVersionInfo> objs = getHiveVersions(distributionIndex);
-        if (objs != null && objs.size() > 0) {
-            return objs.get(hiveVersionIndex);
-        }
-        return null;
-    }
-
-    /**
-     * Returns the hive mode object by the given index of distribution, version and mode. Added by Marvin Wang on Aug
-     * 13, 2012.
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @param hiveModeIndex
-     * @return
-     */
-    public static HiveConnVersionInfo getHiveModeObj(int distributionIndex, int hiveVersionIndex, int hiveModeIndex,
-            int hiveServerIndex) {
-        List<HiveConnVersionInfo> objs = getHiveModes(distributionIndex, hiveVersionIndex, hiveServerIndex);
-        if (objs != null && objs.size() > 0) {
-            return objs.get(hiveModeIndex);
-        }
-        return null;
-    }
-
-    public static String getHiveModeObjKey(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, int hiveServerIndex) {
-        HiveConnVersionInfo hiveModeObj = getHiveModeObj(distributionIndex, hiveVersionIndex, hiveModeIndex, hiveServerIndex);
-        if (hiveModeObj != null) {
-            return hiveModeObj.getKey();
-        }
-        return null;
-    }
-
-    /**
-     * Checkes if the selected distro version supports hive server2. Added by Marvin Wang on Mar 25, 2013.
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @return
-     */
-    public static boolean isSupportHiveServer2(int distributionIndex, int hiveVersionIndex) {
-        HiveConnVersionInfo distroVersion = getHiveVersionObj(distributionIndex, hiveVersionIndex);
-        return distroVersion.isSupportHive2();
-    }
-
-    /**
-     * Checks if the current distro is {@link HiveConnVersionInfo#CUSTOM}. If yes, then return <code>true</code> Added
-     * by Marvin Wang on Mar 26, 2013.
-     * 
-     * @param distributionIndex
-     * @return
-     */
-    public static boolean isCustomDistro(int distributionIndex) {
-        HiveConnVersionInfo obj = getDistributionObj(distributionIndex);
-        if (HiveConnVersionInfo.DISTRO_CUSTOM == obj) {
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    public static boolean isCustomDistro(String distroKey) {
-        int distroIndex = getIndexOfDistribution(distroKey);
-        return isCustomDistro(distroIndex);
-    }
-
-    /**
-     * This is a temporary method to estimate whether or not one hive version support security. We need to refactor
-     * {@link HiveConnVersionInfo}(A really bad class) to support this function later...
-     * <P>
-     * DOC ycbai Comment method "isSupportSecurity".
-     * 
-     * @param distributionIndex
-     * @param hiveVersionIndex
-     * @param hiveModeIndex
-     * @param isHive2
-     * @return
-     */
-    public static boolean isSupportSecurity(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, boolean isHive2,
-            int hiveServerIndex) {
-        HiveConnVersionInfo hiveVersionObj = getHiveVersionObj(distributionIndex, hiveVersionIndex);
-        if (hiveVersionObj.isSupportSecurity()
-                && (isHive2 || isEmbeddedMode(distributionIndex, hiveVersionIndex, hiveModeIndex, hiveServerIndex))) {
-            return true;
-        }
-
-        return false;
-    }
-
-    public static boolean isSupportTez(int distributionIndex, int hiveVersionIndex, int hiveModeIndex, int hiveServerIndex) {
-        HiveConnVersionInfo hiveVersionObj = getHiveVersionObj(distributionIndex, hiveVersionIndex);
-        boolean versionSupportTez = ArrayUtils.contains(HiveConnVersionInfo.getHiveVersionsSupportingTez(), hiveVersionObj);
-        boolean isEmbeddedMode = isEmbeddedMode(distributionIndex, hiveVersionIndex, hiveModeIndex, hiveServerIndex);
-        return versionSupportTez && isEmbeddedMode;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveConnVersionInfo.java

@@ -1,423 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.model.metadata.connection.hive;
-
-import org.talend.core.hadoop.version.EHadoopDistributions;
-import org.talend.core.hadoop.version.EHadoopVersion4Drivers;
-
-/**
- * This class provides the info for Hive. There are 3 levels as follows:
- *
- * <pre>
- * <li> 0 -- stands for "Distribution"
- * <li> 1 -- stands for "Distro version"
- * <li> 2 -- stands for "Hive mode"
- * </pre>
- *
- * The <code>key</code> of this enum is matched to the config file of component.
- *
- * @author Marvin Wang
- * @version 1.0 jdk1.6
- * @date Aug 9, 2012
- */
-public enum HiveConnVersionInfo {
-
-    HORTONWORKS(0, EHadoopDistributions.HORTONWORKS.getName(), EHadoopDistributions.HORTONWORKS.getDisplayName(), true),
-
-    APACHE(0, EHadoopDistributions.APACHE.getName(), EHadoopDistributions.APACHE.getDisplayName(), false),
-
-    CLOUDERA(0, EHadoopDistributions.CLOUDERA.getName(), EHadoopDistributions.CLOUDERA.getDisplayName(), false),
-
-    MAPR(0, EHadoopDistributions.MAPR.getName(), EHadoopDistributions.MAPR.getDisplayName(), false),
-
-    AMAZON_EMR(0, EHadoopDistributions.AMAZON_EMR.getName(), EHadoopDistributions.AMAZON_EMR.getDisplayName(), false),
-
-    PIVOTAL_HD(0, EHadoopDistributions.PIVOTAL_HD.getName(), EHadoopDistributions.PIVOTAL_HD.getDisplayName(), false),
-
-    DISTRO_CUSTOM(0, EHadoopDistributions.CUSTOM.getName(), EHadoopDistributions.CUSTOM.getDisplayName(), false),
-
-    HDP_2_3(
-            1,
-            EHadoopVersion4Drivers.HDP_2_3.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_3.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_2_2(
-            1,
-            EHadoopVersion4Drivers.HDP_2_2.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_2.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_2_1(
-            1,
-            EHadoopVersion4Drivers.HDP_2_1.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_1.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_2_0(
-            1,
-            EHadoopVersion4Drivers.HDP_2_0.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_2_0.getVersionDisplay(),
-            true,
-            false,
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_1_3(
-            1,
-            EHadoopVersion4Drivers.HDP_1_3.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_1_3.getVersionDisplay(),
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    HDP_1_2(
-            1,
-            EHadoopVersion4Drivers.HDP_1_2.getVersionValue(),
-            EHadoopVersion4Drivers.HDP_1_2.getVersionDisplay(),
-            true,
-            true,
-            HiveConnVersionInfo.HORTONWORKS),
-
-    APACHE_1_0_0(1, EHadoopVersion4Drivers.APACHE_1_0_0.getVersionValue(), EHadoopVersion4Drivers.APACHE_1_0_0
-            .getVersionDisplay(), false, HiveConnVersionInfo.APACHE),
-
-    Cloudera_CDH5_5(1, EHadoopVersion4Drivers.CLOUDERA_CDH5_5.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5_5
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5_4(1, EHadoopVersion4Drivers.CLOUDERA_CDH5_4.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5_4
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5_1(1, EHadoopVersion4Drivers.CLOUDERA_CDH5_1.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5_1
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5_1_MR1(
-                        1,
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionValue(),
-                        EHadoopVersion4Drivers.CLOUDERA_CDH5_1_MR1.getVersionDisplay(),
-                        true,
-                        false,
-                        true,
-                        true,
-                        HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH5(1, EHadoopVersion4Drivers.CLOUDERA_CDH5.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH5
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH4_YARN(1, EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH4_YARN
-            .getVersionDisplay(), true, false, true, false, HiveConnVersionInfo.CLOUDERA),
-
-    Cloudera_CDH4(1, EHadoopVersion4Drivers.CLOUDERA_CDH4.getVersionValue(), EHadoopVersion4Drivers.CLOUDERA_CDH4
-            .getVersionDisplay(), true, true, HiveConnVersionInfo.CLOUDERA),
-
-    MAPR5_0_0(
-              1,
-              EHadoopVersion4Drivers.MAPR500.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR500.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR4_1_0(
-              1,
-              EHadoopVersion4Drivers.MAPR410.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR410.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR4_0_1(
-              1,
-              EHadoopVersion4Drivers.MAPR401.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR401.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR3_1_0(
-              1,
-              EHadoopVersion4Drivers.MAPR310.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR310.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR3_0_1(
-              1,
-              EHadoopVersion4Drivers.MAPR301.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR301.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR2_1_3(
-              1,
-              EHadoopVersion4Drivers.MAPR213.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR213.getVersionDisplay(),
-              true,
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR2_1_2(
-              1,
-              EHadoopVersion4Drivers.MAPR212.getVersionValue(),
-              EHadoopVersion4Drivers.MAPR212.getVersionDisplay(),
-              false,
-              HiveConnVersionInfo.MAPR),
-
-    MAPR2(
-          1,
-          EHadoopVersion4Drivers.MAPR2.getVersionValue(),
-          EHadoopVersion4Drivers.MAPR2.getVersionDisplay(),
-          false,
-          HiveConnVersionInfo.MAPR),
-
-    EMR_4_0_0(1, EHadoopVersion4Drivers.EMR_4_0_0.getVersionValue(), "EMR 4.0.0 (Hive 1.0.0)", //$NON-NLS-1$
-              true,
-              false,
-              HiveConnVersionInfo.AMAZON_EMR),
-
-    APACHE_2_4_0_EMR_0_13_1(1, "APACHE_2_4_0_EMR_0_13_1", "Apache 2.4.0 (Hive 0.13.1)", false, HiveConnVersionInfo.AMAZON_EMR), //$NON-NLS-1$ //$NON-NLS-2$
-
-    APACHE_2_4_0_EMR(
-                     1,
-                     EHadoopVersion4Drivers.APACHE_2_4_0_EMR.getVersionValue(),
-                     "Apache 2.4.0 (Hive 0.11.0)", false, HiveConnVersionInfo.AMAZON_EMR), //$NON-NLS-1$
-
-    APACHE_1_0_3_EMR(
-                     1,
-                     EHadoopVersion4Drivers.APACHE_1_0_3_EMR.getVersionValue(),
-                     "Apache 1.0.3 (Hive 0.8.1)", false, HiveConnVersionInfo.AMAZON_EMR), //$NON-NLS-1$
-
-    PIVOTAL_HD_2_0(1, EHadoopVersion4Drivers.PIVOTAL_HD_2_0.getVersionValue(), EHadoopVersion4Drivers.PIVOTAL_HD_2_0
-            .getVersionDisplay(), true, false, true, true, HiveConnVersionInfo.PIVOTAL_HD),
-
-    PIVOTAL_HD_1_0_1(1, EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1.getVersionValue(), EHadoopVersion4Drivers.PIVOTAL_HD_1_0_1
-            .getVersionDisplay(), false, false, true, false, HiveConnVersionInfo.PIVOTAL_HD),
-
-    DISTRO_VERSION_CUSTOM(1, "DISTRO_VERSION_CUSTOM", "Customized Version", true, true, HiveConnVersionInfo.DISTRO_CUSTOM), //$NON-NLS-1$//$NON-NLS-2$
-
-    MODE_EMBEDDED(2, "EMBEDDED",//$NON-NLS-1$
-                  "Embedded",//$NON-NLS-1$
-                  false,
-                  HiveConnVersionInfo.HDP_1_2,
-                  HiveConnVersionInfo.HDP_1_3,
-                  HiveConnVersionInfo.HDP_2_0,
-                  HiveConnVersionInfo.HDP_2_1,
-                  HiveConnVersionInfo.HDP_2_2,
-                  // HiveConnVersionInfo.HDP_2_3,
-                  HiveConnVersionInfo.APACHE_1_0_0,
-                  HiveConnVersionInfo.Cloudera_CDH4,
-                  HiveConnVersionInfo.Cloudera_CDH4_YARN,
-                  HiveConnVersionInfo.Cloudera_CDH5,
-                  HiveConnVersionInfo.Cloudera_CDH5_1,
-                  HiveConnVersionInfo.Cloudera_CDH5_1_MR1,
-                  HiveConnVersionInfo.Cloudera_CDH5_4,
-                  HiveConnVersionInfo.MAPR2,
-                  HiveConnVersionInfo.MAPR2_1_2,
-                  HiveConnVersionInfo.MAPR2_1_3,
-                  HiveConnVersionInfo.MAPR3_0_1,
-                  HiveConnVersionInfo.MAPR3_1_0,
-                  HiveConnVersionInfo.MAPR4_0_1,
-                  HiveConnVersionInfo.MAPR4_1_0,
-                  // HiveConnVersionInfo.MAPR5_0_0,
-                  HiveConnVersionInfo.APACHE_1_0_3_EMR,
-                  HiveConnVersionInfo.APACHE_2_4_0_EMR,
-                  HiveConnVersionInfo.PIVOTAL_HD_1_0_1,
-                  HiveConnVersionInfo.PIVOTAL_HD_2_0,
-                  HiveConnVersionInfo.DISTRO_VERSION_CUSTOM),
-
-    MODE_STANDALONE(2, "STANDALONE",//$NON-NLS-1$
-                    "Standalone",//$NON-NLS-1$
-                    false,
-                    HiveConnVersionInfo.HDP_1_2,
-                    HiveConnVersionInfo.HDP_1_3,
-                    HiveConnVersionInfo.HDP_2_0,
-                    HiveConnVersionInfo.HDP_2_1,
-                    HiveConnVersionInfo.HDP_2_2,
-                    HiveConnVersionInfo.HDP_2_3,
-                    HiveConnVersionInfo.APACHE_1_0_0,
-                    HiveConnVersionInfo.Cloudera_CDH4,
-                    HiveConnVersionInfo.Cloudera_CDH4_YARN,
-                    HiveConnVersionInfo.Cloudera_CDH5,
-                    HiveConnVersionInfo.Cloudera_CDH5_1,
-                    HiveConnVersionInfo.Cloudera_CDH5_1_MR1,
-                    HiveConnVersionInfo.Cloudera_CDH5_4,
-                    HiveConnVersionInfo.Cloudera_CDH5_5,
-                    HiveConnVersionInfo.MAPR2,
-                    HiveConnVersionInfo.MAPR2_1_2,
-                    HiveConnVersionInfo.MAPR2_1_3,
-                    HiveConnVersionInfo.MAPR3_0_1,
-                    HiveConnVersionInfo.MAPR3_1_0,
-                    HiveConnVersionInfo.MAPR4_0_1,
-                    HiveConnVersionInfo.MAPR4_1_0,
-                    HiveConnVersionInfo.MAPR5_0_0,
-                    HiveConnVersionInfo.APACHE_1_0_3_EMR,
-                    HiveConnVersionInfo.APACHE_2_4_0_EMR,
-                    HiveConnVersionInfo.APACHE_2_4_0_EMR_0_13_1,
-                    HiveConnVersionInfo.EMR_4_0_0,
-                    HiveConnVersionInfo.PIVOTAL_HD_1_0_1,
-                    HiveConnVersionInfo.PIVOTAL_HD_2_0,
-                    HiveConnVersionInfo.DISTRO_VERSION_CUSTOM);
-
-    private int level;
-
-    private String key;// The key is mapped to item name of component in t*_java.xml file.
-
-    private String displayName;// That is mapped to the property of component in t*_messages.properties file.
-
-    private HiveConnVersionInfo[] follows;// That stores all are followed by the current object.
-
-    private boolean isSupportHive2; // Till now only MapR2, CDH4, and HDP1.2 support hive server2.
-
-    private boolean isSupportMR1;
-
-    private boolean isSupportYARN;
-
-    private boolean supportSecurity;
-
-    private static HiveConnVersionInfo[] hiveVersions = new HiveConnVersionInfo[] { HiveConnVersionInfo.Cloudera_CDH5,
-            HiveConnVersionInfo.Cloudera_CDH5_1, HiveConnVersionInfo.Cloudera_CDH5_1_MR1, HiveConnVersionInfo.Cloudera_CDH5_4,
-            HiveConnVersionInfo.Cloudera_CDH5_5, HiveConnVersionInfo.HDP_2_3, HiveConnVersionInfo.HDP_2_2,
-            HiveConnVersionInfo.HDP_2_1, HiveConnVersionInfo.HDP_2_0, HiveConnVersionInfo.PIVOTAL_HD_2_0,
-            HiveConnVersionInfo.APACHE_2_4_0_EMR, HiveConnVersionInfo.APACHE_2_4_0_EMR_0_13_1, HiveConnVersionInfo.EMR_4_0_0 };
-
-    private static HiveConnVersionInfo[] hiveVersionsSupportingTez = new HiveConnVersionInfo[] { HiveConnVersionInfo.HDP_2_2,
-            HiveConnVersionInfo.HDP_2_1, HiveConnVersionInfo.MAPR4_0_1, HiveConnVersionInfo.MAPR4_1_0,
-            HiveConnVersionInfo.MAPR5_0_0, HiveConnVersionInfo.DISTRO_VERSION_CUSTOM };
-
-    private HiveConnVersionInfo(int level, String key, String displayName, boolean supportSecurity,
-            HiveConnVersionInfo... follows) {
-        this(level, key, displayName, false, supportSecurity, follows);
-    }
-
-    private HiveConnVersionInfo(int level, String key, String displayName, boolean isSupportHive2, boolean supportSecurity,
-            HiveConnVersionInfo... follows) {
-        this(level, key, displayName, isSupportHive2, true, false, supportSecurity, follows);
-    }
-
-    private HiveConnVersionInfo(int level, String key, String displayName, boolean isSupportHive2, boolean isSupportMR1,
-            boolean isSupportYARN, boolean supportSecurity, HiveConnVersionInfo... follows) {
-        this.level = level;
-        this.key = key;
-        this.displayName = displayName;
-        this.isSupportHive2 = isSupportHive2;
-        this.isSupportMR1 = isSupportMR1;
-        this.isSupportYARN = isSupportYARN;
-        this.follows = follows;
-        this.supportSecurity = supportSecurity;
-    }
-
-    public int getLevel() {
-        return this.level;
-    }
-
-    public String getKey() {
-        return this.key;
-    }
-
-    public String getDisplayName() {
-        return this.displayName;
-    }
-
-    public HiveConnVersionInfo[] getFollows() {
-        return this.follows;
-    }
-
-    /**
-     * Getter for isSupportHive2.
-     *
-     * @return the isSupportHive2
-     */
-    public boolean isSupportHive2() {
-        return this.isSupportHive2;
-    }
-
-    public boolean isSupportHiveServerVersion(HiveServerVersionInfo hiveServerVersion) {
-        boolean isSupport = false;
-
-        if (hiveServerVersion == null) {
-            return isSupport;
-        }
-
-        switch (hiveServerVersion) {
-        case HIVE_SERVER_1:
-            switch (this) {
-            // make them same with components
-            case HDP_2_3:
-            case EMR_4_0_0:
-            case Cloudera_CDH5_4:
-            case Cloudera_CDH5_5:
-            case MAPR4_1_0:
-            case MAPR5_0_0:
-                isSupport = false;
-                break;
-            default:
-                isSupport = true;
-            }
-            break;
-        case HIVE_SERVER_2:
-            isSupport = isSupportHive2();
-            break;
-        default:
-            isSupport = false;
-        }
-
-        return isSupport;
-    }
-
-    public boolean isSupportMR1() {
-        return this.isSupportMR1;
-    }
-
-    public boolean isSupportYARN() {
-        return this.isSupportYARN;
-    }
-
-    public boolean isSupportSecurity() {
-        return this.supportSecurity;
-    }
-
-    public static HiveConnVersionInfo[] getHiveVersionsNotSupportOnWindows() {
-        return hiveVersions;
-    }
-
-    public static HiveConnVersionInfo[] getHiveVersionsSupportingTez() {
-        return hiveVersionsSupportingTez;
-    }
-
-    public static HiveConnVersionInfo getVersionByKey(String key) {
-        HiveConnVersionInfo[] values = values();
-        for (HiveConnVersionInfo version : values) {
-            if (version.getLevel() == 1 && version.getKey().equals(key)) {
-                return version;
-            }
-        }
-
-        return null;
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveModeInfo.java

@@ -0,0 +1,54 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.metadata.connection.hive;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public enum HiveModeInfo {
+    STANDALONE("Standalone"), //$NON-NLS-1$
+    EMBEDDED("Embedded"), //$NON-NLS-1$
+    ;
+
+    private String display;
+
+    private HiveModeInfo(String diaplyName) {
+        this.display = diaplyName;
+    }
+
+    public String getName() {
+        return name();
+    }
+
+    public String getDisplayName() {
+        return display;
+    }
+
+    public static HiveModeInfo getByDisplay(String display) {
+        for (HiveModeInfo m : HiveModeInfo.values()) {
+            if (m.getDisplayName().equals(display)) {
+                return m;
+            }
+        }
+        return null;
+    }
+
+    public static HiveModeInfo get(String name) {
+        for (HiveModeInfo m : HiveModeInfo.values()) {
+            if (m.getName().equals(name)) {
+                return m;
+            }
+        }
+        return null;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveServerVersionInfo.java

@@ -42,15 +42,6 @@ public enum HiveServerVersionInfo {
         return this.key;
     }
 
-    /**
-     * Sets the key.
-     * 
-     * @param key the key to set
-     */
-    public void setKey(String key) {
-        this.key = key;
-    }
-
     /**
      * Getter for name.
      * 
@@ -60,15 +51,6 @@ public enum HiveServerVersionInfo {
         return this.name;
     }
 
-    /**
-     * Sets the name.
-     * 
-     * @param name the name to set
-     */
-    public void setName(String name) {
-        this.name = name;
-    }
-
     /**
      * Getter for displayName.
      * 
@@ -78,13 +60,21 @@ public enum HiveServerVersionInfo {
         return this.displayName;
     }
 
-    /**
-     * Sets the displayName.
-     * 
-     * @param displayName the displayName to set
-     */
-    public void setDisplayName(String displayName) {
-        this.displayName = displayName;
+    public static HiveServerVersionInfo getByDisplay(String display) {
+        for (HiveServerVersionInfo s : HiveServerVersionInfo.values()) {
+            if (s.getDisplayName().equals(display)) {
+                return s;
+            }
+        }
+        return null;
     }
 
+    public static HiveServerVersionInfo getByKey(String key) {
+        for (HiveServerVersionInfo s : HiveServerVersionInfo.values()) {
+            if (s.getKey().equals(key)) {
+                return s;
+            }
+        }
+        return null;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/connection/hive/HiveServerVersionUtils.java

@@ -1,132 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core.model.metadata.connection.hive;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Created by Marvin Wang on Mar 25, 2013.
- */
-public class HiveServerVersionUtils {
-
-    /**
-     * Extracts all display names for hive server version, all versions are from {@link HiveServerVersionInfo}. Added by
-     * Marvin Wang on Mar 25, 2013.
-     * 
-     * @return
-     */
-    public static List<String> extractListDisplayNames() {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        List<String> displayNames = new ArrayList<String>();
-        for (HiveServerVersionInfo info : infos) {
-            displayNames.add(info.getDisplayName());
-        }
-
-        return displayNames;
-    }
-
-    public static List<String> extractAvailableListDisplayNames(HiveConnVersionInfo hadoopVersion) {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        List<String> keys = new ArrayList<String>();
-        for (HiveServerVersionInfo info : infos) {
-            if (hadoopVersion.isSupportHiveServerVersion(info)) {
-                keys.add(info.getDisplayName());
-            }
-        }
-
-        return keys;
-    }
-
-    /**
-     * Extracts the array of display names, refer to this method {@link #extractListDisplayNames()}. Added by Marvin
-     * Wang on Mar 25, 2013.
-     * 
-     * @return
-     */
-    public static String[] extractArrayDisplayNames() {
-        List<String> list = extractListDisplayNames();
-        return list.toArray(new String[list.size()]);
-    }
-
-    public static String[] extractAvailableArrayDisplayNames(HiveConnVersionInfo hadoopVersion) {
-        List<String> list = extractAvailableListDisplayNames(hadoopVersion);
-        return list.toArray(new String[list.size()]);
-    }
-
-    public static List<String> extractListKeys() {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        List<String> keys = new ArrayList<String>();
-        for (HiveServerVersionInfo info : infos) {
-            keys.add(info.getKey());
-        }
-
-        return keys;
-    }
-
-    /**
-     * Gets the index of hive server by display name, if no index found, then return 0. Added by Marvin Wang on Mar 25,
-     * 2013.
-     * 
-     * @param hiveServerDisplayName
-     * @return 0 if there is no index found.
-     */
-    public static int getIndexofHiveServer(String hiveServerDisplayName) {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        for (int i = 0; i < infos.length; i++) {
-            if (infos[i].getDisplayName().equalsIgnoreCase(hiveServerDisplayName)) {
-                return i;
-            }
-        }
-        return 0;
-    }
-
-    public static int getIndexofHiveServerByKey(String hiveKey) {
-        HiveServerVersionInfo[] infos = HiveServerVersionInfo.values();
-        for (int i = 0; i < infos.length; i++) {
-            if (infos[i].getKey().equalsIgnoreCase(hiveKey)) {
-                return i;
-            }
-        }
-        return 0;
-    }
-
-    /**
-     * Extracts the display name by the index. Added by Marvin Wang on Mar 25, 2013.
-     * 
-     * @param index
-     * @return
-     */
-    public static String extractDisplayName(int index) {
-        List<String> displayNames = extractListDisplayNames();
-        if (index < 0) {
-            return null;
-        }
-        return displayNames.get(index);
-    }
-
-    /**
-     * Extracts the key which is mapped to t*component.xml. Added by Marvin Wang on Mar 26, 2013.
-     * 
-     * @param index
-     * @return
-     */
-    public static String extractKey(int index) {
-        List<String> keys = extractListKeys();
-        if (index < 0) {
-            return null;
-        }
-        return keys.get(index);
-    }
-
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -21,11 +21,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.emf.common.util.EList;
 import org.talend.commons.ui.runtime.exception.ExceptionHandler;
 import org.talend.commons.ui.utils.PathUtils;
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.database.EDatabaseTypeName;
 import org.talend.core.database.conn.ConnParameterKeys;
@@ -910,18 +910,10 @@ public class RepositoryToComponentProperty {
             }
         }
         if (value.equals("SERVER_NAME")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
         if (value.equals("PORT")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getPort())) {
-                return connection.getPort();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getPort());
-            }
+            return getAppropriateValue(connection, connection.getPort());
         }
         if (value.equals("SID") || value.equals("DATABASE_ALIAS")) { //$NON-NLS-1$ //$NON-NLS-2$
             if (("").equals(connection.getSID()) || connection.getSID() == null) { //$NON-NLS-1$
@@ -939,54 +931,25 @@ public class RepositoryToComponentProperty {
             }
         }
         if (value.equals("DATASOURCE")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getDatasourceName())) {
-                return connection.getDatasourceName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDatasourceName());
-            }
+            return getAppropriateValue(connection, connection.getDatasourceName());
         }
         if (value.equals("USERNAME")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getUsername())) {
-                return connection.getUsername();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getUsername());
-            }
+            return getAppropriateValue(connection, connection.getUsername());
         }
         if (value.equals("PASSWORD")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getPassword())) {
-                return connection.getPassword();
-            } else {
-                String pwd = TalendQuoteUtils.checkAndAddBackslashes(connection.getRawPassword());
-                return TalendQuoteUtils.addQuotes(pwd);
-            }
+            return getAppropriateValue(connection, connection.getValue(connection.getPassword(), false));
         }
         if (value.equals("NULL_CHAR")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getNullChar())) {
-                return connection.getNullChar();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getNullChar());
-            }
+            return getAppropriateValue(connection, connection.getNullChar());
         }
         if (value.equals("SCHEMA")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getUiSchema())) {
-                return connection.getUiSchema();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getUiSchema());
-            }
+            return getAppropriateValue(connection, connection.getUiSchema());
         }
         if (value.equals("FILE")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getFileFieldName())) {
-                return connection.getFileFieldName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getFileFieldName());
-            }
+            return getAppropriateValue(connection, connection.getFileFieldName());
         }
         if (value.equals("PROPERTIES_STRING")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getAdditionalParams())) {
-                return connection.getAdditionalParams();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getAdditionalParams());
-            }
+            return getAppropriateValue(connection, connection.getAdditionalParams());
         }
 
         if (value.equals("CDC_MODE")) { //$NON-NLS-1$
@@ -1163,12 +1126,7 @@ public class RepositoryToComponentProperty {
         }
         // add this for tJavaDB embeded "DB Root Path"
         if (value.equals("DIRECTORY")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDBRootPath())) {
-                return connection.getDBRootPath();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDBRootPath());
-            }
-
+            return getAppropriateValue(connection, connection.getDBRootPath());
         }
         // add for feature 11674
         if (value.equals("RUNNING_MODE")) {//$NON-NLS-1$       
@@ -1183,25 +1141,13 @@ public class RepositoryToComponentProperty {
             return runningMode;
         }
         if (value.equals("DBPATH")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDBRootPath())) {
-                return connection.getDBRootPath();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDBRootPath());
-            }
+            return getAppropriateValue(connection, connection.getDBRootPath());
         }
         if (value.equals("DBNAME")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDatasourceName())) {
-                return connection.getDatasourceName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDatasourceName());
-            }
+            return getAppropriateValue(connection, connection.getDatasourceName());
         }
         if (value.equals("RAC_URL")) {
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
 
         if (value.equals("DISTRIBUTION")) {
@@ -1229,13 +1175,15 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("HBASE_MASTER_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_MASTERPRINCIPAL));
+            String hbaseMasterPrinc = connection.getParameters().get(
+                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_MASTERPRINCIPAL);
+            return getAppropriateValue(connection, hbaseMasterPrinc);
         }
 
         if (value.equals("HBASE_REGIONSERVER_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_REGIONSERVERPRINCIPAL));
+            String hbaseRegPrinc = connection.getParameters().get(
+                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_REGIONSERVERPRINCIPAL);
+            return getAppropriateValue(connection, hbaseRegPrinc);
         }
 
         if (value.equals("HIVE_SERVER")) {
@@ -1300,11 +1248,7 @@ public class RepositoryToComponentProperty {
         if (value.equals(EParameterNameForComponent.PARA_NAME_MAPRED_JT.getName())
                 || value.equals(EParameterNameForComponent.PARA_NAME_RESOURCE_MANAGER.getName())) {
             String jobTrackerURL = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_URL);
-            if (isContextMode(connection, jobTrackerURL)) {
-                return jobTrackerURL;
-            } else {
-                return TalendQuoteUtils.addQuotes(jobTrackerURL);
-            }
+            return getAppropriateValue(connection, jobTrackerURL);
         }
 
         if (value.equals(EParameterNameForComponent.PARA_NAME_USE_YARN.getName())) {
@@ -1343,20 +1287,12 @@ public class RepositoryToComponentProperty {
 
         if (value.equals("MAPRED_JOB_TRACKER") || value.equals("MAPRED_RESOURCE_MANAGER")) {
             String mapredJobTracker = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_URL);
-            if (mapredJobTracker == null) {
-                return mapredJobTracker;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(mapredJobTracker));
-            }
+            return getAppropriateValue(connection, mapredJobTracker);
         }
 
         if (value.equals("NAMENODE_PRINCIPAL")) {
             String nameNodePrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_NAME_NODE_PRINCIPAL);
-            if (nameNodePrincipal == null) {
-                return nameNodePrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(nameNodePrincipal));
-            }
+            return getAppropriateValue(connection, nameNodePrincipal);
         }
 
         /**
@@ -1366,36 +1302,20 @@ public class RepositoryToComponentProperty {
          */
         if (value.equals("JOBTRACKER_PRINCIPAL") || value.equals("RESOURCEMANAGER_PRINCIPAL")) {
             String jtOrRmPrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_PRINCIPAL);
-            if (jtOrRmPrincipal == null) {
-                return jtOrRmPrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(jtOrRmPrincipal));
-            }
+            return getAppropriateValue(connection, jtOrRmPrincipal);
         }
 
         if (value.equals("JOBHISTORY_PRINCIPAL")) {
             String jobHistoryPrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_HISTORY_PRINCIPAL);
-            if (jobHistoryPrincipal == null) {
-                return jobHistoryPrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(jobHistoryPrincipal));
-            }
+            return getAppropriateValue(connection, jobHistoryPrincipal);
         }
 
         if (value.equals("ZOOKEEPER_QUORUM")) {
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
 
         if (value.equals("ZOOKEEPER_CLIENT_PORT")) {
-            if (isContextMode(connection, connection.getPort())) {
-                return connection.getPort();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getPort());
-            }
+            return getAppropriateValue(connection, connection.getPort());
         }
 
         if (value.equals("COLUMN_MAPPING")) { //$NON-NLS-1$
@@ -1403,24 +1323,28 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("HIVE_PRINCIPAL")) {
-            return TalendQuoteUtils
-                    .addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_HIVEPRINCIPLA));
+            String hivePrincipal = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_HIVEPRINCIPLA);
+            return getAppropriateValue(connection, hivePrincipal);
         }
 
         if (value.equals("METASTORE_JDBC_URL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_METASTOREURL));
+            String metadataUrl = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_METASTOREURL);
+            return getAppropriateValue(connection, metadataUrl);
         }
 
         if (value.equals("METASTORE_CLASSNAME")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_DRIVERCLASS));
+            String metastoreDC = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_DRIVERCLASS);
+            return getAppropriateValue(connection, metastoreDC);
         }
 
         if (value.equals("METASTORE_USERNAME")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_USERNAME));
+            String metastoreUser = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_USERNAME);
+            return getAppropriateValue(connection, metastoreUser);
         }
 
         if (value.equals("METASTORE_PASSWORD")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_PASSWORD));
+            String metastorePwd = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_PASSWORD);
+            return getAppropriateValue(connection, metastorePwd);
         }
 
         if (value.equals("USE_KEYTAB")) {
@@ -1433,15 +1357,18 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("PRINCIPAL") || value.equals("KEYTAB_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB_PRINCIPAL));
+            String principal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB_PRINCIPAL);
+            return getAppropriateValue(connection, principal);
         }
 
         if (value.equals("KEYTAB_PATH")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB));
+            String keytab = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB);
+            return getAppropriateValue(connection, keytab);
         }
 
         if (value.equals("IMPALA_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA));
+            String principal = connection.getParameters().get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA);
+            return getAppropriateValue(connection, principal);
         }
 
         if (value.equals("IMPALA_VERSION")) {
@@ -1461,8 +1388,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("RESOURCEMANAGER_SCHEDULER_ADDRESS")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_RESOURCEMANAGER_SCHEDULER_ADDRESS));
+            String rmSchAdr = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_RESOURCEMANAGER_SCHEDULER_ADDRESS);
+            return getAppropriateValue(connection, rmSchAdr);
         }
 
         if (value.equals("SET_JOBHISTORY_ADDRESS")) {
@@ -1470,7 +1397,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("JOBHISTORY_ADDRESS")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOBHISTORY_ADDRESS));
+            String jhAddr = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOBHISTORY_ADDRESS);
+            return getAppropriateValue(connection, jhAddr);
         }
 
         if (value.equals("SET_STAGING_DIRECTORY")) {
@@ -1478,7 +1406,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("STAGING_DIRECTORY")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_STAGING_DIRECTORY));
+            String stageDir = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_STAGING_DIRECTORY);
+            return getAppropriateValue(connection, stageDir);
         }
 
         if (value.equals("USE_DATANODE_HOSTNAME")) {
@@ -1490,6 +1419,14 @@ public class RepositoryToComponentProperty {
 
     }
 
+    private static String getAppropriateValue(Connection connection, String rawValue) {
+        if (isContextMode(connection, rawValue)) {
+            return rawValue;
+        } else {
+            return TalendQuoteUtils.addQuotesIfNotExist(rawValue);
+        }
+    }
+
     /**
      * DOC nrousseau Comment method "getDatabaseValue".
      * 
@@ -2695,10 +2632,11 @@ public class RepositoryToComponentProperty {
         return value;
     }
 
-    public static boolean isGenericRepositoryValue(Connection connection, String paramName) {
+    public static boolean isGenericRepositoryValue(Connection connection, List<ComponentProperties> componentProperties,
+            String paramName) {
         for (IDragAndDropServiceHandler handler : DragAndDropManager.getHandlers()) {
             if (handler.canHandle(connection)) {
-                return handler.isGenericRepositoryValue(connection, paramName);
+                return handler.isGenericRepositoryValue(componentProperties, paramName);
             }
         }
         return false;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/types/Java2STLangTypesHelper.java

@@ -60,8 +60,8 @@ public class Java2STLangTypesHelper {
         return "string"; //$NON-NLS-1$
 
     }
-	
-	/**
+
+    /**
      * DOC return as: int, long, double, float, string.
      * 
      * @author rdubois
@@ -69,32 +69,34 @@ public class Java2STLangTypesHelper {
      * @param columnLabel
      * @return
      */
-	
-	public static String getPigType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig type according to the talendType
-            IMetadataColumn column = metadataTable.getColumn(columnLabel);
-            String talendType = column.getTalendType();
 
-            String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
-            // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
-            if (typeToGenerate.equals("int") || typeToGenerate.equals("short") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                return "int"; //$NON-NLS-1$
-            } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
-                return "long"; //$NON-NLS-1$
-            } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$
-                return "double"; //$NON-NLS-1$
-            }
+    public static String getPigType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig
+                                                                                        // type according to the
+                                                                                        // talendType
+        IMetadataColumn column = metadataTable.getColumn(columnLabel);
+        String talendType = column.getTalendType();
 
-            // Floats: float(Float)
-            if (typeToGenerate.equals("float")) { //$NON-NLS-1$ //$NON-NLS-2$
-                return "float"; //$NON-NLS-1$
-            }
+        String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
+        // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
+        if (typeToGenerate.equals("int") || typeToGenerate.equals("short") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+            return "int"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
+            return "long"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$
+            return "double"; //$NON-NLS-1$
+        }
 
-            // others treat as string
-            return "chararray"; //$NON-NLS-1$
-	
-       }
-	   
-	   	/**
+        // Floats: float(Float)
+        if (typeToGenerate.equals("float")) { //$NON-NLS-1$ 
+            return "float"; //$NON-NLS-1$
+        }
+
+        // others treat as string
+        return "chararray"; //$NON-NLS-1$
+
+    }
+
+    /**
      * DOC return as: int, long, double, float, string.
      * 
      * @author rdubois
@@ -102,34 +104,36 @@ public class Java2STLangTypesHelper {
      * @param columnLabel
      * @return
      */
-	
-	public static String getHCatType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig type according to the talendType
-            IMetadataColumn column = metadataTable.getColumn(columnLabel);
-            String talendType = column.getTalendType();
 
-            String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
-            // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
-            if (typeToGenerate.equals("int") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                return "int"; //$NON-NLS-1$
-            } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
-                return "bigint"; //$NON-NLS-1$
-			} else if (typeToGenerate.equals("short")) { //$NON-NLS-1$
-				return "smallint";
-			} else if (typeToGenerate.equals("boolean")) { //$NON-NLS-1$
-				return "boolean";
-            } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$
-                return "double"; //$NON-NLS-1$
-            }
+    public static String getHCatType(IMetadataTable metadataTable, String columnLabel) { // This method returns the Pig
+                                                                                         // type according to the
+                                                                                         // talendType
+        IMetadataColumn column = metadataTable.getColumn(columnLabel);
+        String talendType = column.getTalendType();
 
-            // Floats: float(Float)
-            if (typeToGenerate.equals("float")) { //$NON-NLS-1$ //$NON-NLS-2$
-                return "float"; //$NON-NLS-1$
-            }
+        String typeToGenerate = JavaTypesManager.getTypeToGenerate(talendType, false);
+        // Integers: int(Integer), long(Long), short(Short), byte(Byte), BigDecimal, double(Double).
+        if (typeToGenerate.equals("int") || typeToGenerate.equals("byte")) { //$NON-NLS-1$ //$NON-NLS-2$ 
+            return "int"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("long")) { //$NON-NLS-1$
+            return "bigint"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("short")) { //$NON-NLS-1$
+            return "smallint"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("boolean")) { //$NON-NLS-1$
+            return "boolean"; //$NON-NLS-1$
+        } else if (typeToGenerate.equals("BigDecimal") || typeToGenerate.equals("double")) { //$NON-NLS-1$ //$NON-NLS-2$
+            return "double"; //$NON-NLS-1$
+        }
 
-            // others treat as string
-            return "string"; //$NON-NLS-1$
-	
-       }
+        // Floats: float(Float)
+        if (typeToGenerate.equals("float")) { //$NON-NLS-1$ 
+            return "float"; //$NON-NLS-1$
+        }
+
+        // others treat as string
+        return "string"; //$NON-NLS-1$
+
+    }
 
     /**
      * DOC return as: %d, %f, %b, %s.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EParameterFieldType.java

@@ -99,6 +99,7 @@ public enum EParameterFieldType {
     NAME_SELECTION_REFERENCE, // ycbai added for generic wizard
     COMPONENT_REFERENCE, // ycbai added for generic wizard
     HIDDEN_TEXT, // hcyi added for generic wizard/component properties
+    SCHEMA_REFERENCE, // hcyi added for component properties
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ElementParameterParser.java

@@ -317,7 +317,7 @@ public final class ElementParameterParser {
         return null;
     }
 
-    private static List<Map<String, String>> createTableValues(final List<Map<String, Object>> paramValues,
+    public static List<Map<String, String>> createTableValues(final List<Map<String, Object>> paramValues,
             final IElementParameter param) {
         List<Map<String, String>> tableValues = new ArrayList<Map<String, String>>();
         if (paramValues != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IGenericProvider.java

@@ -0,0 +1,26 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+import java.util.List;
+
+/**
+ * created by hcyi on Mar 25, 2016 Detailled comment
+ *
+ */
+public interface IGenericProvider {
+
+    public void loadComponentsFromExtensionPoint();
+
+    public List<?> addPaletteEntry();
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/relationship/RelationshipItemBuilder.java

@@ -434,6 +434,11 @@ public class RelationshipItemBuilder {
         }
     }
 
+    public void clearAllItemsRelations() {
+        currentProjectItemsRelations.clear();
+        referencesItemsRelations.clear();
+    }
+
     public void mergeRelationship(Map<Relation, Set<Relation>> itemRelations, Map<Relation, Set<Relation>> newRelations) {
         if (itemRelations != null && newRelations != null) {
             for (Relation relation : newRelations.keySet()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -87,18 +87,16 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static final String PROD_DQ = ProductUtils.PROD_DQ;
 
-    public static final String PROD_CAMEL = ProductUtils.PROD_CAMEL;
-
     public static final String PROD_MDM = ProductUtils.PROD_MDM;
 
     public final static ERepositoryObjectType GENERAL_FILE = new ERepositoryObjectType("repository.generalfile", "",
-            "GENERAL_FILE", true, 1, new String[] { PROD_DI, PROD_CAMEL, PROD_DQ, PROD_MDM }, new String[] {}, false);
+            "GENERAL_FILE", true, 1, new String[] { PROD_DI, PROD_DQ, PROD_MDM }, new String[] {}, false);
 
     public final static ERepositoryObjectType GIT_ROOT = new ERepositoryObjectType("repository.gitroot", "", "GIT_ROOT", true, 1,
-            new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
+            new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SVN_ROOT = new ERepositoryObjectType("repository.svnroot", "", "SVN_ROOT", true, 1,
-            new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
+            new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SVG_BUSINESS_PROCESS = new ERepositoryObjectType("repository.svgBusinessProcess",
             "businessProcessSVG", "SVG_BUSINESS_PROCESS", true, 3, new String[] { PROD_DI }, new String[] {});
@@ -110,7 +108,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
     // "documentations", "DOCUMENTATION", true, 11, new String[] { PROD_DI }, new String[] {});
 
     public final static ERepositoryObjectType METADATA = new ERepositoryObjectType("repository.metadata", "metadata", "METADATA",
-            true, 8, new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
+            true, 80, new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType METADATA_CON_TABLE = new ERepositoryObjectType("repository.metadataTable",
             "METADATA_CON_TABLE", 13, true, true, new String[] { PROD_DI }, new String[] {}, false);
@@ -152,7 +150,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             "MDM_ELEMENT_TYPE", 25, true, true, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType RECYCLE_BIN = new ERepositoryObjectType("repository.recyclebin", "", "RECYCLE_BIN",
-            true, 26, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false);
+            true, 900, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType METADATA_COLUMN = new ERepositoryObjectType("repository.column", "",
             "METADATA_COLUMN", true, 27, new String[] { PROD_DI }, new String[] {}, false);
@@ -187,22 +185,6 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             "REFERENCED_PROJECTS", 51, true, "repository.referencedProjects.alias", new String[] { PROD_DI, PROD_DQ },
             new String[] {}, false);
 
-    public final static ERepositoryObjectType GENERATED = new ERepositoryObjectType("repository.generated", //$NON-NLS-1$
-            "documentations/generated", "GENERATED", true, 52, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOBS = new ERepositoryObjectType(
-            "repository.jobs", "documentations/generated/jobs", //$NON-NLS-1$
-            "JOBS", true, 53, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOB_DOC = new ERepositoryObjectType("repository.jobdoc", //$NON-NLS-1$
-            "documentations/generated/jobs", "JOB_DOC", true, 54, new String[] { PROD_DI }, new String[] {});
-
-    public final static ERepositoryObjectType JOBLETS = new ERepositoryObjectType("repository.joblets", //$NON-NLS-1$
-            "documentations/generated/joblets", "JOBLETS", true, 58, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOBLET_DOC = new ERepositoryObjectType("repository.jobletdoc", //$NON-NLS-1$
-            "documentations/generated/joblets", "JOBLET_DOC", true, 59, new String[] { PROD_DI }, new String[] {});
-
     public final static ERepositoryObjectType COMPONENTS = new ERepositoryObjectType("repository.components", "components", //$NON-NLS-1$
             "COMPONENTS", true, 61, new String[] { PROD_DI }, new String[] {}, false);
 
@@ -364,10 +346,10 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             true, "repository.systemIndicators.phoneNumberStatistics.alias", new String[] { PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SERVICESOPERATION = new ERepositoryObjectType("repository.servicesOperation", //$NON-NLS-1$
-            "SERVICESOPERATION", 101, true, true, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false); //$NON-NLS-1$
+            "SERVICESOPERATION", 101, true, true, new String[] { PROD_DI }, new String[] {}, false); //$NON-NLS-1$
 
     public final static ERepositoryObjectType SERVICESPORT = new ERepositoryObjectType("repository.servicesPort", "SERVICESPORT", //$NON-NLS-1$ //$NON-NLS-2$
-            102, true, true, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false);
+            102, true, true, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType SYSTEM_INDICATORS_FRAUDDETECTION = new ERepositoryObjectType(
             "repository.systemIndicators.fraudDetection", "TDQ_Libraries/Indicators/System Indicators/Fraud Detection", //$NON-NLS-1$ //$NON-NLS-2$
@@ -439,8 +421,22 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType SQLPATTERNS = ERepositoryObjectType.valueOf("SQLPATTERNS"); //$NON-NLS-1$
 
+    /**
+     * Docs
+     * 
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
     public final static ERepositoryObjectType DOCUMENTATION = ERepositoryObjectType.valueOf("DOCUMENTATION"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType GENERATED = ERepositoryObjectType.valueOf("GENERATED");//$NON-NLS-1$
+
+    public final static ERepositoryObjectType JOB_DOC = ERepositoryObjectType.valueOf("JOB_DOC");//$NON-NLS-1$
+
+    public final static ERepositoryObjectType JOBLET_DOC = ERepositoryObjectType.valueOf("JOBLET_DOC");//$NON-NLS-1$
+
+    /**
+     * Metadatas
+     */
     public final static ERepositoryObjectType METADATA_FILE_POSITIONAL = ERepositoryObjectType
             .valueOf("METADATA_FILE_POSITIONAL"); //$NON-NLS-1$
 
@@ -463,11 +459,6 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType METADATA_MDMCONNECTION = ERepositoryObjectType.valueOf("METADATA_MDMCONNECTION"); //$NON-NLS-1$
 
-    /**
-     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
-     */
-    public final static ERepositoryObjectType PROCESS_MR = ERepositoryObjectType.valueOf("PROCESS_MR"); //$NON-NLS-1$
-
     public final static ERepositoryObjectType METADATA_CONNECTIONS = ERepositoryObjectType.valueOf("METADATA_CONNECTIONS");
 
     public final static ERepositoryObjectType METADATA_FILE_DELIMITED = ERepositoryObjectType.valueOf("METADATA_FILE_DELIMITED");
@@ -485,6 +476,11 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType METADATA_EDIFACT = ERepositoryObjectType.valueOf("METADATA_EDIFACT");
 
+    /**
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
+    public final static ERepositoryObjectType PROCESS_MR = ERepositoryObjectType.valueOf("PROCESS_MR"); //$NON-NLS-1$
+
     /**
      * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
      */
@@ -518,6 +514,18 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      */
     public final static ERepositoryObjectType PROCESS_SPARKSTREAMING = ERepositoryObjectType.valueOf("PROCESS_SPARKSTREAMING");
 
+    public final static ERepositoryObjectType METADATA_SAP_BW_DATASOURCE = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_DATASOURCE"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_DATASTOREOBJECT = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_DATASTOREOBJECT"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_INFOCUBE = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_INFOCUBE"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_INFOOBJECT = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_INFOOBJECT"); //$NON-NLS-1$
+
     private static Map<String, ERepositoryObjectType> typeCacheById = new HashMap<String, ERepositoryObjectType>();
 
     ERepositoryObjectType(String key, String folder, String type, boolean isStaticNode, int ordinal, String[] products,
@@ -613,8 +621,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             // limit with DI items only.
             List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
             for (ERepositoryObjectType currentType : values(ERepositoryObjectType.class)) {
-                if (ArrayUtils.contains(currentType.getProducts(), PROD_DI)
-                        || ArrayUtils.contains(currentType.getProducts(), PROD_CAMEL)) {
+                if (ArrayUtils.contains(currentType.getProducts(), PROD_DI)) {
                     toReturn.add(currentType);
                 }
             }
@@ -710,9 +717,9 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         return type;
     }
 
-    
     /**
      * Getter for typeCacheById.
+     * 
      * @return the typeCacheById
      */
     public static Map<String, ERepositoryObjectType> getTypeCacheById() {
@@ -733,12 +740,12 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static String getFolderName(ERepositoryObjectType type) {
 
-        if (type == GENERATED || type == JOBS || type == JOB_DOC) {
+        if (type == GENERATED || type == JOB_DOC) {
             if ((PluginChecker.isDocumentationPluginLoaded())) {
                 return type.getFolder();
             }
         }
-        if (type == JOBLETS || type == JOBLET_DOC) {
+        if (type == JOBLET_DOC) {
             if (PluginChecker.isJobLetPluginLoaded()) {
                 return type.getFolder();
             }
@@ -846,6 +853,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      * Please use instead the method : public static ERepositoryObjectType getType(Property property)<br>
      * 
      * This method might change to private in the future, since many calls will have low performance.
+     * 
      * @param item
      * @return
      */
@@ -1361,6 +1369,10 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         if (ERepositoryObjectType.PROCESS_ROUTELET != null) {
             allTypes.add(ERepositoryObjectType.PROCESS_ROUTELET);
         }
+        // testContainer is also instance of process
+        if (ERepositoryObjectType.TEST_CONTAINER != null) {
+            allTypes.add(ERepositoryObjectType.TEST_CONTAINER);
+        }
 
         return allTypes;
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/temp/ECodePart.java

@@ -34,9 +34,11 @@ public enum ECodePart {
     SPARKCODE("sparkcode"), //$NON-NLS-1$
     SPARKCONFIG("sparkconfig"), //$NON-NLS-1$
     SPARKJOBFOOTER("sparkjobfooter"), //$NON-NLS-1$
+    SPARKPOSTPROCESSING("sparkpostprocessing"), //$NON-NLS-1$
     SPARKSTREAMINGCODE("sparkstreamingcode"), //$NON-NLS-1$
     SPARKSTREAMINGCONFIG("sparkstreamingconfig"), //$NON-NLS-1$
-    SPARKSTREAMINGJOBFOOTER("sparkstreamingjobfooter"); //$NON-NLS-1$
+    SPARKSTREAMINGJOBFOOTER("sparkstreamingjobfooter"), //$NON-NLS-1$
+    SPARKSTREAMINGPOSTPROCESSING("sparkstreamingpostprocessing"); //$NON-NLS-1$
 
     private ECodePart(String name) {
         this.name = name;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/update/RepositoryUpdateManager.java

@@ -93,6 +93,7 @@ import org.talend.core.service.IMRProcessService;
 import org.talend.core.service.IMetadataManagmentService;
 import org.talend.core.service.IStormProcessService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.runprocess.ItemCacheManager;
 import org.talend.repository.model.IProxyRepositoryFactory;
@@ -1642,6 +1643,11 @@ public abstract class RepositoryUpdateManager {
 
     public static boolean updateMultiSchema(ConnectionItem connItem, List<IMetadataTable> oldMetadataTable,
             Map<String, String> oldTableMap) {
+        return updateMultiSchema(connItem, oldMetadataTable, oldTableMap, null);
+    }
+
+    public static boolean updateMultiSchema(ConnectionItem connItem, List<IMetadataTable> oldMetadataTable,
+            Map<String, String> oldTableMap, String bwTableType) {
         if (connItem == null) {
             return false;
         }
@@ -1653,8 +1659,8 @@ public abstract class RepositoryUpdateManager {
 
         if (!update) {
             if (oldMetadataTable != null) {
-                List<IMetadataTable> newMetadataTable = RepositoryUpdateManager.getConversionMetadataTables(connItem
-                        .getConnection());
+                List<IMetadataTable> newMetadataTable = RepositoryUpdateManager.getConversionMetadataTables(
+                        connItem.getConnection(), bwTableType);
                 update = !RepositoryUpdateManager.sameAsMetadatTable(newMetadataTable, oldMetadataTable, oldTableMap);
                 isAddColumn = isAddColumn(newMetadataTable, oldMetadataTable);
             }
@@ -1665,9 +1671,14 @@ public abstract class RepositoryUpdateManager {
 
             List<IMetadataTable> newMetadataTable = new ArrayList<IMetadataTable>();
             if (coreService != null
-                    && ((connection instanceof DatabaseConnection) || (connection instanceof GenericSchemaConnection))) {
-                Set<org.talend.core.model.metadata.builder.connection.MetadataTable> newTables = ConnectionHelper
-                        .getTables(connection);
+                    && ((connection instanceof DatabaseConnection) || (connection instanceof GenericSchemaConnection))
+                    || (connection instanceof SAPConnection)) {
+                Set<org.talend.core.model.metadata.builder.connection.MetadataTable> newTables = null;
+                if (bwTableType == null) {
+                    newTables = ConnectionHelper.getTables(connection);
+                } else {
+                    newTables = SAPBWTableHelper.getBWTables(connection, bwTableType);
+                }
                 if (newTables != null) {
                     for (org.talend.core.model.metadata.builder.connection.MetadataTable originalTable : newTables) {
                         IMetadataTable conversionTable = coreService.convert(originalTable);
@@ -1783,6 +1794,15 @@ public abstract class RepositoryUpdateManager {
                 String tableId = newTable.getId();
                 if (tableLabel.equals(oldtableLabel)) {
                     isDeleted = false;
+                    String newInnerIOType = newTable.getAdditionalProperties().get(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE);
+                    String oldInnerIOType = oldTable.getAdditionalProperties().get(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE);
+                    if (newInnerIOType != null) {
+                        if (newInnerIOType.equals(oldInnerIOType) && !tableId.equals(oldtableId)) {
+                            prefix = connItem.getProperty().getId() + UpdatesConstants.SEGMENT_LINE;
+                            deletedOrReselectTables.put(prefix + tableLabel, EUpdateResult.RELOAD);
+                        }
+                        continue;
+                    }
                     /* if table name is same but tableId is not same,means table has been deselect and reselect */
                     if (!tableId.equals(oldtableId)) {
                         prefix = connItem.getProperty().getId() + UpdatesConstants.SEGMENT_LINE;
@@ -2197,14 +2217,22 @@ public abstract class RepositoryUpdateManager {
         return repositoryUpdateManager.doWork();
     }
 
-    @SuppressWarnings("unchecked")
     public static List<IMetadataTable> getConversionMetadataTables(Connection conn) {
+        return getConversionMetadataTables(conn, null);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static List<IMetadataTable> getConversionMetadataTables(Connection conn, String bwTableType) {
         if (conn == null) {
             return Collections.emptyList();
         }
         List<IMetadataTable> tables = new ArrayList<IMetadataTable>();
-
-        Set tables2 = ConnectionHelper.getTables(conn);
+        Set tables2 = null;
+        if (bwTableType != null) {
+            tables2 = SAPBWTableHelper.getBWTables(conn, bwTableType);
+        } else {
+            tables2 = ConnectionHelper.getTables(conn);
+        }
         if (tables2 != null) {
             for (org.talend.core.model.metadata.builder.connection.MetadataTable originalTable : (Set<org.talend.core.model.metadata.builder.connection.MetadataTable>) tables2) {
                 if (GlobalServiceRegister.getDefault().isServiceRegistered(IMetadataManagmentService.class)) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/AbstractDragAndDropServiceHandler.java

@@ -12,6 +12,9 @@
 // ============================================================================
 package org.talend.core.model.utils;
 
+import java.util.List;
+
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.utils.TalendQuoteUtils;
@@ -57,7 +60,7 @@ public abstract class AbstractDragAndDropServiceHandler implements IDragAndDropS
     }
 
     @Override
-    public boolean isGenericRepositoryValue(Connection connection, String paramName) {
+    public boolean isGenericRepositoryValue(List<ComponentProperties> componentProperties, String paramName) {
         return false;
     }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/IDragAndDropServiceHandler.java

@@ -14,6 +14,7 @@ package org.talend.core.model.utils;
 
 import java.util.List;
 
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
@@ -117,5 +118,5 @@ public interface IDragAndDropServiceHandler {
      * @param paramName
      * @return
      */
-    public boolean isGenericRepositoryValue(Connection connection, String paramName);
+    public boolean isGenericRepositoryValue(List<ComponentProperties> componentProperties, String paramName);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusConstants.java

@@ -20,6 +20,8 @@ public class NexusConstants {
 
     public static final String SERVICES_SEARCH = "service/local/lucene/search?";//$NON-NLS-1$ 
 
+    public static final String SERVICES_RESOLVE = "service/local/artifact/maven/resolve?";//$NON-NLS-1$ 
+
     public static final String CONTENT_REPOSITORIES = "/content/repositories/";//$NON-NLS-1$ 
 
     public static final String SLASH = "/";//$NON-NLS-1$ 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusServerUtils.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.nexus;
 
+import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.net.Authenticator;
 import java.net.HttpURLConnection;
@@ -313,6 +314,47 @@ public class NexusServerUtils {
         }
     }
 
+    public static String resolveSha1(String nexusUrl, final String userName, final String password, String repositoryId,
+            String groupId, String artifactId, String version) throws Exception {
+        HttpURLConnection urlConnection = null;
+        final Authenticator defaultAuthenticator = NetworkUtil.getDefaultAuthenticator();
+        if (userName != null && !"".equals(userName)) {
+            Authenticator.setDefault(new Authenticator() {
+
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(userName, password.toCharArray());
+                }
+
+            });
+        }
+        try {
+            String service = NexusConstants.SERVICES_RESOLVE + "a=" + artifactId + "&g=" + groupId + "&r=" + repositoryId + "&v="
+                    + version;
+            urlConnection = getHttpURLConnection(nexusUrl, service, userName, password);
+            SAXReader saxReader = new SAXReader();
+
+            InputStream inputStream = urlConnection.getInputStream();
+            Document document = saxReader.read(inputStream);
+
+            Node sha1Node = document.selectSingleNode("/artifact-resolution/data/sha1");
+            String sha1 = null;
+            if (sha1Node != null) {
+                sha1 = sha1Node.getText();
+            }
+            return sha1;
+
+        } catch (FileNotFoundException e) {
+            // jar not existing on remote nexus
+            return null;
+        } finally {
+            Authenticator.setDefault(defaultAuthenticator);
+            if (null != urlConnection) {
+                urlConnection.disconnect();
+            }
+        }
+    }
+
     private static String getSearchQuery(String repositoryId, String groupId, String artifactId, String version, int from,
             int count) {
         String query = "";//$NON-NLS-1$ 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/PreferenceManipulator.java

@@ -21,6 +21,7 @@ import java.util.StringTokenizer;
 import org.eclipse.jface.preference.IPersistentPreferenceStore;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.ui.PlatformUI;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.Hex;
 import org.talend.core.model.general.ConnectionBean;
 import org.talend.utils.json.JSONArray;
@@ -44,6 +45,8 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
     /** The preference store manipulated. */
     private IPreferenceStore store;
 
+    private String PROJECT_BRANCH_MANAGEMENT = "localBranchManagement";
+
     /**
      * Constructs a new PreferenceManipulator.
      * 
@@ -248,12 +251,14 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
         save();
     }
 
+    @Deprecated
     public int getLastLogonMode(String projectUrl, String projectName, String branch) {
         String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
         int mode = store.getInt(hexKey);
         return mode;
     }
 
+    @Deprecated
     public void setLastLogonMode(String projectUrl, String projectName, String branch, int mode) {
         String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
         store.setValue(hexKey, mode);
@@ -309,4 +314,26 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
         String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
         return store.getString(hexKey);
     }
+
+    public JSONObject getLogonLocalBranchStatus(String projectUrl, String projectName) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + PROJECT_BRANCH_MANAGEMENT).getBytes());
+        String jsonString = store.getString(hexKey);
+        if (jsonString == null || "".equals(jsonString))
+            return null;
+        try {
+            JSONObject jsonObject = new JSONObject(jsonString);
+            return jsonObject;
+        } catch (JSONException e) {
+            // TODO Auto-generated catch block
+            ExceptionHandler.process(e);
+        }
+        return null;
+    }
+
+    public void setLogonLocalBranchStatus(String projectUrl, String projectName, JSONObject json) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + PROJECT_BRANCH_MANAGEMENT).getBytes());
+        store.setValue(hexKey, json.toString());
+        save();
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/conf/DefaultConfsManager.java

@@ -82,7 +82,7 @@ public abstract class DefaultConfsManager {
     }
 
     private static Object getValue(Object object, String... args) {
-        if (args.length > 0) {
+        if (args.length > 0 && object instanceof JSONObject) {
             String arg = args[0];
             Object obj = ((JSONObject) object).get(arg);
             String[] newArgs = (String[]) ArrayUtils.remove(args, 0);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IDistributionsManager.java

@@ -0,0 +1,37 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IDistributionsManager {
+
+    /**
+     * 
+     * Get all Distributions for this service manager.
+     */
+    IHDistribution[] getDistributions();
+
+    /**
+     * 
+     * Find the name of distribution.
+     */
+    IHDistribution getDistribution(String name, boolean byDisplay);
+
+    /**
+     * 
+     * Find the version of distribution.
+     */
+    IHDistributionVersion getDistributionVersion(String version, boolean byDisplay);
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDConstants.java

@@ -0,0 +1,49 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IHDConstants {
+
+    /**
+     * same as ICustomDistribution.DISTRIBUTION_NAME
+     */
+    static final String DISTRIBUTION_CUSTOM = "CUSTOM";
+
+    /**
+     * same as IHortonworksDistribution.DISTRIBUTION_NAME
+     */
+    static final String DISTRIBUTION_HORTONWORKS = "HORTONWORKS";
+
+    /**
+     * same as IAmazonEMRDistribution.DISTRIBUTION_NAME
+     */
+    static final String DISTRIBUTION_AMAZON_EMR = "AMAZON_EMR";
+
+    /**
+     * must be full class name of {@link HadoopComponent}
+     */
+    static final String SERVICE_HADOOP = "org.talend.hadoop.distribution.component.HadoopComponent";
+
+    /**
+     * must be full class name of {@link SparkComponent}
+     */
+    static final String SERVICE_SPARK = "org.talend.hadoop.distribution.component.SparkComponent";
+
+    /**
+     * must be full class name of {@link SparkStreamingComponent}
+     */
+    static final String SERVICE_SPARK_STREAMING = "org.talend.hadoop.distribution.component.SparkStreamingComponent";
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDistribution.java

@@ -0,0 +1,34 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IHDistribution {
+
+    String getName();
+
+    String getDisplayName();
+
+    IHDistributionVersion getDefaultVersion();
+
+    String[] getVersionsDisplay();
+
+    IHDistributionVersion[] getHDVersions();
+
+    IHDistributionVersion getHDVersion(String v, boolean byDisplay);
+
+    boolean useCustom();
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDistributionVersion.java

@@ -0,0 +1,25 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public interface IHDistributionVersion {
+
+    IHDistribution getDistribution();
+
+    String getVersion();
+
+    String getDisplayVersion();
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/hive/HiveMetadataHelper.java

@@ -0,0 +1,225 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.hd.hive;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.talend.core.GlobalServiceRegister;
+import org.talend.core.database.EDatabaseTypeName;
+import org.talend.core.hadoop.IHadoopDistributionService;
+import org.talend.core.model.metadata.connection.hive.HiveModeInfo;
+import org.talend.core.model.metadata.connection.hive.HiveServerVersionInfo;
+import org.talend.core.runtime.hd.IDistributionsManager;
+import org.talend.core.runtime.hd.IHDistribution;
+import org.talend.core.runtime.hd.IHDistributionVersion;
+
+/**
+ * DOC ggu class global comment. Detailled comment
+ */
+public class HiveMetadataHelper {
+
+    private static IHadoopDistributionService getHadoopDistributionService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
+            return (IHadoopDistributionService) GlobalServiceRegister.getDefault().getService(IHadoopDistributionService.class);
+        }
+        return null;
+    }
+
+    private static IDistributionsManager getHiveDistributionsManager() {
+        IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+        if (hadoopDistributionService != null) {
+            return hadoopDistributionService.getHiveDistributionManager();
+        }
+        return null;
+    }
+
+    public static String[] getDistributionsDisplay() {
+        List<String> distributionItems = new ArrayList<String>();
+        IDistributionsManager hiveDistributionManager = getHiveDistributionsManager();
+        if (hiveDistributionManager != null) {
+            IHDistribution[] distributions = hiveDistributionManager.getDistributions();
+            for (IHDistribution d : distributions) {
+                if (!d.useCustom()) {// custom should be add
+                    String[] distributionVersionsDisplay = getDistributionVersionsDisplay(d.getName(), false);
+                    if (distributionVersionsDisplay == null || distributionVersionsDisplay.length == 0) {
+                        continue; // if no version support, ignore it.
+                    }
+                }
+                distributionItems.add(d.getDisplayName());
+            }
+        }
+        return distributionItems.toArray(new String[distributionItems.size()]);
+    }
+
+    public static IHDistribution getDistribution(String hiveDistribution, boolean byDisplay) {
+        return getDistribution(hiveDistribution, byDisplay, false);
+    }
+
+    public static IHDistribution getDistribution(String hiveDistribution, boolean byDisplay, boolean withDefault) {
+        IDistributionsManager hiveDistributionManager = getHiveDistributionsManager();
+        if (hiveDistributionManager != null && hiveDistribution != null) {
+            IHDistribution distribution = hiveDistributionManager.getDistribution(hiveDistribution, byDisplay);
+            if (withDefault && distribution == null) {
+                IHDistribution[] distributions = hiveDistributionManager.getDistributions();
+                if (distributions.length > 0) {
+                    distribution = distributions[0];
+                }
+            }
+            return distribution;
+        }
+        return null;
+    }
+
+    public static String[] getDistributionVersionsDisplay(String hiveDistribution, boolean byDisplay) {
+        List<String> versionsItems = new ArrayList<String>();
+        IHDistribution distribution = getDistribution(hiveDistribution, byDisplay);
+        if (distribution != null) {
+            // IHDistributionVersion[] hdVersions = distribution.getHDVersions();
+            // for (IHDistributionVersion v : hdVersions) {
+            // String[] hiveModesDisplay = getHiveModesDisplay(distribution.getName(), v.getVersion(), null, false);
+            // if (hiveModesDisplay == null || hiveModesDisplay.length == 0) {
+            // continue; // if no hive mode to support, ignore this version?
+            // }
+            // String[] hiveServersDisplay = getHiveServersDisplay(distribution.getName(), v.getVersion(), false);
+            // if (hiveServersDisplay == null || hiveServersDisplay.length == 0) {
+            // continue; // if no hive server to support, ignore this version?
+            // }
+            // String displayVersion = v.getDisplayVersion();
+            // if (displayVersion != null) {
+            // versionsItems.add(displayVersion);
+            // }
+            // }
+            versionsItems.addAll(Arrays.asList(distribution.getVersionsDisplay()));
+        }
+        return versionsItems.toArray(new String[versionsItems.size()]);
+    }
+
+    public static String[] getHiveModesDisplay(String hiveDistribution, String hiveVersion, String hiveServer, boolean byDisplay) {
+        List<String> hiveModeItems = new ArrayList<String>();
+        if (doSupportEmbeddedMode(hiveDistribution, hiveVersion, byDisplay)) {
+            hiveModeItems.add(HiveModeInfo.EMBEDDED.getDisplayName());
+        }
+        if (doSupportStandaloneMode(hiveDistribution, hiveVersion, byDisplay)) {
+            HiveServerVersionInfo mode = byDisplay ? HiveServerVersionInfo.getByDisplay(hiveServer) : HiveServerVersionInfo
+                    .getByKey(hiveServer);
+            if (mode == null) {
+                String[] hiveServersDisplay = getHiveServersDisplay(hiveDistribution, hiveVersion, byDisplay);
+                if (hiveServersDisplay != null && hiveServersDisplay.length > 0) {
+                    mode = HiveServerVersionInfo.getByDisplay(hiveServersDisplay[0]);
+                }
+            }
+
+            /*
+             * According to the tHiveXXX component to set it
+             */
+            if (mode == null // add always?
+                    || mode == HiveServerVersionInfo.HIVE_SERVER_2 // server 2
+                    || (HiveServerVersionInfo.HIVE_SERVER_1 == mode && doSupportMethod(hiveDistribution, hiveVersion, byDisplay,
+                            "doSupportHive1Standalone"))) {//$NON-NLS-1$
+                hiveModeItems.add(HiveModeInfo.STANDALONE.getDisplayName());
+            }
+        }
+        return hiveModeItems.toArray(new String[0]);
+    }
+
+    public static boolean isHiveEmbeddedMode(String dbTypeDisplay, String hiveModeDisplay) {
+        if (dbTypeDisplay != null && hiveModeDisplay != null
+                && EDatabaseTypeName.HIVE.getDisplayName().equalsIgnoreCase(dbTypeDisplay)) {
+            return HiveModeInfo.getByDisplay(hiveModeDisplay) == HiveModeInfo.EMBEDDED;
+        }
+        return false;
+    }
+
+    public static String[] getHiveServersDisplay(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        List<String> hiveServerItems = new ArrayList<String>();
+        if (doSupportHive1(hiveDistribution, hiveVersion, byDisplay)) {
+            hiveServerItems.add(HiveServerVersionInfo.HIVE_SERVER_1.getDisplayName());
+        }
+        if (doSupportHive2(hiveDistribution, hiveVersion, byDisplay)) {
+            hiveServerItems.add(HiveServerVersionInfo.HIVE_SERVER_2.getDisplayName());
+        }
+        return hiveServerItems.toArray(new String[0]);
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportHive1}.
+     */
+    public static boolean doSupportHive1(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportHive1");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportHive2}.
+     */
+    public static boolean doSupportHive2(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportHive2");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportTezForHive}.
+     */
+    public static boolean doSupportTez(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportTezForHive");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportSecurity}. also, when hive service or embedded
+     */
+    public static boolean doSupportSecurity(String hiveDistribution, String hiveVersion, String hiveMode,
+            String hiveServerVersion, boolean byDisplay) {
+        HiveModeInfo hiveModeInfo = byDisplay ? HiveModeInfo.getByDisplay(hiveMode) : HiveModeInfo.get(hiveMode);
+        HiveServerVersionInfo hiveServerVersionInfo = byDisplay ? HiveServerVersionInfo.getByDisplay(hiveServerVersion)
+                : HiveServerVersionInfo.valueOf(hiveServerVersion);
+
+        boolean supportSecurity = doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportSecurity");//$NON-NLS-1$
+        if (supportSecurity
+                && (HiveServerVersionInfo.HIVE_SERVER_2 == hiveServerVersionInfo || hiveModeInfo == HiveModeInfo.EMBEDDED)) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportEmbeddedMode}.
+     */
+    public static boolean doSupportEmbeddedMode(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportEmbeddedMode");//$NON-NLS-1$
+    }
+
+    /**
+     * return true/false of {@link HiveComponent#doSupportStandaloneMode}.
+     */
+    public static boolean doSupportStandaloneMode(String hiveDistribution, String hiveVersion, boolean byDisplay) {
+        return doSupportMethod(hiveDistribution, hiveVersion, byDisplay, "doSupportStandaloneMode");//$NON-NLS-1$
+    }
+
+    public static boolean doSupportMethod(String hiveDistribution, String hiveVersion, boolean byDisplay, String supportMethodName) {
+        IHDistribution distribution = getDistribution(hiveDistribution, byDisplay);
+        if (distribution != null) {
+            IHDistributionVersion version = distribution.getHDVersion(hiveVersion, byDisplay);
+            IHadoopDistributionService hadoopDistributionService = getHadoopDistributionService();
+
+            if (version != null && hadoopDistributionService != null) {
+                try {
+                    return hadoopDistributionService.doSupportMethod(version, supportMethodName);
+                } catch (Exception e) {
+                    // ignore if NoSuchMethodException
+                }
+            }
+        }
+        return false;
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenUrlHelper.java

@@ -208,6 +208,9 @@ public class MavenUrlHelper {
         if (parseMvnUrl == null) {
             return null;
         }
+        if (!MavenConstants.DEFAULT_LIB_GROUP_ID.equals(parseMvnUrl.getGroupId())) {
+            return mavenUri; // snapshot url same as maven url
+        }
 
         return generateMvnUrl(parseMvnUrl.getGroupId(), parseMvnUrl.getArtifactId(), parseMvnUrl.getVersion()
                 + MavenConstants.SNAPSHOT, parseMvnUrl.getType(), parseMvnUrl.getClassifier());
@@ -219,6 +222,9 @@ public class MavenUrlHelper {
         if (parseMvnUrl == null) {
             return null;
         }
+        if (!MavenConstants.DEFAULT_LIB_GROUP_ID.equals(parseMvnUrl.getGroupId())) {
+            return snapshotMvnuri; // snapshot url same as maven url
+        }
         String snapshotVerstion = parseMvnUrl.getVersion();
         String verstion = snapshotVerstion.substring(0, snapshotVerstion.indexOf(MavenConstants.SNAPSHOT));
         return generateMvnUrl(parseMvnUrl.getGroupId(), parseMvnUrl.getArtifactId(), verstion, parseMvnUrl.getType(),

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/JobInfoProperties.java

@@ -47,6 +47,8 @@ public class JobInfoProperties extends Properties {
 
     public static final String JOB_NAME = "job"; //$NON-NLS-1$
 
+    public static final String JOB_TYPE = "jobType"; //$NON-NLS-1$
+
     public static final String JOB_VERSION = "jobVersion"; //$NON-NLS-1$
 
     public static final String DATE = "date"; //$NON-NLS-1$
@@ -94,6 +96,7 @@ public class JobInfoProperties extends Properties {
 
         setProperty(JOB_ID, jobInfo.getJobId());
         setProperty(JOB_NAME, jobInfo.getJobName());
+        setProperty(JOB_TYPE, processItem.getProcess().getJobType());
         setProperty(JOB_VERSION, jobInfo.getJobVersion());
         setProperty(CONTEXT_NAME, this.contextName);
         setProperty(DATE, DATAFORMAT.format(new Date()));

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericService.java

@@ -0,0 +1,42 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2015 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.services;
+
+import java.util.List;
+import java.util.Map;
+
+import org.talend.core.IService;
+import org.talend.core.model.process.IElementParameter;
+
+/**
+ * created by ycbai on 2016年3月24日 Detailled comment
+ *
+ */
+public interface IGenericService extends IService {
+
+    /**
+     * Call method <code>callBeforeActivate()</code> of <code>parameter</code>.
+     * 
+     * 
+     * @param parameter
+     * @return
+     */
+    public boolean callBeforeActivate(IElementParameter parameter);
+    
+    /**
+     * Get all installed generic components information.
+     * 
+     */
+    public List<Map<String, String>> getAllGenericComponentsInfo();
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericWizardService.java

@@ -16,7 +16,9 @@ import java.util.List;
 
 import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.widgets.Composite;
+import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.IService;
+import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.EComponentCategory;
@@ -93,4 +95,24 @@ public interface IGenericWizardService extends IService {
      */
     public Composite creatDynamicComposite(Composite composite, Element element, EComponentCategory sectionCategory,
             boolean isCompactView);
+
+    /**
+     * Update component schema for metadata table of repository.
+     * 
+     * @param metadataTable
+     */
+    public void updateComponentSchema(MetadataTable metadataTable);
+
+    /**
+     * Update component schema for node metadata table of node.
+     * 
+     * @param componentProperties
+     * @param schemaPropertyName
+     * @param metadataTable
+     */
+    public void updateComponentSchema(ComponentProperties componentProperties, String schemaPropertyName,
+            IMetadataTable metadataTable);
+
+    public List<ComponentProperties> getAllComponentProperties(Connection connection);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/IJobletProviderService.java

@@ -20,6 +20,7 @@ import org.eclipse.ui.IEditorPart;
 import org.eclipse.ui.IWorkbenchPage;
 import org.eclipse.ui.IWorkbenchPart;
 import org.eclipse.ui.part.MultiPageEditorPart;
+import org.talend.commons.exception.PersistenceException;
 import org.talend.core.IService;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.metadata.IMetadataTable;
@@ -34,6 +35,7 @@ import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.IRepositoryEditorInput;
 import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 
@@ -132,4 +134,7 @@ public interface IJobletProviderService extends IService {
 
     public void clearJobletComponent();
 
+    public IRepositoryEditorInput getJobletProcessEditorInput(JobletProcessItem processItem, boolean load, boolean lastVersion,
+            boolean readOnly) throws PersistenceException;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/branding/IBrandingConfiguration.java

@@ -31,8 +31,6 @@ public interface IBrandingConfiguration extends IActionBarHelper {
 
     public static final String PERSPECTIVE_MDM_ID = ProductUtils.PERSPECTIVE_MDM_ID;
 
-    public static final String PERSPECTIVE_CAMEL_ID = ProductUtils.PERSPECTIVE_CAMEL_ID;
-
     /**
      * 
      * init perspective.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/images/RepositoryImageProvider.java

@@ -150,6 +150,8 @@ public class RepositoryImageProvider {
                 || type == ERepositoryObjectType.SYSTEM_INDICATORS_PHONENUMBER_STATISTICS
                 || type == ERepositoryObjectType.SYSTEM_INDICATORS_FRAUDDETECTION) {
             return ECoreImage.TDQ_INDICATOR_ICON;
+        } else if (type == ERepositoryObjectType.TDQ_DATA_PROFILING) {
+            return ECoreImage.TDQ_DATA_PROFILING_ICON;
         } else {
             IImage image = null;
             for (IRepositoryContentHandler handler : RepositoryContentManager.getHandlers()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/utils/ProductUtils.java

@@ -26,8 +26,6 @@ public final class ProductUtils {
      */
     public static final String PERSPECTIVE_DI_ID = "org.talend.rcp.perspective"; //$NON-NLS-1$
 
-    public static final String PERSPECTIVE_CAMEL_ID = "org.talend.camel.perspective"; //$NON-NLS-1$
-
     public static final String PERSPECTIVE_DQ_ID = "org.talend.dataprofiler.DataProfilingPerspective"; //$NON-NLS-1$
 
     public static final String PERSPECTIVE_MDM_ID = "org.talend.mdm.perspective"; //$NON-NLS-1$
@@ -54,7 +52,6 @@ public final class ProductUtils {
     static {
         productPerspectiveMap.put(PROD_DI, PERSPECTIVE_DI_ID);
         productPerspectiveMap.put(PROD_DQ, PERSPECTIVE_DQ_ID);
-        productPerspectiveMap.put(PROD_CAMEL, PERSPECTIVE_CAMEL_ID);
         productPerspectiveMap.put(PROD_MDM, PERSPECTIVE_MDM_ID);
         productPerspectiveMap.put(PROD_MAPPING, PERSPECTIVE_MAPPING_ID);
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -161,6 +161,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         syncRoutine(routineItem, copyToTemp, false);
     }
 
+    @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -43,6 +43,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException;
 
+    void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceToUpdate) throws SystemException;
+
     IFile getFile(Item item) throws SystemException;
 
     IFile getRoutinesFile(Item routineItem) throws SystemException;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -46,6 +46,9 @@ import org.talend.repository.model.IRepositoryNode;
 import org.talend.repository.model.nodes.IProjectRepositoryNode;
 import org.talend.repository.ui.views.IRepositoryView;
 
+import us.monoid.json.JSONException;
+import us.monoid.json.JSONObject;
+
 /**
  * ggu class global comment. Detailled comment
  */
@@ -576,4 +579,23 @@ public final class ProjectManager {
     public void clearFolderCache() {
         foldersMap.clear();
     }
+    
+    /**
+     * Returns the type of project (local / svn / git).
+     * @param project
+     * @return
+     */
+    public String getProjectType(Project project) {
+        String projectType = "local";
+        if (!currentProject.getEmfProject().isLocal()) {
+            JSONObject projectInfo;
+            try {
+                projectInfo = new JSONObject(currentProject.getEmfProject().getUrl());
+                projectType = projectInfo.getString("storage");
+            } catch (JSONException e) {
+                // nothing to do, consider local by default
+            }
+        }
+        return projectType;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryConstants.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.repository.model;
 
+import org.eclipse.core.runtime.Path;
 import org.talend.core.model.repository.ERepositoryObjectType;
 
 /**
@@ -143,14 +144,16 @@ public class RepositoryConstants {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.GENERATED.toString());
+        return ERepositoryObjectType.GENERATED != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.GENERATED.getFolder()).lastSegment());
     }
 
     public static boolean isJobsFolder(String label) {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.JOBS.toString());
+        return ERepositoryObjectType.JOB_DOC != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment());
     }
 
     /**
@@ -163,6 +166,7 @@ public class RepositoryConstants {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.JOBLETS.toString());
+        return ERepositoryObjectType.JOBLET_DOC != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment());
     }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryNodeUtilities.java

@@ -31,6 +31,8 @@ import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.hadoop.IHadoopClusterService;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.model.properties.ConnectionItem;
 import org.talend.core.model.properties.FolderItem;
 import org.talend.core.model.properties.FolderType;
@@ -41,11 +43,13 @@ import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.model.repository.ISubRepositoryObject;
 import org.talend.core.model.utils.RepositoryManagerHelper;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.i18n.Messages;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.ui.ITestContainerProviderService;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IRepositoryNode.ENodeType;
 import org.talend.repository.model.IRepositoryNode.EProperties;
@@ -75,8 +79,8 @@ public class RepositoryNodeUtilities {
         if (node.isBin()) {
             return new Path(""); //$NON-NLS-1$
         }
-        if ((node.getType() == ENodeType.STABLE_SYSTEM_FOLDER && node.getContentType() != ERepositoryObjectType.JOBS && node
-                .getContentType() != ERepositoryObjectType.JOBLETS) || node.getType() == ENodeType.SYSTEM_FOLDER) {
+        if ((node.getType() == ENodeType.STABLE_SYSTEM_FOLDER && node.getContentType() != ERepositoryObjectType.JOB_DOC && node
+                .getContentType() != ERepositoryObjectType.JOBLET_DOC) || node.getType() == ENodeType.SYSTEM_FOLDER) {
             return new Path(""); //$NON-NLS-1$
         }
         if (node.getType() == ENodeType.SIMPLE_FOLDER) {
@@ -87,11 +91,11 @@ public class RepositoryNodeUtilities {
         String label = node.getLabel();
         // checks if node is under Documentations/Generatated/Jobs
         if (node.getType() == ENodeType.STABLE_SYSTEM_FOLDER
-                && (node.getContentType() == ERepositoryObjectType.JOBS || node.getContentType() == ERepositoryObjectType.JOBLETS)) {
+                && (node.getContentType() == ERepositoryObjectType.JOB_DOC || node.getContentType() == ERepositoryObjectType.JOBLET_DOC)) {
 
             Object nodeLabel = node.getProperties(EProperties.LABEL);
 
-            if (nodeLabel == ERepositoryObjectType.JOBS) {
+            if (nodeLabel == ERepositoryObjectType.JOB_DOC) {
                 return new Path(""); //$NON-NLS-1$
             } else {
                 return getPath(node.getParent()).append(label);
@@ -463,7 +467,7 @@ public class RepositoryNodeUtilities {
         if (rootContextType == ERepositoryObjectType.REFERENCED_PROJECTS || rootNode.getType() == ENodeType.REFERENCED_PROJECT) {
             expandParentNode(view, rootNode);
         }
-        if (rootContextType != null) {
+        if (rootContextType != null && curType != null) {
 
             ERepositoryObjectType tmpType = null;
             if (curType == ERepositoryObjectType.METADATA_CON_TABLE || curType == ERepositoryObjectType.METADATA_CON_VIEW
@@ -490,8 +494,8 @@ public class RepositoryNodeUtilities {
                 tmpType = ERepositoryObjectType.METADATA;
             } else if (curType == ERepositoryObjectType.ROUTINES || curType == ERepositoryObjectType.SNIPPETS) {
                 tmpType = ERepositoryObjectType.ROUTINES;
-            } else if (curType == ERepositoryObjectType.DOCUMENTATION || curType == ERepositoryObjectType.JOB_DOC
-                    || curType == ERepositoryObjectType.JOBLET_DOC) {
+            } else if (curType.equals(ERepositoryObjectType.DOCUMENTATION) || curType.equals(ERepositoryObjectType.JOB_DOC)
+                    || curType.equals(ERepositoryObjectType.JOBLET_DOC)) {
                 tmpType = ERepositoryObjectType.DOCUMENTATION;
             }
 
@@ -570,15 +574,49 @@ public class RepositoryNodeUtilities {
 
     private static RepositoryNode getSAPSchemaFromConnection(RepositoryNode realNode, String name) {
         String[] values = name.split(" - "); //$NON-NLS-1$
-        if (values.length != 3) {
-            return null;
+        String metadataName = null;
+        String innerIOType = null;
+        if (values.length >= 2) {
+            metadataName = values[1];
         }
-        String metadataName = values[2];
-        String repositoryId = name.substring(0, name.lastIndexOf(" - ")); //$NON-NLS-1$
-        RepositoryNode functionNode = getSAPFunctionFromConnection(repositoryId);
-        for (IRepositoryNode node : functionNode.getChildren()) {
-            if (metadataName.equals(node.getProperties(EProperties.LABEL))) {
-                return (RepositoryNode) node;
+        if (values.length == 3) {
+            innerIOType = values[2];
+        }
+        if (innerIOType == null) {
+            RepositoryNode functionNode = getSAPFunctionFromConnection(realNode, name);
+            if (functionNode != null) {
+                return functionNode;
+            }
+        }
+
+        if (innerIOType == null || SAPBWTableHelper.isInnerIOType(innerIOType)) {
+            for (IRepositoryNode node : realNode.getChildren()) {
+                if (node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapTable")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapDataSource")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapDSO")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapInfoCube")) //$NON-NLS-1$
+                        || node.getLabel().equals(Messages.getString("RepositoryContentProvider.repositoryLabel.sapInfoObject"))) { //$NON-NLS-1$
+                    for (IRepositoryNode metadataNode : node.getChildren()) {
+                        IRepositoryViewObject metadataObject = metadataNode.getObject();
+                        if (metadataObject instanceof ISubRepositoryObject) {
+                            AbstractMetadataObject metadataTable = ((ISubRepositoryObject) metadataObject)
+                                    .getAbstractMetadataObject();
+                            if (metadataName.equals(metadataTable.getLabel())) {
+                                String innerIOTypeInMetaTable = null;
+                                if (metadataTable instanceof SAPBWTable) {
+                                    innerIOTypeInMetaTable = ((SAPBWTable) metadataTable).getInnerIOType();
+                                }
+                                if (innerIOType == null) {
+                                    return (RepositoryNode) metadataNode;
+                                } else {
+                                    if (innerIOType.equals(innerIOTypeInMetaTable)) {
+                                        return (RepositoryNode) metadataNode;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
             }
         }
         return null;
@@ -599,19 +637,40 @@ public class RepositoryNodeUtilities {
     }
 
     public static RepositoryNode getSAPFunctionFromConnection(String id) {
+        return getSAPFunctionFromConnection(null, id);
+    }
+
+    public static RepositoryNode getSAPFunctionFromConnection(RepositoryNode realNode, String id) {
         String[] values = id.split(" - "); //$NON-NLS-1$
         String repositoryID = values[0];
         String functionName = values[1];
 
         try {
-            final RepositoryNode realNode = getRepositoryNode(repositoryID);
+            if (realNode == null) {
+                realNode = getRepositoryNode(repositoryID);
+            }
             if (realNode.getObject() != null) {
                 if (ERepositoryObjectType.METADATA_SAPCONNECTIONS.equals(realNode.getObject().getRepositoryObjectType())) {
                     for (IRepositoryNode node : realNode.getChildren()) {
-                        if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapFunction").equals(node.getLabel())) { //$NON-NLS-1$
+                        if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapBapi").equals(node.getLabel())) {
                             for (IRepositoryNode function : node.getChildren()) {
-                                if (functionName.equals(function.getProperties(EProperties.LABEL))) {
-                                    return (RepositoryNode) function;
+                                List<IRepositoryNode> inputAndOutputs = function.getChildren();
+                                for (IRepositoryNode inputAndOutput : inputAndOutputs) {
+                                    if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapBapi.input").equals(
+                                            inputAndOutput.getLabel())) {
+                                        for (IRepositoryNode input : inputAndOutput.getChildren()) {
+                                            if (input.getProperties(EProperties.LABEL).equals(functionName)) {
+                                                return (RepositoryNode) input;
+                                            }
+                                        }
+                                    } else if (Messages.getString("RepositoryContentProvider.repositoryLabel.sapBapi.output")
+                                            .equals(inputAndOutput.getLabel())) {
+                                        for (IRepositoryNode output : inputAndOutput.getChildren()) {
+                                            if (output.getLabel().equals(functionName)) {
+                                                return (RepositoryNode) output;
+                                            }
+                                        }
+                                    }
                                 }
                             }
                         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/component/ComponentPaletteUtilities.java

@@ -108,7 +108,7 @@ public class ComponentPaletteUtilities {
                 }
 
                 if (extraPaletteEntry == null || extraPaletteEntry.size() == 0) {
-                    extraPaletteEntry = designerCoreUIService.createJobletEtnry();
+                    extraPaletteEntry = designerCoreUIService.createPaletteEtnry();
                 }
             }
         });
@@ -156,7 +156,7 @@ public class ComponentPaletteUtilities {
 
                 @Override
                 public void run() {
-                    extraPaletteEntry = designerCoreUIService.createJobletEtnry();
+                    extraPaletteEntry = designerCoreUIService.createPaletteEtnry();
                 }
             });
         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/composite/ElementsSelectionComposite.java

@@ -28,6 +28,7 @@ import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.layout.GridData;
 import org.eclipse.swt.layout.GridLayout;
 import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
 import org.talend.core.ui.advanced.composite.FilteredCheckboxTree;
 import org.talend.core.ui.advanced.composite.PatternFilter;
 
@@ -75,6 +76,22 @@ public class ElementsSelectionComposite<T> extends Composite {
         treeGridData.heightHint = 270;
     }
 
+    @Override
+    public void setEnabled(boolean enabled) {
+        super.setEnabled(enabled);
+        updateChildrenEnableStatus(this, enabled);
+    }
+
+    private void updateChildrenEnableStatus(Composite comp, boolean enabled) {
+        Control[] children = comp.getChildren();
+        for (Control control : children) {
+            control.setEnabled(enabled);
+            if (control instanceof Composite) {
+                updateChildrenEnableStatus(((Composite) control), enabled);
+            }
+        }
+    }
+
     protected void doSelectionChanged() {
         // Do nothing by default.
     }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/view/Contexts.java

@@ -42,7 +42,7 @@ public class Contexts {
             refreshView(cxtView);
         }
     }
-    
+
     public static void switchToCurContextsView(final IWorkbenchPart part) {
         final AbstractContextView cxtView = getViewWithPerspectiveIDs();
         Display.getDefault().asyncExec(new Runnable() {
@@ -60,7 +60,6 @@ public class Contexts {
     private static AbstractContextView getViewWithPerspectiveIDs() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         return getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
     }
 
@@ -91,7 +90,6 @@ public class Contexts {
     public static void refreshContextsView() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         AbstractContextView cxtView = getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
         if (cxtView != null) {
             refreshView(cxtView);
@@ -189,7 +187,6 @@ public class Contexts {
     public static void clearAll() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         AbstractContextView cxtView = getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
         refreshView(cxtView);
         if (cxtView != null) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/metadata/dialog/MetadataDialog.java

@@ -314,7 +314,7 @@ public class MetadataDialog extends Dialog {
         // TDI-29264:improve here for adapt two or more schemas with different mode,such as tHiveCreateTable etc
         boolean nodeModeFlag = false;
         for (IElementParameter param : node.getElementParameters()) {
-            if (param.getFieldType() == EParameterFieldType.SCHEMA_TYPE
+            if ((param.getFieldType() == EParameterFieldType.SCHEMA_TYPE || param.getFieldType() == EParameterFieldType.SCHEMA_REFERENCE)
                     && (param.getContext() == null || param.getContext().equals(currentTable.getAttachedConnector()))) {
                 IElementParameter schemaParam = param.getChildParameters().get("SCHEMA_TYPE");
                 if (schemaParam.getValue() != null) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/services/IDesignerCoreUIService.java

@@ -51,7 +51,7 @@ public interface IDesignerCoreUIService extends IService {
 
     void removePreferenceStorePropertyChangeListener(IPropertyChangeListener listener);
 
-    List<PaletteEntry> createJobletEtnry();
+    List<PaletteEntry> createPaletteEtnry();
 
     boolean executeCommand(IGEFProcess process, Command cmd);
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/AbstractTokenCollector.java

@@ -19,9 +19,7 @@ import us.monoid.json.JSONObject;
  */
 public abstract class AbstractTokenCollector implements ITokenCollector {
 
-    protected static final TokenKey TOKEN_STUDIO = new TokenKey("tokenStudio"); //$NON-NLS-1$
-
-    protected static final TokenKey PROPERTIES = new TokenKey("properties"); //$NON-NLS-1$
+    protected static final TokenKey PROJECTS_REPOSITORY = new TokenKey("projects.repository"); //$NON-NLS-1$
 
     public AbstractTokenCollector() {
         //
@@ -33,25 +31,12 @@ public abstract class AbstractTokenCollector implements ITokenCollector {
 
     public JSONObject collect() throws Exception {
         JSONObject result = new JSONObject();
-
-        // tokenStudio
-        JSONObject tokenStudioObject = new JSONObject();
-        result.put(TOKEN_STUDIO.getKey(), tokenStudioObject);
-        collectTokenStudio(tokenStudioObject);
-
-        // properties
-        JSONObject propertiesObject = new JSONObject();
-        tokenStudioObject.put(PROPERTIES.getKey(), propertiesObject);
-        collectProperties(propertiesObject);
+        // default is empty
 
         return result;
     }
-
-    protected void collectTokenStudio(JSONObject tokenStudioObject) throws Exception {
-        // will do something
-    }
-
+    
+    @Deprecated
     protected void collectProperties(JSONObject propertiesObject) throws Exception {
-        // will do something
     }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/DefaultTokenCollector.java

@@ -12,18 +12,14 @@
 // ============================================================================
 package org.talend.core.ui.token;
 
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.talend.commons.utils.VersionUtils;
-import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.ui.CoreUIPlugin;
 import org.talend.core.ui.branding.IBrandingService;
-import org.talend.cwm.helper.ConnectionHelper;
-import org.talend.utils.security.CryptoHelper;
+import org.talend.daikon.security.CryptoHelper;
+import org.talend.daikon.token.TokenGenerator;
 
 import us.monoid.json.JSONObject;
 
@@ -36,20 +32,37 @@ public class DefaultTokenCollector extends AbstractTokenCollector {
 
     private static final TokenKey UNIQUE_ID = new TokenKey("uniqueId"); //$NON-NLS-1$
 
-    private static final TokenKey TYPE_STUDIO = new TokenKey("typeStudio"); //$NON-NLS-1$
+    private static final TokenKey TYPE_STUDIO = new TokenKey("studio.type"); //$NON-NLS-1$
 
-    private static final TokenKey STOP_COLLECTOR = new TokenKey("stopUsageCollection"); //$NON-NLS-1$
+    private static final TokenKey STOP_COLLECTOR = new TokenKey("stop.collection"); //$NON-NLS-1$
+    
+    private static final TokenKey SYNC_NB = new TokenKey("sync.nb"); //$NON-NLS-1$
+
+    private static final TokenKey OS = new TokenKey("os"); //$NON-NLS-1$
+
+    public static final String COLLECTOR_SYNC_NB = "COLLECTOR_SYNC_NB"; //$NON-NLS-1$
 
     public DefaultTokenCollector() {
         super();
     }
 
+    public static String calcUniqueId() {
+        return TokenGenerator.generateMachineToken(new CryptoHelper(CryptoHelper.PASSPHRASE));
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.talend.core.ui.token.AbstractTokenCollector#collect()
+     */
     @Override
-    protected void collectTokenStudio(JSONObject tokenStudioObject) throws Exception {
+    public JSONObject collect() throws Exception {
+        JSONObject tokenStudioObject = new JSONObject();
         // version
-        tokenStudioObject.put(VERSION.getKey(), VersionUtils.getVersion());
+        tokenStudioObject.put(VERSION.getKey(), VersionUtils.getInternalVersion());
         // uniqueId
         tokenStudioObject.put(UNIQUE_ID.getKey(), calcUniqueId());
+
         // typeStudio
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IBrandingService.class)) {
             IBrandingService brandingService = (IBrandingService) GlobalServiceRegister.getDefault().getService(
@@ -57,35 +70,22 @@ public class DefaultTokenCollector extends AbstractTokenCollector {
             tokenStudioObject.put(TYPE_STUDIO.getKey(), brandingService.getAcronym());
             // tokenStudioObject.put(TYPE_STUDIO.getKey(), brandingService.getShortProductName());
         }
+        JSONObject jsonObject = new JSONObject();
+
+        jsonObject.put("os.name", System.getProperty("os.name"));
+        jsonObject.put("os.arch", System.getProperty("os.arch"));
+        jsonObject.put("os.version", System.getProperty("os.version"));
+        tokenStudioObject.put(OS.getKey(), jsonObject);
 
         final IPreferenceStore preferenceStore = CoreUIPlugin.getDefault().getPreferenceStore();
+        long syncNb = preferenceStore.getLong(COLLECTOR_SYNC_NB);
+        tokenStudioObject.put(SYNC_NB.getKey(), syncNb);
+
         if (!preferenceStore.getBoolean(ITalendCorePrefConstants.DATA_COLLECTOR_ENABLED)) {
             tokenStudioObject.put(STOP_COLLECTOR.getKey(), "1"); //$NON-NLS-1$
         } else {
             tokenStudioObject.put(STOP_COLLECTOR.getKey(), "0"); //$NON-NLS-1$
         }
-
-    }
-
-    public static String calcUniqueId() {
-        StringBuffer sb = new StringBuffer();
-        String macAddress = NetworkUtil.getMacAddress();
-        if (macAddress != null) {
-            sb.append(macAddress);
-            sb.append('-');
-        }
-        String osName = System.getProperty("os.name"); //$NON-NLS-1$
-        String osVersion = System.getProperty("os.version"); //$NON-NLS-1$
-        sb.append(osName);
-        sb.append(osVersion);
-        try {
-            String hostName = InetAddress.getLocalHost().getHostName();
-            sb.append('-');
-            sb.append(hostName);
-        } catch (UnknownHostException e) {
-            //
-        }
-        CryptoHelper cryptoHelper = new CryptoHelper(ConnectionHelper.PASSPHRASE);
-        return cryptoHelper.encrypt(sb.toString());
+        return tokenStudioObject;
     }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/TokenCollectorFactory.java

@@ -12,6 +12,8 @@
 // ============================================================================
 package org.talend.core.ui.token;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.net.Authenticator;
@@ -20,6 +22,8 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 import org.apache.log4j.Logger;
 import org.apache.log4j.Priority;
@@ -32,7 +36,7 @@ import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.ui.preferences.ScopedPreferenceStore;
-import org.talend.commons.ui.runtime.exception.ExceptionHandler;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.prefs.ITalendCorePrefConstants;
@@ -58,6 +62,7 @@ public final class TokenCollectorFactory {
     private static TokenCollectorFactory factory;
 
     private static final Map<String, TokenInforProvider> providers;
+
     static {
         providers = new HashMap<String, TokenInforProvider>();
 
@@ -122,11 +127,10 @@ public final class TokenCollectorFactory {
             if (collector != null) {
                 JSONObject collectionObject = collector.collect();
                 if (collectionObject != null) {
-                    TokenInforUtil.integrateJSONObject(result, collectionObject);
+                    TokenInforUtil.mergeJSON(collectionObject, result);
                 }
             }
         }
-
         return result;
     }
 
@@ -135,6 +139,11 @@ public final class TokenCollectorFactory {
         if (preferenceStore.getBoolean(ITalendCorePrefConstants.DATA_COLLECTOR_ENABLED)) {
             String last = preferenceStore.getString(ITalendCorePrefConstants.DATA_COLLECTOR_LAST_TIME);
             int days = preferenceStore.getInt(ITalendCorePrefConstants.DATA_COLLECTOR_UPLOAD_PERIOD);
+            
+            long syncNb = preferenceStore.getLong(DefaultTokenCollector.COLLECTOR_SYNC_NB);
+            if (syncNb < 15) {
+                days = 2;
+            }
             Date lastDate = null;
             if (last != null && !"".equals(last.trim())) { //$NON-NLS-1$
                 // parse the last date;
@@ -167,20 +176,12 @@ public final class TokenCollectorFactory {
 
         // collect
         try {
-            if (isActiveAndValid(true)) {
+            if (isActiveAndValid(false)) {
+                // collect the data each time, if the token is active
                 TokenCollectorFactory.getFactory().priorCollect();
+            }
+            if (isActiveAndValid(true)) {
                 send();
-                // set new days
-                final IPreferenceStore preferenceStore = CoreUIPlugin.getDefault().getPreferenceStore();
-                preferenceStore.setValue(ITalendCorePrefConstants.DATA_COLLECTOR_LAST_TIME, DATE_FORMAT.format(new Date()));
-                if (preferenceStore instanceof ScopedPreferenceStore) {
-                    try {
-                        ((ScopedPreferenceStore) preferenceStore).save();
-                    } catch (IOException e) {
-                        ExceptionHandler.process(e);
-                    }
-                }
-
                 result = true;
             }
         } catch (Exception e) {
@@ -210,7 +211,6 @@ public final class TokenCollectorFactory {
                         Authenticator defaultAuth = NetworkUtil.getDefaultAuthenticator();
                         try {
                             JSONObject tokenInfors = collectTokenInfors();
-
                             Resty r = new Resty();
                             // set back the rath for Resty.
                             Field rathField = Resty.class.getDeclaredField("rath"); //$NON-NLS-1$
@@ -218,16 +218,34 @@ public final class TokenCollectorFactory {
                             Authenticator auth = (Authenticator) rathField.get(null);
                             Authenticator.setDefault(auth);
 
-                            AbstractContent ac = Resty.content(tokenInfors);
+                            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                            GZIPOutputStream gzos = new GZIPOutputStream(baos);
+                            gzos.write(tokenInfors.toString().getBytes());
+                            gzos.close();
+                            AbstractContent ac = Resty.content(baos.toByteArray());
+                            baos.close();
                             MultipartContent mpc = Resty.form(new FormData("data", ac)); //$NON-NLS-1$
 
-                            TextResource result = r.text("https://www.talend.com/TalendRegisterWS/tokenstudio.php", mpc); //$NON-NLS-1$
+                            TextResource result = r.text("https://www.talend.com/TalendRegisterWS/tokenstudio_v2.php", mpc); //$NON-NLS-1$
                             String resultStr = new JSONObject(result.toString()).getString("result"); //$NON-NLS-1$
                             boolean okReturned = (resultStr != null && resultStr.endsWith("OK")); //$NON-NLS-1$
-                            // returned value not checked yet, we might need in the future, but actually the token is
-                            // still "optional".
+                            if (okReturned) {
+                                // set new days
+                                final IPreferenceStore preferenceStore = CoreUIPlugin.getDefault().getPreferenceStore();
+                                preferenceStore.setValue(ITalendCorePrefConstants.DATA_COLLECTOR_LAST_TIME, DATE_FORMAT.format(new Date()));
+                                if (preferenceStore instanceof ScopedPreferenceStore) {
+                                    try {
+                                        ((ScopedPreferenceStore) preferenceStore).save();
+                                    } catch (IOException e) {
+                                        ExceptionHandler.process(e);
+                                    }
+                                }
+                                long syncNb = preferenceStore.getLong(DefaultTokenCollector.COLLECTOR_SYNC_NB);
+                                syncNb++;
+                                preferenceStore.setValue(DefaultTokenCollector.COLLECTOR_SYNC_NB, syncNb);
+                            }
                         } catch (Exception e) {
-                            // nothing
+                            ExceptionHandler.process(e);
                         } finally {
                             Authenticator.setDefault(defaultAuth);
                         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/TokenInforUtil.java

@@ -21,6 +21,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import us.monoid.json.JSONArray;
 import us.monoid.json.JSONException;
@@ -48,46 +49,6 @@ public final class TokenInforUtil {
         return 0;
     }
 
-    /**
-     * 
-     * ggu Comment method "integrateJSONObject".
-     * 
-     * if have same key and is not json value, will be error
-     */
-    public static void integrateJSONObject(JSONObject target, JSONObject source) throws Exception {
-        if (target != null && source != null) {
-            Iterator keys = source.keys();
-            while (keys.hasNext()) {
-                String key = (String) keys.next();
-                Object sourceValue = source.get(key);
-                if (target.has(key)) {
-                    Object targetValue = target.get(key);
-                    boolean valid = true;
-                    if (sourceValue instanceof JSONObject && targetValue instanceof JSONObject) {
-                        integrateJSONObject((JSONObject) targetValue, (JSONObject) sourceValue);
-                        valid = true;
-                    } else if (sourceValue instanceof JSONArray && targetValue instanceof JSONArray) {
-                        JSONArray sourceArray = (JSONArray) sourceValue;
-                        JSONArray targetArray = (JSONArray) targetValue;
-
-                        for (int i = 0; i < sourceArray.length(); i++) {
-                            targetArray.put(sourceArray.get(i));
-                        }
-                        valid = true;
-                    } else {
-                        throw new IllegalArgumentException("Have same value existed in target: " + key + "," + targetValue);
-                    }
-                    if (!valid) {
-                        throw new IllegalArgumentException("the type is not match, target is "
-                                + targetValue.getClass().getSimpleName());
-                    }
-                } else { // if not contain, add directly.
-                    target.put(key, sourceValue);
-                }
-            }
-        }
-    }
-
     /**
      * 
      * ggu Comment method "getDateAfter".
@@ -141,4 +102,46 @@ public final class TokenInforUtil {
         }
         return topComponentsArray;
     }
+
+    public static void mergeJSON(JSONObject source, JSONObject target) throws JSONException {
+        Iterator<String> keys = source.keys();
+        while (keys.hasNext()) {
+            String key = keys.next();
+            Object o = source.get(key);
+            if (target.has(key)) {
+                // concatenate the data
+                if (o instanceof Integer) {
+                    // need to add to the total
+                    int nbSource = source.getInt(key);
+                    int nbTarget = target.getInt(key);
+                    target.put(key, nbSource + nbTarget);
+                } else if (o instanceof JSONObject) {
+                    JSONObject objectSource = (JSONObject) o;
+                    JSONObject objectTarget = target.getJSONObject(key);
+                    mergeJSON(objectSource, objectTarget);
+                } else if (o instanceof JSONArray) {
+                    JSONArray sourceArray = (JSONArray) o;
+                    JSONArray targetArray = target.getJSONArray(key);
+
+                    Set<Object> data = new HashSet<>();
+                    for (int i = 0; i < sourceArray.length(); i++) {
+                        data.add(sourceArray.get(i));
+                    }
+                    for (int i = 0; i < targetArray.length(); i++) {
+                        data.add(targetArray.get(i));
+                    }
+                    targetArray = new JSONArray();
+                    for (Object obj : data) {
+                        targetArray.put(obj);
+                    }
+                    target.put(key, targetArray);
+                } else {
+                    // for simple string / other data
+                    target.put(key, o);
+                }
+            } else {
+                target.put(key, o);
+            }
+        }
+    }
 }

main/plugins/org.talend.core/src/main/java/org/talend/core/model/repository/documentation/DocumentationHelper.java

@@ -18,6 +18,7 @@ import java.util.List;
 
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Path;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.CorePlugin;
@@ -124,10 +125,12 @@ public class DocumentationHelper {
                     && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.JOBLET.toString());
             boolean isNotGenerated = ERepositoryObjectType.GENERATED != null
                     && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.GENERATED.toString());
-            boolean isNotJobs = ERepositoryObjectType.JOBS != null
-                    && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.JOBS.toString());
-            boolean isNotJoblets = ERepositoryObjectType.JOBLETS != null
-                    && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.JOBLETS.toString());
+            boolean isNotJobs = ERepositoryObjectType.JOB_DOC != null
+                    && !node.getProperties(EProperties.LABEL).toString()
+                            .equals(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment());
+            boolean isNotJoblets = ERepositoryObjectType.JOBLET_DOC != null
+                    && !node.getProperties(EProperties.LABEL).toString()
+                            .equals(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment());
 
             if (isNotProcess && isNotJoblet && isNotGenerated && isNotJobs && isNotJoblets) {
                 folderName = node.getProperties(EProperties.LABEL).toString();
@@ -344,7 +347,7 @@ public class DocumentationHelper {
 
             // Goto the Node "Jobs";
             for (IRepositoryNode subNode : node.getChildren()) {
-                if (subNode.getContentType() != ERepositoryObjectType.JOBS) {
+                if (subNode.getContentType() != ERepositoryObjectType.JOB_DOC) {
                     continue;
                 }
 

main/plugins/org.talend.core/src/main/java/org/talend/core/model/repository/documentation/generation/JobHTMLScriptsManager.java

@@ -20,6 +20,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Path;
 import org.talend.core.model.genhtml.HTMLDocUtils;
 import org.talend.core.model.genhtml.IHTMLDocConstants;
 import org.talend.core.model.properties.Item;
@@ -144,12 +145,13 @@ public class JobHTMLScriptsManager implements IDocumentationManager {
 
         String subFolder = null;
 
-        if (targetPath.endsWith(ERepositoryObjectType.GENERATED.toString().toLowerCase())) {
+        if (ERepositoryObjectType.GENERATED != null
+                && targetPath.endsWith(new Path(ERepositoryObjectType.GENERATED.getFolder()).lastSegment().toLowerCase())) {
 
-            if (item instanceof JobDocumentationItem) {
-                subFolder = ERepositoryObjectType.JOBS.toString().toLowerCase();
-            } else if (item instanceof JobletDocumentationItem) {
-                subFolder = ERepositoryObjectType.JOBLETS.toString().toLowerCase();
+            if (item instanceof JobDocumentationItem && ERepositoryObjectType.JOB_DOC != null) {
+                subFolder = new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment().toLowerCase();
+            } else if (item instanceof JobletDocumentationItem && ERepositoryObjectType.JOBLET_DOC != null) {
+                subFolder = new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment().toLowerCase();
             } else {
                 subFolder = ""; //$NON-NLS-1$
             }
@@ -158,8 +160,8 @@ public class JobHTMLScriptsManager implements IDocumentationManager {
         }
 
         // Used for generating/updating all jobs' documentaiton only.
-        if (targetPath.endsWith(ERepositoryObjectType.JOBS.toString().toLowerCase())
-                || targetPath.endsWith(ERepositoryObjectType.JOBLETS.toString().toLowerCase())) {
+        if (targetPath.endsWith(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment().toLowerCase())
+                || targetPath.endsWith(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment().toLowerCase())) {
             targetPath = targetPath + IPath.SEPARATOR + jobPath + IPath.SEPARATOR + jobName;
         }
         String version = ""; //$NON-NLS-1$

main/plugins/org.talend.designer.maven/pom.xml

@@ -1,12 +1,44 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.talend.studio</groupId>
-    <artifactId>tcommon-studio-se</artifactId>
-    <version>6.2.0-SNAPSHOT</version>
-    <relativePath>../../../</relativePath>
-  </parent>
-  <artifactId>org.talend.designer.maven</artifactId>
-  <packaging>eclipse-plugin</packaging>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.talend.studio</groupId>
+		<artifactId>tcommon-studio-se</artifactId>
+		<version>6.2.0-SNAPSHOT</version>
+		<relativePath>../../../</relativePath>
+	</parent>
+	<artifactId>org.talend.designer.maven</artifactId>
+	<packaging>eclipse-plugin</packaging>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-dependency-plugin</artifactId>
+				<version>2.8</version>
+				<executions>
+					<execution>
+						<id>copy-maven-repository</id>
+						<phase>generate-sources</phase>
+						<goals>
+							<goal>copy</goal>
+						</goals>
+						<configuration>
+							<artifactItems>
+								<artifactItem>
+									<groupId>org.talend.studio</groupId>
+									<artifactId>studio-maven-repository</artifactId>
+									<version>6.2.0-SNAPSHOT</version>
+									<type>zip</type>
+									<overWrite>true</overWrite>
+									<outputDirectory>${project.basedir}/resources/repository</outputDirectory>
+									<destFileName>maven_repository.zip</destFileName>
+								</artifactItem>
+							</artifactItems>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
 </project>

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/talendlib/TalendLibsServerManager.java

@@ -251,4 +251,10 @@ public class TalendLibsServerManager {
             String groupIdToSearch, String artifactId, String versionToSearch) throws Exception {
         return NexusServerUtils.search(nexusUrl, userName, password, repositoryId, groupIdToSearch, artifactId, versionToSearch);
     }
+
+    public String resolveSha1(String nexusUrl, String userName, String password, String repositoryId,
+            String groupId, String artifactId, String version) throws Exception {
+        return NexusServerUtils.resolveSha1(nexusUrl, userName, password, repositoryId, groupId, artifactId, version);
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/template/ETalendMavenVariables.java

@@ -54,6 +54,7 @@ public enum ETalendMavenVariables {
     JobBatAddition,
     JobShClasspath,
     JobShAddition,
+    Framework,
 
     //
     ;

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenCodesTemplatePom.java

@@ -20,6 +20,7 @@ import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
 import org.eclipse.core.resources.IFile;
 import org.talend.core.model.general.ModuleNeeded;
+import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
 import org.talend.designer.maven.utils.PomUtil;
 
@@ -63,7 +64,10 @@ public abstract class AbstractMavenCodesTemplatePom extends AbstractMavenGeneral
             }
 
             for (ModuleNeeded module : runningModules) {
-                Dependency dependency = PomUtil.createModuleDependency(module.getModuleName());
+                Dependency dependency = null;
+                if (module.getMavenUriSnapshot() != null && !module.getMavenUriSnapshot().isEmpty() && module.getStatus() != ELibraryInstallStatus.NOT_INSTALLED) {
+                    dependency = PomUtil.createModuleDependency(module.getMavenUriSnapshot());
+                }
                 if (dependency != null) {
                     existedDependencies.add(dependency);
                 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -48,8 +48,8 @@ import org.eclipse.core.runtime.IProgressMonitor;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.VersionUtils;
 import org.talend.core.model.process.IContext;
-import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.IProcess;
+import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.ProcessItem;
@@ -249,6 +249,12 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                 jobInfoProp.getProperty(JobInfoProperties.CONTEXT_NAME, context.getName()));
         checkPomProperty(properties, "talend.job.id", ETalendMavenVariables.JobId,
                 jobInfoProp.getProperty(JobInfoProperties.JOB_ID, process.getId()));
+        checkPomProperty(properties, "talend.job.type", ETalendMavenVariables.JobType,
+                jobInfoProp.getProperty(JobInfoProperties.JOB_TYPE));
+        if (process instanceof IProcess2) {
+            String framework = (String) ((IProcess2) process).getAdditionalProperties().get("FRAMEWORK"); //$NON-NLS-1$
+            checkPomProperty(properties, "talend.job.framework", ETalendMavenVariables.Framework, framework); //$NON-NLS-1$
+        }
 
         // checkPomProperty(properties, "talend.job.class", ETalendMavenVariables.JobClass, jProcessor.getMainClass());
         checkPomProperty(properties, "talend.job.class", ETalendMavenVariables.JobClass,
@@ -439,7 +445,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
     }
 
     private void setPomForHDLight(IProgressMonitor monitor) {
-        if(ProcessUtils.isHDInsight()) {
+        if (ProcessUtils.isHDInsight()) {
             try {
                 Model model = MODEL_MANAGER.readMavenModel(getPomFile());
                 List<Plugin> plugins = new ArrayList<Plugin>(model.getBuild().getPlugins());
@@ -455,7 +461,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                                     Xpp3Dom includeNode = new Xpp3Dom("include"); //$NON-NLS-1$
                                     includeNode.setValue("${talend.job.path}/contexts/*.properties"); //$NON-NLS-1$
                                     includesNode.addChild(includeNode);
-                                    
+
                                     model.getBuild().setPlugins(plugins);
                                     PomUtil.savePom(monitor, model, getPomFile());
                                     break out;