TUP-4273 : used servicemix.avro instead of avro.

Feature/tup 3998

main/features/org.talend.commons.feature/feature.xml

@@ -109,4 +109,18 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="jackson-core-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="jackson-mapper-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

main/features/org.talend.utils.feature/feature.xml

@@ -114,4 +114,38 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="jackson-core-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="jackson-mapper-asl"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.apache.servicemix.bundles.avro"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"/>
+
+   <plugin
+         id="com.thoughtworks.paranamer"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.apache.commons.compress"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

main/plugins/org.talend.core.repository/plugin.xml

@@ -17,6 +17,7 @@
    <extension-point id="repository_review_filter" name="repository_review_filter" schema="schema/repository_review_filter.exsd"/>
    <extension-point id="repository_context" name="Repository_context" schema="schema/repository_context.exsd"/>
    <extension-point id="document_generator" name="document_generator" schema="schema/document_generator.exsd"/>
+   <extension-point id="git_content_service" name="git_content_service" schema="schema/git_content_service.exsd"/>
 
    <extension
          point="org.talend.core.runtime.service">

main/plugins/org.talend.core.repository/schema/git_content_service.exsd

@@ -0,0 +1,102 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!-- Schema file written by PDE -->
+<schema targetNamespace="org.talend.core.repository" xmlns="http://www.w3.org/2001/XMLSchema">
+<annotation>
+      <appinfo>
+         <meta.schema plugin="org.talend.core.repository" id="git_content_service" name="git_content_service"/>
+      </appinfo>
+      <documentation>
+         [Enter description of this extension point.]
+      </documentation>
+   </annotation>
+
+   <element name="extension">
+      <annotation>
+         <appinfo>
+            <meta.element />
+         </appinfo>
+      </annotation>
+      <complexType>
+         <sequence>
+            <element ref="service" minOccurs="1" maxOccurs="unbounded"/>
+         </sequence>
+         <attribute name="point" type="string" use="required">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="id" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="name" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+               <appinfo>
+                  <meta.attribute translatable="true"/>
+               </appinfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <element name="service">
+      <complexType>
+         <attribute name="class" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+               <appinfo>
+                  <meta.attribute kind="java" basedOn=":org.talend.core.repository.services.IGitRepositoryService"/>
+               </appinfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="since"/>
+      </appinfo>
+      <documentation>
+         [Enter the first release in which this extension point appears.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="examples"/>
+      </appinfo>
+      <documentation>
+         [Enter extension point usage example here.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="apiinfo"/>
+      </appinfo>
+      <documentation>
+         [Enter API information here.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="implementation"/>
+      </appinfo>
+      <documentation>
+         [Enter information about supplied implementation of this extension point.]
+      </documentation>
+   </annotation>
+
+
+</schema>

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/i18n/messages.properties

@@ -163,6 +163,10 @@ ProjectRepositoryNode.sapFunctions.inputSchema=Input
 ProjectRepositoryNode.sapFunctions.outputSchema=Output
 ProjectRepositoryNode.sapIDocs=SAP iDocs
 ProjectRepositoryNode.sapTables=SAP Tables
+ProjectRepositoryNode.sapBWDataSource=SAP DataSource
+ProjectRepositoryNode.sapBWDataStoreObject=SAP DSO
+ProjectRepositoryNode.sapBWInfoCube=SAP InfoCube
+ProjectRepositoryNode.sapBWInfoObject=SAP InfoObject
 
 RepositoryDropAdapter_copyingItems=Copying items...
 RepositoryDropAdapter_errorMsg=This directory contains the locked item,it can not be moved now

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/IRepositoryFactory.java

@@ -38,6 +38,7 @@ import org.talend.core.model.repository.IRepositoryWorkUnitListener;
 import org.talend.core.model.repository.LockInfo;
 import org.talend.core.repository.utils.XmiResourceManager;
 import org.talend.repository.RepositoryWorkUnit;
+import org.talend.utils.json.JSONArray;
 
 /**
  * Defines all methods that a repository provider plug-in must provides to client.<br/>
@@ -414,4 +415,6 @@ public interface IRepositoryFactory {
     public void setStorage(String storage);
 
     public String getStorage();
+
+    public JSONArray getAllRemoteLocks();
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProjectRepositoryNode.java

@@ -39,7 +39,6 @@ import org.talend.commons.utils.data.container.Container;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
 import org.talend.core.database.EDatabaseTypeName;
-import org.talend.core.model.genhtml.IHTMLDocConstants;
 import org.talend.core.model.metadata.MetadataManager;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
@@ -47,6 +46,7 @@ import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.metadata.builder.connection.QueriesConnection;
 import org.talend.core.model.metadata.builder.connection.Query;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.model.metadata.builder.connection.SAPConnection;
 import org.talend.core.model.metadata.builder.connection.SAPFunctionUnit;
 import org.talend.core.model.metadata.builder.connection.SAPIDocUnit;
@@ -92,9 +92,9 @@ import org.talend.core.ui.ICDCProviderService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.core.ui.branding.IBrandingService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.TableHelper;
-import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.BinRepositoryNode;
@@ -980,51 +980,6 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         return false;
     }
 
-    @SuppressWarnings("rawtypes")
-    private void convertDocumentation(org.talend.core.model.general.Project newProject, Container generatedContainer,
-            RepositoryNode parent, ERepositoryObjectType type) {
-        // for folder Documentation/generated
-        // RepositoryNode generatedFolder = getRootRepositoryNode(ERepositoryObjectType.GENERATED);
-
-        // for folder Documentation/generated/jobs
-        convertDocumentation(newProject, generatedContainer, parent, type, ERepositoryObjectType.JOBS,
-                ERepositoryObjectType.JOB_DOC);
-
-        // for folder Documentation/generated/joblets
-        convertDocumentation(newProject, generatedContainer, parent, type, ERepositoryObjectType.JOBLETS,
-                ERepositoryObjectType.JOBLET_DOC);
-
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ICamelDesignerCoreService.class)) {
-            ICamelDesignerCoreService service = (ICamelDesignerCoreService) GlobalServiceRegister.getDefault().getService(
-                    ICamelDesignerCoreService.class);
-            if (service.getRouteDocsType() != null && service.getRouteDocType() != null) {
-                convertDocumentation(newProject, generatedContainer, parent, type, service.getRouteDocsType(),
-                        service.getRouteDocType());
-            }
-        }
-
-    }
-
-    @SuppressWarnings("rawtypes")
-    private void convertDocumentation(org.talend.core.model.general.Project newProject, Container generatedContainer,
-            RepositoryNode parent, ERepositoryObjectType type, ERepositoryObjectType parentDocType, ERepositoryObjectType docType) {
-        RepositoryNode docsFolder = getRootRepositoryNode(parentDocType);
-
-        Container docsNode = null;
-        for (Object object : generatedContainer.getSubContainer()) {
-            if (((Container) object).getLabel().equalsIgnoreCase(parentDocType.name().toLowerCase())) {
-                docsNode = (Container) object;
-                break;
-            }
-        }
-
-        // get the files under generated/nodes.
-        if (docsNode != null) {
-            convert(newProject, docsNode, docsFolder, docType);
-        }
-
-    }
-
     private RepositoryNode getSQLPatternNode(String parentLabel, String label) {
         if (getMergeRefProject()) {
             List<IRepositoryNode> sqlChildren = getRootRepositoryNode(ERepositoryObjectType.SQLPATTERNS).getChildren();
@@ -1093,7 +1048,7 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             Folder oFolder = new Folder((Property) container.getProperty(), type);
             boolean found = false;
             // add for bug TDI-26084, whether or not hide folders under job_doc/joblet_doc.
-            if (ERepositoryObjectType.JOB_DOC.equals(type) || ERepositoryObjectType.JOBLET_DOC.equals(type)) {
+            if (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC)) {
                 for (FolderItem delFolder : delFolderItems) {
                     String parentName = ((FolderItem) delFolder.getParent()).getProperty().getLabel();
                     String oFolderPath = oFolder.getPath();
@@ -1126,19 +1081,6 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             if (label.equals("bin") || label.startsWith(".")) { //$NON-NLS-1$ //$NON-NLS-2$
                 continue;
             }
-            // currently, only process the docs for job, joblet and route.
-            if (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC)
-                    || type.equals(ERepositoryObjectType.JOBS) || type.equals(ERepositoryObjectType.JOBLETS)
-                    || type.equals(ERepositoryObjectType.valueOf("ROUTE_DOCS"))//$NON-NLS-1$ 
-                    || type.equals(ERepositoryObjectType.valueOf("ROUTE_DOC"))) {//$NON-NLS-1$ 
-                boolean isJobDocRootFolder = ((label.indexOf("_") != -1) && (label.indexOf(".") != -1)); //$NON-NLS-1$ //$NON-NLS-2$
-                boolean isPicFolderName = label.equals(IHTMLDocConstants.PIC_FOLDER_NAME);
-
-                // Do not show job documentation root folder and Foder "pictures" on the repository view.
-                if (isJobDocRootFolder || isPicFolderName) {
-                    continue;
-                }
-            }
             // for system folder
             if (RepositoryConstants.SYSTEM_DIRECTORY.equals(label)) {
                 if (getMergeRefProject()) {
@@ -1166,10 +1108,13 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
                     parent.getChildren().add(folder);
                 }
 
-            } else if (ERepositoryObjectType.GENERATED.name().equalsIgnoreCase(label)) {
-                if (PluginChecker.isDocumentationPluginLoaded()) {
-                    convertDocumentation(newProject, container, parent, type);
-                }
+            } else
+            // ERepositoryObjectType.GENERATED
+            if (type.equals(ERepositoryObjectType.DOCUMENTATION) && label.equalsIgnoreCase("generated")) {//$NON-NLS-1$ 
+                // if (PluginChecker.isDocumentationPluginLoaded()) {
+                // use CNF content provider instead
+                // convertDocumentation(newProject, container, parent, type);
+                // }
                 continue;
             } else {
                 if (getMergeRefProject()) {
@@ -1632,6 +1577,18 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             iDocNode.setChildrenObjectType(ERepositoryObjectType.METADATA_SAP_IDOC);
             node.getChildren().add(iDocNode);
             createSAPIDocNodes(repObj, metadataConnection, iDocNode);
+
+            // 4. BW DataSource:
+            createSAPBWDataSourceNodes(repObj, metadataConnection, node, validationRules);
+
+            // 5. BW DataStoreObject:
+            createSAPBWDataStoreObjectNodes(repObj, metadataConnection, node, validationRules);
+
+            // 6. BW InfoCube:
+            createSAPBWInfoCubeNodes(repObj, metadataConnection, node, validationRules);
+
+            // 7. BW InfoObject:
+            createSAPBWInfoObjectNodes(repObj, metadataConnection, node, validationRules);
         } else if (metadataConnection instanceof SalesforceSchemaConnection) {
             createSalesforceModuleNodes(repObj, metadataConnection, node, validationRules);
         } else {
@@ -1658,6 +1615,66 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
 
     }
 
+    private void createSAPBWDataSourceNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWDataSource"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_DATASOURCE);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> datasources = ((SAPConnection) metadataConnection).getBWDataSources();
+        EList tables = new BasicEList();
+        tables.addAll(datasources);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWDataStoreObjectNodes(IRepositoryViewObject repObj, Connection metadataConnection,
+            RepositoryNode node, List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWDataStoreObject"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_DATASTOREOBJECT);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> dataStoreObjects = ((SAPConnection) metadataConnection).getBWDataStoreObjects();
+        EList tables = new BasicEList();
+        tables.addAll(dataStoreObjects);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWInfoCubeNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWInfoCube"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_INFOCUBE);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> infoCubes = ((SAPConnection) metadataConnection).getBWInfoCubes();
+        EList tables = new BasicEList();
+        tables.addAll(infoCubes);
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
+    private void createSAPBWInfoObjectNodes(IRepositoryViewObject repObj, Connection metadataConnection, RepositoryNode node,
+            List<IRepositoryViewObject> validationRules) {
+        StableRepositoryNode container = new StableRepositoryNode(node,
+                Messages.getString("ProjectRepositoryNode.sapBWInfoObject"), ECoreImage.FOLDER_CLOSE_ICON); //$NON-NLS-1$
+        container.setChildrenObjectType(ERepositoryObjectType.METADATA_CON_TABLE);
+        container.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.METADATA_SAP_BW_INFOOBJECT);
+        node.getChildren().add(container);
+
+        EList<SAPBWTable> infoObjects = ((SAPConnection) metadataConnection).getBWInfoObjects();
+        EList<SAPBWTable> tables = new BasicEList<SAPBWTable>();
+        for (SAPBWTable bwTable : infoObjects) {
+            if (!SAPBWTableHelper.IO_INNERTYPE_BASIC.equals(bwTable.getInnerIOType())) {
+                tables.add(bwTable);
+            }
+        }
+        createTables(container, repObj, tables, ERepositoryObjectType.METADATA_CON_TABLE, validationRules);
+    }
+
     private void createSalesforceModuleNodes(IRepositoryViewObject rebObj, Connection metadataConnection,
             RepositoryNode connectionNode, List<IRepositoryViewObject> validationRules) {
         EList modules = ((SalesforceSchemaConnection) metadataConnection).getModules();

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -126,6 +126,7 @@ import org.talend.repository.documentation.ERepositoryActionName;
 import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.repository.model.RepositoryConstants;
 import org.talend.utils.io.FilesUtils;
+import org.talend.utils.json.JSONArray;
 import orgomg.cwm.objectmodel.core.ModelElement;
 
 /**
@@ -2227,4 +2228,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         }
     }
 
+    public JSONArray getAllRemoteLocks() {
+        return repositoryFactoryFromProvider.getAllRemoteLocks();
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/provider/GitContentServiceProviderManager.java

@@ -0,0 +1,49 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2015 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.repository.model.provider;
+
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IExtension;
+import org.eclipse.core.runtime.IExtensionPoint;
+import org.eclipse.core.runtime.IExtensionRegistry;
+import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.repository.services.IGitContentService;
+
+public class GitContentServiceProviderManager {
+
+    public static IGitContentService getGitContentService() {
+        IGitContentService service = null;
+        IExtensionRegistry extensionRegistry = Platform.getExtensionRegistry();
+        IExtensionPoint extensionPoint = extensionRegistry.getExtensionPoint("org.talend.core.repository.git_content_service"); //$NON-NLS-1$
+        if (extensionPoint != null) {
+            IExtension[] extensions = extensionPoint.getExtensions();
+            for (IExtension extension : extensions) {
+                IConfigurationElement[] configurationElements = extension.getConfigurationElements();
+                for (IConfigurationElement configurationElement : configurationElements) {
+                    try {
+                        Object object = configurationElement.createExecutableExtension("class"); //$NON-NLS-1$
+                        if (object instanceof IGitContentService) {
+                            service = (IGitContentService) object;
+                            break;
+                        }
+                    } catch (CoreException e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+            }
+        }
+        return service;
+    }
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/repositoryObject/MetadataTableRepositoryObject.java

@@ -27,6 +27,8 @@ import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
 import org.talend.core.model.metadata.builder.connection.Concept;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.MDMConnection;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
+import org.talend.core.model.metadata.builder.connection.SAPConnection;
 import org.talend.core.model.metadata.builder.connection.SAPFunctionUnit;
 import org.talend.core.model.metadata.builder.connection.SalesforceModuleUnit;
 import org.talend.core.model.properties.ConnectionItem;
@@ -38,6 +40,7 @@ import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.repository.ISubRepositoryObject;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.SwitchHelpers;
 import org.talend.cwm.relational.TdTable;
@@ -121,6 +124,12 @@ public class MetadataTableRepositoryObject extends MetadataTable implements ISub
             moduleUnit.getTables().remove(table);
             return;
         }
+        
+        if (table.eContainer() instanceof SAPConnection && table instanceof SAPBWTable) {
+            SAPBWTableHelper.removeBWTable((Connection) table.eContainer(), ((SAPBWTable) table).getModelType(),
+                    (SAPBWTable) table);
+        }
+
         if (table.getNamespace() instanceof Package) {
             Package pkg = (Package) table.getNamespace();
             if (pkg.getOwnedElement().contains(table)) {
@@ -184,6 +193,10 @@ public class MetadataTableRepositoryObject extends MetadataTable implements ISub
         } else if (table.eContainer() instanceof SAPFunctionUnit) {
             SAPFunctionUnit funUnit = (SAPFunctionUnit) table.eContainer();
             tables = ConnectionHelper.getTables(connection, funUnit);
+        } else if (table instanceof SAPBWTable) {
+            String bwTableType = ((SAPBWTable) table).getModelType();
+            String innerIOType = ((SAPBWTable) table).getInnerIOType();
+            tables = SAPBWTableHelper.getBWTables(connection, bwTableType, innerIOType, true);
         } else {
             tables = ConnectionHelper.getTables(connection);
         }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/recyclebin/RecycleBinManager.java

@@ -20,6 +20,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.eclipse.core.resources.IProject;
+import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.common.util.URI;
 import org.eclipse.emf.ecore.resource.Resource;
@@ -88,8 +89,15 @@ public class RecycleBinManager {
         List<TalendItem> notDeletedItems = new ArrayList<TalendItem>();
         for (TalendItem deletedItem : deletedItems) {
             try {
+                final ERepositoryObjectType type = ERepositoryObjectType.getType(deletedItem.getType());
+                // ignore the generated doc in recycle bin
+                if (type != null
+                        && (type.equals(ERepositoryObjectType.JOB_DOC) || type.equals(ERepositoryObjectType.JOBLET_DOC) || type
+                                .equals(ERepositoryObjectType.valueOf("ROUTE_DOC")))) { //$NON-NLS-1$
+                    continue;
+                }
                 IRepositoryViewObject object = ProxyRepositoryFactory.getInstance().getLastVersion(project, deletedItem.getId(),
-                        deletedItem.getPath(), ERepositoryObjectType.getType(deletedItem.getType()));
+                        deletedItem.getPath(), type);
                 if (object == null) {
                     object = ProxyRepositoryFactory.getInstance().getLastVersion(project, deletedItem.getId());
                 }
@@ -164,6 +172,11 @@ public class RecycleBinManager {
         }
     }
 
+    public RecycleBin getRecycleBin(Project project) {
+        loadRecycleBin(project);
+        return projectRecyclebins.get(project.getTechnicalLabel());
+    }
+
     private void loadRecycleBin(Project project) {
         if (projectRecyclebins.get(project.getTechnicalLabel()) != null) {
             // already loaded, nothing to do. Don't do any force reload
@@ -186,6 +199,16 @@ public class RecycleBinManager {
         }
     }
 
+    public RecycleBin loadRecycleBin(IPath recycleBinIndexPath) throws Exception {
+        Resource resource = createRecycleBinResource(recycleBinIndexPath);
+        resource.load(null);
+        return loadRecycleBin(resource);
+    }
+
+    public RecycleBin loadRecycleBin(Resource resource) {
+        return (RecycleBin) EcoreUtil.getObjectByType(resource.getContents(), RecycleBinPackage.eINSTANCE.getRecycleBin());
+    }
+
     public void saveRecycleBin(Project project) {
         if (projectRecyclebins.get(project.getTechnicalLabel()) == null) {
             loadRecycleBin(project);
@@ -219,7 +242,12 @@ public class RecycleBinManager {
 
     private Resource createRecycleBinResource(Project project) {
         IProject eclipseProject = ProjectManager.getInstance().getResourceProject(project.getEmfProject());
-        URI uri = URIHelper.convert(eclipseProject.getFullPath().append(TALEND_RECYCLE_BIN_INDEX));
+
+        return createRecycleBinResource(eclipseProject.getFullPath().append(TALEND_RECYCLE_BIN_INDEX));
+    }
+
+    public Resource createRecycleBinResource(IPath recycleBinIndexPath) {
+        URI uri = URIHelper.convert(recycleBinIndexPath);
 
         XMLResourceFactoryImpl resourceFact = new XMLResourceFactoryImpl();
         XMLResource resource = (XMLResource) resourceFact.createResource(uri);
@@ -234,5 +262,4 @@ public class RecycleBinManager {
         resource.getDefaultLoadOptions().put(XMLResource.OPTION_USE_LEXICAL_HANDLER, Boolean.TRUE);
         return resource;
     }
-
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/services/IGitContentService.java

@@ -0,0 +1,12 @@
+package org.talend.core.repository.services;
+
+public interface IGitContentService {
+
+    public abstract void setMenu(Object object);
+
+    public abstract boolean isGIT();
+
+    public abstract void createDropdownCombo(Object composite);
+
+    public abstract void configureCombo(String descriptor);
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CopyObjectAction.java

@@ -426,7 +426,8 @@ public class CopyObjectAction {
             }
         }
         if (sourceNode.getObjectType() != ERepositoryObjectType.PROCESS
-                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_MR) {
+                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_MR
+                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_STORM) {
             return false;
         }
         if (!(newItem instanceof ProcessItem)) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CreateFolderAction.java

@@ -107,8 +107,9 @@ public class CreateFolderAction extends AContextualAction {
                 canWork = false;
                 break;
             case SYSTEM_FOLDER:
-                if (ERepositoryObjectType.GENERATED.equals(property) || ERepositoryObjectType.JOBS.equals(property)
-                        || ERepositoryObjectType.JOBLETS.equals(property) || ERepositoryObjectType.SQLPATTERNS.equals(property)
+                if (property == null || property.equals(ERepositoryObjectType.GENERATED)
+                        || property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
+                        || ERepositoryObjectType.SQLPATTERNS.equals(property)
                         || ERepositoryObjectType.REFERENCED_PROJECTS.equals(property)
                         || ERepositoryObjectType.SVN_ROOT.equals(property)) {
                     canWork = false;
@@ -122,7 +123,7 @@ public class CreateFolderAction extends AContextualAction {
                 }
                 break;
             case SIMPLE_FOLDER:
-                if (ERepositoryObjectType.JOB_DOC.equals(property) || ERepositoryObjectType.JOBLET_DOC.equals(property)
+                if (property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
                         || (ERepositoryObjectType.SQLPATTERNS.equals(property) && !isUnderUserDefined(node))) {
                     canWork = false;
                 } else if (property != null

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DeleteAction.java

@@ -1197,9 +1197,6 @@ public class DeleteAction extends AContextualAction {
             return true;
         }
 
-        if (node.getContentType() == ERepositoryObjectType.JOBS) {
-            return true;
-        }
         if (node.getContentType() == ERepositoryObjectType.GENERATED) {
             return true;
         }
@@ -1495,8 +1492,8 @@ public class DeleteAction extends AContextualAction {
                         label = (String) obj;
                     }
                     boolean isGointoCondition = false;
-                    if (node.getContentType() == ERepositoryObjectType.JOB_DOC
-                            || node.getContentType() == ERepositoryObjectType.JOBLET_DOC
+                    if (node.getContentType().equals(ERepositoryObjectType.JOB_DOC)
+                            || node.getContentType().equals(ERepositoryObjectType.JOBLET_DOC)
                             || RepositoryConstants.USER_DEFINED.equals(label)) {
                         visible = false;
                         isGointoCondition = true;

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DuplicateAction.java

@@ -152,8 +152,8 @@ public class DuplicateAction extends AContextualAction {
                     } else if (node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC) {
                         canWork = false;
-                    } else if (node.getContentType() == ERepositoryObjectType.JOBS
-                            || node.getContentType() == ERepositoryObjectType.JOBLETS
+                    } else if (node.getContentType() == ERepositoryObjectType.JOB_DOC
+                            || node.getContentType() == ERepositoryObjectType.JOBLET_DOC
                             || node.getContentType() == ERepositoryObjectType.GENERATED
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                             || node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC) {
@@ -511,7 +511,7 @@ public class DuplicateAction extends AContextualAction {
     private boolean isKeyword(String itemName) {
         ERepositoryObjectType itemType = sourceNode.getObjectType();
         ERepositoryObjectType[] types = { ERepositoryObjectType.PROCESS, ERepositoryObjectType.ROUTINES,
-                ERepositoryObjectType.JOBS, ERepositoryObjectType.JOBLET, ERepositoryObjectType.JOBLETS,
+                ERepositoryObjectType.JOB_DOC, ERepositoryObjectType.JOBLET, ERepositoryObjectType.JOBLET_DOC,
                 ERepositoryObjectType.JOB_SCRIPT };
         List<ERepositoryObjectType> arraysList = Arrays.asList(types);
         List<ERepositoryObjectType> typeList = new ArrayList<ERepositoryObjectType>();

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/PasteAction.java

@@ -166,8 +166,8 @@ public class PasteAction extends AContextualAction {
         IProxyRepositoryFactory proxyFactory = ProxyRepositoryFactory.getInstance();
         IRepositoryViewObject object = target.getObject();
 
-        if (target.getContentType() == ERepositoryObjectType.JOBS
-                || target.getContentType() == ERepositoryObjectType.JOBLETS
+        if (target.getContentType() == ERepositoryObjectType.JOB_DOC
+                || target.getContentType() == ERepositoryObjectType.JOBLET_DOC
                 || target.getContentType() == ERepositoryObjectType.GENERATED
                 || target.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC
                 || target.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOBLET_DOC

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/view/RepositoryLabelProvider.java

@@ -60,6 +60,7 @@ import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.repository.model.IRepositoryNode.ENodeType;
 import org.talend.repository.model.RepositoryNode;
 import org.talend.repository.ui.views.IRepositoryView;
+import org.talend.utils.string.MD5;
 
 /**
  * Label provider for the repository view. <code>DEBUG</code> boolean field specify if details (such as objects ids)
@@ -297,11 +298,12 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
 
             ImageDescriptor imageDesc = ImageUtils.createImageFromData(item.getIcon().getInnerContent());
             imageDesc = ImageUtils.scale(imageDesc, ICON_SIZE.ICON_32);
-
-            image = cachedImages.get(item.getIcon().getInnerContent());
+            String md5Desc = MD5.getMD5(item.getIcon().getInnerContent());
+            image = cachedImages.get(md5Desc);
+           
             if (image == null || image.isDisposed()) {
                 image = imageDesc.createImage();
-                cachedImages.put(item.getIcon().getInnerContent(), image);
+            	cachedImages.put(md5Desc, image);
             } else {
                 // image = imageDesc.createImage();
             }
@@ -309,7 +311,7 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
         return image;
     }
 
-    private static Map<byte[], Image> cachedImages = new HashMap<byte[], Image>();
+    private static Map<String, Image> cachedImages = new HashMap<String, Image>();
 
     @Override
     public Image getImage(Object obj) {

main/plugins/org.talend.core.runtime/META-INF/MANIFEST.MF

@@ -110,7 +110,8 @@ Require-Bundle: org.eclipse.jdt.core,
  org.talend.commons.runtime;visibility:=reexport,
  org.talend.commons.ui,
  org.ops4j.pax.url.mvn,
- org.talend.components.api
+ org.talend.components.api,
+ org.apache.servicemix.bundles.avro
 Bundle-Activator: org.talend.core.runtime.CoreRuntimePlugin
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,

main/plugins/org.talend.core.runtime/mappings/mapping_Cassandra22_datastax.xml

@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<mapping>
+    <dbms product="Cassandra" id="cassandra22_datastax_id" label="Mapping Cassandra"
+        default="true">
+        <dbTypes>
+            <dbType type="ascii" ignorePre="true" />
+            <dbType type="bigint" ignorePre="true" />
+            <dbType type="blob" ignorePre="true" />
+            <dbType type="boolean" ignorePre="true" />
+            <dbType type="counter" ignorePre="true" />
+            <dbType type="date" ignorePre="true" />
+            <dbType type="decimal" ignorePre="true" />
+            <dbType type="double" ignorePre="true" />
+            <dbType type="float" ignorePre="true" />
+            <dbType type="inet" ignorePre="true" />
+            <dbType type="int" ignorePre="true" />
+            <dbType type="list" ignorePre="true" />
+            <dbType type="map" ignorePre="true" />
+            <dbType type="set" ignorePre="true" />
+            <dbType type="smallint" ignorePre="true" />
+            <dbType type="text" ignorePre="true" />
+            <dbType type="time" ignorePre="true" />
+            <dbType type="timestamp" ignorePre="true" />
+            <dbType type="timeuuid" ignorePre="true" />
+            <dbType type="tinyint" ignorePre="true" />
+            <dbType type="uuid" ignorePre="true" />
+            <dbType type="varchar" ignorePre="true" />
+            <dbType type="varint" ignorePre="true" />
+        </dbTypes>
+
+        <language name="java">
+            <talendToDbTypes>
+                <talendType type="id_List">
+                    <dbType type="list" default="true"/>
+                </talendType>
+                <talendType type="id_Boolean">
+                    <dbType type="boolean" default="true" />
+                </talendType>
+                <talendType type="id_Byte">
+                    <dbType type="tinyint" default="true" />
+                </talendType>
+                <talendType type="id_byte[]">
+                    <dbType type="blob" default="true" />
+                </talendType>
+                <talendType type="id_Character">
+                    <dbType type="ascii" default="true" />
+                    <dbType type="text" />
+                    <dbType type="varchar" />
+                </talendType>
+                <talendType type="id_Date">
+                    <dbType type="timestamp" default="true" />
+                    <dbType type="date" />
+                </talendType>
+                <talendType type="id_BigDecimal">
+                    <dbType type="decimal" default="true" />
+                </talendType>
+                <talendType type="id_Double">
+                    <dbType type="double" default="true" />
+                </talendType>
+                <talendType type="id_Float">
+                    <dbType type="float" default="true" />
+                </talendType>
+                <talendType type="id_Integer">
+                    <dbType type="int" default="true" />
+                </talendType>
+                <talendType type="id_Long">
+                    <dbType type="bigint" default="true" />
+                    <dbType type="counter" />
+                    <dbType type="time" />
+                </talendType>
+                <talendType type="id_Object">
+                    <dbType type="varint" default="true" />
+                    <dbType type="inet" />
+                    <dbType type="map" />
+                    <dbType type="set" />
+                </talendType>    
+                <talendType type="id_Short">
+                    <dbType type="smallint" default="true" />
+                </talendType>
+                <talendType type="id_String">
+                    <dbType type="ascii" default="true" />
+                    <dbType type="text" />
+                    <dbType type="timeuuid" />
+                    <dbType type="uuid" />
+                    <dbType type="varchar" />
+                </talendType>
+            </talendToDbTypes>
+            <dbToTalendTypes>
+                <dbType type="ascii">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="bigint">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="blob">
+                    <talendType type="id_byte[]" default="true" />
+                </dbType>
+                <dbType type="boolean">
+                    <talendType type="id_Boolean" default="true" />
+                </dbType>
+                <dbType type="counter">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="date">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="decimal">
+                    <talendType type="id_BigDecimal" default="true" />
+                </dbType>
+                <dbType type="double">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="float">
+                    <talendType type="id_Float" default="true" />
+                </dbType>
+                <dbType type="inet">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="int">
+                    <talendType type="id_Integer" default="true" />
+                </dbType>
+                <dbType type="list">
+                    <talendType type="id_List" default="true" />
+                </dbType>
+                <dbType type="map">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="set">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+                <dbType type="smallint">
+                    <talendType type="id_Short" default="true" />
+                </dbType>
+                <dbType type="text">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="time">
+                    <talendType type="id_Long" default="true" />
+                </dbType>
+                <dbType type="timestamp">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="timeuuid">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="tinyint">
+                    <talendType type="id_Byte" default="true" />
+                </dbType>
+                <dbType type="uuid">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="varchar">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="varint">
+                    <talendType type="id_Object" default="true" />
+                </dbType>
+            </dbToTalendTypes>
+        </language>
+    </dbms>
+
+</mapping>

main/plugins/org.talend.core.runtime/mappings/mapping_Cassandra_datastax.xml

@@ -82,7 +82,6 @@
             <dbToTalendTypes>
                 <dbType type="ascii">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="bigint">
                     <talendType type="id_Long" default="true" />
@@ -110,8 +109,6 @@
                 </dbType>
                 <dbType type="int">
                     <talendType type="id_Integer" default="true" />
-                    <talendType type="id_Short" />
-                    <talendType type="id_Byte"/>
                 </dbType>
                 <dbType type="list">
                     <talendType type="id_List" default="true" />
@@ -124,7 +121,6 @@
                 </dbType>
                 <dbType type="text">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="timestamp">
                     <talendType type="id_Date" default="true" />
@@ -137,7 +133,6 @@
                 </dbType>
                 <dbType type="varchar">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="varint">
                     <talendType type="id_Object" default="true" />

main/plugins/org.talend.core.runtime/mappings/mapping_Redshift_new.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0"?>
+<mapping>
+	<dbms product="Redshift" id="redshift_new_id" label="Mapping Redshift"
+		default="true">
+		<dbTypes>
+            <dbType type="SMALLINT" ignoreLen="true" ignorePre="true" />
+            <dbType type="INTEGER" ignoreLen="true" ignorePre="true" />
+            <dbType type="BIGINT" ignoreLen="true" ignorePre="true" />
+            <dbType type="DECIMAL" ignoreLen="true" />
+            <dbType type="REAL" ignoreLen="true" />
+            <dbType type="DOUBLE PRECISION" ignoreLen="true" />
+            <dbType type="BOOLEAN" ignoreLen="true" ignorePre="true" />
+            <dbType type="CHAR" ignoreLen="true" ignorePre="true" />
+            <dbType type="VARCHAR" ignorePre="true" />
+            <dbType type="DATE" ignoreLen="true" ignorePre="true" />
+            <dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true" />
+		</dbTypes>
+		<language name="java">
+            <talendToDbTypes>
+                <talendType type="id_List">
+                </talendType>
+                <talendType type="id_Boolean">
+                    <dbType type="BOOLEAN" default="true" />
+                </talendType>
+                <talendType type="id_Byte">
+                </talendType>
+                <talendType type="id_byte[]">
+                </talendType>
+                <talendType type="id_Character">
+                </talendType>
+                <talendType type="id_Date">
+                    <dbType type="DATE" default="true" />
+                    <dbType type="TIMESTAMP" />
+                </talendType>
+                <talendType type="id_BigDecimal">
+                    <dbType type="DECIMAL" default="true" />
+                    <dbType type="BIGINT" />
+                </talendType>
+                <talendType type="id_Double">
+                    <dbType type="DOUBLE PRECISION" default="true" />
+                    <dbType type="REAL" />
+                </talendType>
+                <talendType type="id_Float">
+                </talendType>
+                <talendType type="id_Integer">
+                    <dbType type="INTEGER" default="true" />
+                    <dbType type="SMALLINT" />
+                </talendType>
+                <talendType type="id_Long">
+                    <dbType type="BIGINT" default="true" />
+                    <dbType type="INTEGER" />
+                </talendType>
+                <talendType type="id_Object">
+                </talendType>    
+                <talendType type="id_Short">
+                </talendType>
+                <talendType type="id_String">
+                    <dbType type="VARCHAR" default="true" />
+                    <dbType type="CHAR" />
+                </talendType>
+            </talendToDbTypes>
+            <dbToTalendTypes>
+                <dbType type="SMALLINT">
+                    <talendType type="id_Integer" default="true" />
+                </dbType>
+                <dbType type="INTEGER">
+                    <talendType type="id_Integer" default="true" />
+                    <talendType type="id_Long" />
+                </dbType>
+                <dbType type="BIGINT">
+                    <talendType type="id_Long" default="true" />
+                    <talendType type="id_BigDecimal" />
+                </dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true" />
+                </dbType>
+                <dbType type="REAL">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="DOUBLE PRECISION">
+                    <talendType type="id_Double" default="true" />
+                </dbType>
+                <dbType type="BOOLEAN">
+                    <talendType type="id_Boolean" default="true" />
+                </dbType>
+                <dbType type="CHAR">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="VARCHAR">
+                    <talendType type="id_String" default="true" />
+                </dbType>
+                <dbType type="DATE">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true" />
+                </dbType>
+            </dbToTalendTypes>		  
+		</language>
+	</dbms>
+</mapping>
+        

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/EDatabaseConnTemplate.java

@@ -81,7 +81,7 @@ public enum EDatabaseConnTemplate {
             "jdbc:odbc:<datasource>")), //$NON-NLS-1$
 
     ACCESS(new DbConnStr(EDatabaseTypeName.ACCESS, //
-            "jdbc:ucanaccess://<filename>")), //$NON-NLS-1$
+            "jdbc:ucanaccess://<filename>;<property>",null,"memory=false")), //$NON-NLS-1$
 
     SYBASEASE(new DbConnStr(EDatabaseTypeName.SYBASEASE, //
             "jdbc:sybase:Tds:<host>:<port>/<sid>?<property>", //$NON-NLS-1$ 
@@ -357,6 +357,7 @@ public enum EDatabaseConnTemplate {
             case TERADATA: // for feature 0013719
             case VERTICA:
             case EXASOL:
+            case ACCESS:
                 return true;
             default:
             }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConstants.java

@@ -32,6 +32,8 @@ public class HadoopConstants {
     public static final String FRAMEWORK_STORM = "Storm";
 
     public static final String FRAMEWORK_SPARKSTREAMING = "Spark Streaming";
+    
+    public static final String FRAMEWORK_SPARK_STREAMING = "Spark_Streaming";
 
     public static final String FRAMEWORK_MAPREDUCE = "MapReduce";
 
@@ -40,6 +42,10 @@ public class HadoopConstants {
     public static final String JOBTYPEBDSTREAMING = "Big Data Streaming";
 
     public static final String JOBTYPEBDBATCH = "Big Data Batch";
+    
+    public static final String JOBTYPE_BDSTREAMING = "Big_Data_Streaming";
+
+    public static final String JOBTYPE_BDBATCH = "Big_Data_Batch";
 
     public static final String SPARKSTREAMING_TYPE = "SPARKSTREAMING";
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopClusterService.java

@@ -216,6 +216,8 @@ public interface IHadoopClusterService extends IService {
      */
     public boolean hasDiffsFromClusterToProcess(Item item, IProcess process);
 
+    public Connection getHadoopClusterConnectionBySubConnection(Connection hadoopSubConnection);
+
     public String getHadoopClusterProperties(Connection hadoopSubConnection);
 
     /**

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/repository/HadoopRepositoryUtil.java

@@ -139,8 +139,11 @@ public class HadoopRepositoryUtil {
         }
         List<Map<String, Object>> parentProperties = null;
         if (hadoopClusterService != null) {
-            parentProperties = getHadoopPropertiesList(hadoopClusterService.getHadoopClusterProperties(connection),
-                    connection.isContextMode(), true);
+            Connection hadoopClusterConnection = hadoopClusterService.getHadoopClusterConnectionBySubConnection(connection);
+            if (hadoopClusterConnection != null) {
+                parentProperties = getHadoopPropertiesList(hadoopClusterService.getHadoopClusterProperties(connection),
+                        hadoopClusterConnection.isContextMode(), true);
+            }
         }
         List<Map<String, Object>> properties = getHadoopPropertiesList(propertiesJsonStr, connection.isContextMode(), true);
         Map<String, Map<String, Object>> propertiesMap = new HashMap<String, Map<String, Object>>();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/ComponentCategory.java

@@ -17,6 +17,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.service.IMRProcessService;
+import org.talend.core.service.IStormProcessService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 
 /**
@@ -96,6 +97,16 @@ public enum ComponentCategory {
                 return ComponentCategory.CATEGORY_4_MAPREDUCE;
             }
         }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IStormProcessService.class)) {
+            IStormProcessService stormService = (IStormProcessService) GlobalServiceRegister.getDefault().getService(
+                    IStormProcessService.class);
+            if (stormService.isSparkStreaming(item)) {
+                return ComponentCategory.CATEGORY_4_SPARKSTREAMING;
+            }
+            if (stormService.isStormItem(item)) {
+                return ComponentCategory.CATEGORY_4_STORM;
+            }
+        }
         if (item instanceof ProcessItem || item instanceof JobletProcessItem) {
             return ComponentCategory.CATEGORY_4_DI;
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleToInstall.java

@@ -43,6 +43,8 @@ public class ModuleToInstall {
 
     private String distribution = "";
 
+    private boolean fromCustomNexus;
+
     public String getName() {
         return this.name;
     }
@@ -176,4 +178,16 @@ public class ModuleToInstall {
         return this.mavenUris;
     }
 
+    /**
+     * Sets the fromCustomNexus.
+     * 
+     * @param fromCustomNexus the fromCustomNexus to set
+     */
+    public void setFromCustomNexus(boolean fromCustomNexus) {
+        this.fromCustomNexus = fromCustomNexus;
+    }
+
+    public boolean isFromCustomNexus() {
+        return this.fromCustomNexus;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -158,26 +158,15 @@ public class MetadataTable implements IMetadataTable, Cloneable {
 
     @Override
     public boolean isDynamicSchema() {
-
-        int sizeListColumns = listColumns.size();
-        boolean hasDynamic = false;
-        for (int i = 0; i < sizeListColumns; i++) {
-            if (listColumns.get(i).getTalendType().equals("id_Dynamic")) {
-                hasDynamic = true;
-                break;
-            }
-        }
-        return hasDynamic;
+        return getDynamicColumn() != null;
 
     }
 
     @Override
     public IMetadataColumn getDynamicColumn() {
-        if (isDynamicSchema()) {
-            for (int i = 0; i < listColumns.size(); i++) {
-                if (listColumns.get(i).getTalendType().equals("id_Dynamic")) {
-                    return listColumns.get(i);
-                }
+        for (IMetadataColumn column : listColumns) {
+            if (column.getTalendType().equals("id_Dynamic")) {
+                return column;
             }
         }
         return null;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolAvroHelper.java

@@ -0,0 +1,436 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.LogicalTypes.Decimal;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.SchemaBuilder.BaseFieldTypeBuilder;
+import org.apache.avro.SchemaBuilder.FieldAssembler;
+import org.apache.avro.SchemaBuilder.FieldBuilder;
+import org.apache.avro.SchemaBuilder.PropBuilder;
+import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.talend.core.model.metadata.builder.connection.ConnectionFactory;
+import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.types.JavaTypesManager;
+import org.talend.core.utils.TalendQuoteUtils;
+import org.talend.cwm.helper.TaggedValueHelper;
+import org.talend.daikon.avro.SchemaConstants;
+import org.talend.daikon.avro.util.AvroUtils;
+import org.talend.daikon.talend6.Talend6SchemaConstants;
+import orgomg.cwm.objectmodel.core.Expression;
+import orgomg.cwm.objectmodel.core.TaggedValue;
+
+/**
+ * Utility classes for conversion between a {@link MetadataTable} and an Avro {@link Schema}.
+ */
+public final class MetadataToolAvroHelper {
+
+    /**
+     * @return An Avro schema with enriched properties from the incoming metadata table.
+     */
+    public static org.apache.avro.Schema convertToAvro(MetadataTable in) {
+        RecordBuilder<Schema> builder = SchemaBuilder.builder().record(in.getLabel());
+        copyTableProperties(builder, in);
+
+        FieldAssembler<Schema> fa = builder.fields();
+        for (org.talend.core.model.metadata.builder.connection.MetadataColumn column : in.getColumns()) {
+            fa = convertToAvro(fa, column);
+        }
+        return fa.endRecord();
+    }
+
+    /**
+     * Copy all of the information from the MetadataTable in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyTableProperties(PropBuilder<T> builder, MetadataTable in) {
+
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Table-specific properties.
+        if (in.getName() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_NAME, in.getName());
+        }
+        if (in.getTableType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_TYPE, in.getTableType());
+        }
+
+        return builder;
+    }
+
+    /**
+     * Build a field into a schema using enriched properties from the incoming column.
+     */
+    private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        FieldBuilder<Schema> fb = fa.name(in.getLabel());
+        copyColumnProperties(fb, in);
+        BaseFieldTypeBuilder<Schema> ftb = in.isNullable() ? fb.type() : fb.type().nullable();
+
+        String defaultValue = null;
+        Expression initialValue = in.getInitialValue();
+        if (initialValue != null) {
+            defaultValue = initialValue.getBody();
+        }
+
+        String tt = in.getTalendType();
+
+        // Numeric types.
+        if (JavaTypesManager.LONG.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.longType().noDefault()
+                    : ftb.longType().longDefault(Long.parseLong(defaultValue));
+        }
+        if (JavaTypesManager.INTEGER.getId().equals(tt) || JavaTypesManager.SHORT.getId().equals(tt)
+                || JavaTypesManager.BYTE.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.intType().noDefault()
+                    : ftb.intType().intDefault(Integer.parseInt(defaultValue));
+        }
+        if (JavaTypesManager.DOUBLE.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.doubleType().noDefault()
+                    : ftb.doubleType().doubleDefault(Double.parseDouble(defaultValue));
+        }
+        if (JavaTypesManager.FLOAT.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.floatType().noDefault()
+                    : ftb.floatType().floatDefault(Float.parseFloat(defaultValue));
+        }
+        if (JavaTypesManager.BIGDECIMAL.getId().equals(tt)) {
+            // decimal(precision, scale) == column length and precision?
+            Decimal d = LogicalTypes.decimal((int) in.getLength(), (int) in.getPrecision());
+            Schema bigdecimal = d.addToSchema(Schema.create(Schema.Type.BYTES));
+            return fb.type(bigdecimal).withDefault(defaultValue);
+        }
+
+        // Other primitive types that map directly to Avro.
+        if (JavaTypesManager.BOOLEAN.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.booleanType().noDefault()
+                    : ftb.booleanType().booleanDefault(Boolean.parseBoolean(defaultValue));
+        }
+        if (JavaTypesManager.BYTE_ARRAY.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.bytesType().noDefault()
+                    : ftb.bytesType().bytesDefault(defaultValue);
+        }
+        if (JavaTypesManager.DATE.getId().equals(tt)) {
+            // Date is saved as a long, like System.currentTimeMillis()
+            return ftb.longType().noDefault();
+        }
+        // String-ish types.
+        if (JavaTypesManager.STRING.getId().equals(tt) || JavaTypesManager.FILE.getId().equals(tt)
+                || JavaTypesManager.DIRECTORY.getId().equals(tt) || JavaTypesManager.VALUE_LIST.getId().equals(tt)
+                || JavaTypesManager.CHARACTER.getId().equals(tt) || JavaTypesManager.PASSWORD.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.stringType().noDefault()
+                    : ftb.stringType().stringDefault(defaultValue);
+        }
+
+        // Types with unknown elements, store as binary
+        if (JavaTypesManager.OBJECT.getId().equals(tt) || JavaTypesManager.DYNAMIC.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.bytesType().noDefault()
+                    : ftb.bytesType().bytesDefault(defaultValue);
+        }
+        if (JavaTypesManager.LIST.getId().equals(tt)) {
+            return ftb.array().items().bytesType().noDefault();
+        }
+
+        // Can this occur?
+        throw new UnsupportedOperationException("Unrecognized type " + tt); //$NON-NLS-1$
+    }
+
+    /**
+     * Copy all of the information from the IMetadataColumn in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyColumnProperties(PropBuilder<T> builder,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Column-specific properties.
+        if (in.isKey()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
+        }
+        if (in.getType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
+        }
+        if (in.getTalendType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
+        }
+        if (in.getPattern() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN, TalendQuoteUtils.addQuotesIfNotExist(in.getPattern()));
+        }
+        if (in.getLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH, String.valueOf((int) in.getLength()));
+        }
+        if (in.getOriginalLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH, String.valueOf(in.getOriginalLength()));
+        }
+        if (in.isNullable()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
+        }
+        if (in.getPrecision() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION, String.valueOf(in.getPrecision()));
+        }
+        if (in.getInitialValue() != null && in.getInitialValue().getBody() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
+        }
+        if (in.getName() != null) {
+            // keyword fixes?
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
+        }
+        if (in.getRelatedEntity() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
+        }
+        if (in.getRelationshipType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE, in.getRelationshipType());
+        }
+        return builder;
+    }
+
+    /**
+     * @param in A schema which must be of type {@link Schema.Type#RECORD}.
+     * @return A MetadataTable containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static MetadataTable convertFromAvro(org.apache.avro.Schema in) {
+        MetadataTable table = ConnectionFactory.eINSTANCE.createMetadataTable();
+
+        // Set the defaults values to the name (the only information guaranteed to be present in the schema.)
+        table.setId(in.getName());
+        table.setLabel(in.getName());
+        table.setName(in.getName());
+        table.setSourceName(in.getName());
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            table.setId(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            table.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            table.setLabel(null);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            table.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : in.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
+                table.getTaggedValue().add(tv);
+            }
+        }
+
+        // Table-specific properties.
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_NAME))) {
+            table.setName(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_TYPE))) {
+            table.setTableType(prop);
+        }
+
+        // Add the columns.
+        List<org.talend.core.model.metadata.builder.connection.MetadataColumn> columns = new ArrayList<>(in.getFields().size());
+        for (Schema.Field f : in.getFields()) {
+            columns.add(convertFromAvro(f));
+        }
+        table.getColumns().addAll(columns);
+        return table;
+    }
+
+    /**
+     * @param in A field from an incoming schema
+     * @return A MetadataColumn containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvro(Schema.Field field) {
+        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
+        Schema in = field.schema();
+
+        // Set the defaults values to the name (the only information guaranteed to be available in every field).
+        col.setId(field.name());
+        col.setLabel(field.name());
+        col.setName(field.name());
+        Schema nonnullable = AvroUtils.unwrapIfNullable(in);
+        switch (nonnullable.getType()) {
+        case ARRAY:
+            col.setTalendType(JavaTypesManager.LIST.getId());
+            break;
+        case BOOLEAN:
+            col.setTalendType(JavaTypesManager.BOOLEAN.getId());
+            break;
+        case BYTES:
+        case FIXED:
+            col.setTalendType(JavaTypesManager.BYTE_ARRAY.getId());
+            break;
+        case DOUBLE:
+            col.setTalendType(JavaTypesManager.DOUBLE.getId());
+            break;
+        case FLOAT:
+            col.setTalendType(JavaTypesManager.FLOAT.getId());
+            break;
+        case INT:
+            col.setTalendType(JavaTypesManager.INTEGER.getId());
+            break;
+        case LONG:
+            String prop = null;
+            if (null != (prop = nonnullable.getProp(SchemaConstants.TALEND_COLUMN_PATTERN))) {
+                col.setTalendType(JavaTypesManager.DATE.getId());
+                col.setPattern(TalendQuoteUtils.addQuotesIfNotExist(prop));
+            } else {
+                col.setTalendType(JavaTypesManager.LONG.getId());
+            }
+            break;
+        case ENUM:
+        case STRING:
+            col.setTalendType(JavaTypesManager.STRING.getId());
+            break;
+        case RECORD:
+        case NULL:
+        case MAP:
+        case UNION:
+        default:
+            // Can this occur in the studio?
+            break;
+        }
+        // TODO setSourceType from the field Schema type.
+        col.setNullable(AvroUtils.isNullable(in));
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            col.setId(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            col.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            col.setLabel(null);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            col.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : in.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
+                col.getTaggedValue().add(tv);
+            }
+        }
+
+        // Column-specific properties.
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
+            col.setKey(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
+            col.setSourceType(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
+            col.setTalendType(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
+            col.setPattern(TalendQuoteUtils.addQuotesIfNotExist(prop));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setOriginalLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
+            col.setNullable(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
+            Long value = Long.parseLong(prop);
+            col.setPrecision(value > 0 ? value : -1);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
+            col.setDefaultValue(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
+            col.setName(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
+            col.setRelatedEntity(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
+            col.setRelationshipType(prop);
+        }
+
+        // If the source type wasn't set, there is an issue. Can this occur in the studio.
+        if (col.getTalendType() == null) {
+            throw new UnsupportedOperationException("Unrecognized type " + in); //$NON-NLS-1$
+        }
+
+        return col;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -604,6 +604,7 @@ public final class MetadataToolHelper {
         target.getListColumns().addAll(columnsTAdd);
         target.sortCustomColumns();
         target.setLabel(source.getLabel());
+        target.getAdditionalProperties().putAll(source.getAdditionalProperties());
     }
 
     public static void copyTable(List<IMetadataColumn> sourceColumns, IMetadataTable target, List<IMetadataColumn> targetColumns) {
@@ -1310,7 +1311,6 @@ public final class MetadataToolHelper {
             }
         }
         return null;
-
     }
 
     public static IMetadataTable convert(MetadataTable old) {
@@ -1398,5 +1398,4 @@ public final class MetadataToolHelper {
         }
         return result;
     }
-
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/builder/ConvertionHelper.java

@@ -40,9 +40,11 @@ import org.talend.core.model.metadata.builder.connection.DelimitedFileConnection
 import org.talend.core.model.metadata.builder.connection.MDMConnection;
 import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.builder.connection.SAPBWTable;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.utils.KeywordsValidator;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.model.bridge.ReponsitoryContextBridge;
 import orgomg.cwm.objectmodel.core.TaggedValue;
 
@@ -337,6 +339,12 @@ public final class ConvertionHelper {
         result.setId(old.getId());
         result.setLabel(old.getLabel());
         result.setTableType(old.getTableType());
+        if (old instanceof SAPBWTable) {
+            String innerType = ((SAPBWTable)old).getInnerIOType();
+            if (innerType != null) {
+                result.getAdditionalProperties().put(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE, innerType);
+            }
+        }
         String sourceName = old.getName();
         if (sourceName == null) {
             sourceName = old.getLabel();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -21,7 +21,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.emf.common.util.EList;
 import org.talend.commons.ui.runtime.exception.ExceptionHandler;
@@ -910,18 +909,10 @@ public class RepositoryToComponentProperty {
             }
         }
         if (value.equals("SERVER_NAME")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
         if (value.equals("PORT")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getPort())) {
-                return connection.getPort();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getPort());
-            }
+            return getAppropriateValue(connection, connection.getPort());
         }
         if (value.equals("SID") || value.equals("DATABASE_ALIAS")) { //$NON-NLS-1$ //$NON-NLS-2$
             if (("").equals(connection.getSID()) || connection.getSID() == null) { //$NON-NLS-1$
@@ -939,54 +930,25 @@ public class RepositoryToComponentProperty {
             }
         }
         if (value.equals("DATASOURCE")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getDatasourceName())) {
-                return connection.getDatasourceName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDatasourceName());
-            }
+            return getAppropriateValue(connection, connection.getDatasourceName());
         }
         if (value.equals("USERNAME")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getUsername())) {
-                return connection.getUsername();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getUsername());
-            }
+            return getAppropriateValue(connection, connection.getUsername());
         }
         if (value.equals("PASSWORD")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getPassword())) {
-                return connection.getPassword();
-            } else {
-                String pwd = TalendQuoteUtils.checkAndAddBackslashes(connection.getRawPassword());
-                return TalendQuoteUtils.addQuotes(pwd);
-            }
+            return getAppropriateValue(connection, connection.getValue(connection.getPassword(), false));
         }
         if (value.equals("NULL_CHAR")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getNullChar())) {
-                return connection.getNullChar();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getNullChar());
-            }
+            return getAppropriateValue(connection, connection.getNullChar());
         }
         if (value.equals("SCHEMA")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getUiSchema())) {
-                return connection.getUiSchema();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getUiSchema());
-            }
+            return getAppropriateValue(connection, connection.getUiSchema());
         }
         if (value.equals("FILE")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getFileFieldName())) {
-                return connection.getFileFieldName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getFileFieldName());
-            }
+            return getAppropriateValue(connection, connection.getFileFieldName());
         }
         if (value.equals("PROPERTIES_STRING")) { //$NON-NLS-1$
-            if (isContextMode(connection, connection.getAdditionalParams())) {
-                return connection.getAdditionalParams();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getAdditionalParams());
-            }
+            return getAppropriateValue(connection, connection.getAdditionalParams());
         }
 
         if (value.equals("CDC_MODE")) { //$NON-NLS-1$
@@ -1163,12 +1125,7 @@ public class RepositoryToComponentProperty {
         }
         // add this for tJavaDB embeded "DB Root Path"
         if (value.equals("DIRECTORY")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDBRootPath())) {
-                return connection.getDBRootPath();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDBRootPath());
-            }
-
+            return getAppropriateValue(connection, connection.getDBRootPath());
         }
         // add for feature 11674
         if (value.equals("RUNNING_MODE")) {//$NON-NLS-1$       
@@ -1183,25 +1140,13 @@ public class RepositoryToComponentProperty {
             return runningMode;
         }
         if (value.equals("DBPATH")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDBRootPath())) {
-                return connection.getDBRootPath();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDBRootPath());
-            }
+            return getAppropriateValue(connection, connection.getDBRootPath());
         }
         if (value.equals("DBNAME")) {//$NON-NLS-1$
-            if (isContextMode(connection, connection.getDatasourceName())) {
-                return connection.getDatasourceName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getDatasourceName());
-            }
+            return getAppropriateValue(connection, connection.getDatasourceName());
         }
         if (value.equals("RAC_URL")) {
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
 
         if (value.equals("DISTRIBUTION")) {
@@ -1229,13 +1174,15 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("HBASE_MASTER_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_MASTERPRINCIPAL));
+            String hbaseMasterPrinc = connection.getParameters().get(
+                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_MASTERPRINCIPAL);
+            return getAppropriateValue(connection, hbaseMasterPrinc);
         }
 
         if (value.equals("HBASE_REGIONSERVER_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_REGIONSERVERPRINCIPAL));
+            String hbaseRegPrinc = connection.getParameters().get(
+                    ConnParameterKeys.CONN_PARA_KEY_HBASE_AUTHENTICATION_REGIONSERVERPRINCIPAL);
+            return getAppropriateValue(connection, hbaseRegPrinc);
         }
 
         if (value.equals("HIVE_SERVER")) {
@@ -1300,11 +1247,7 @@ public class RepositoryToComponentProperty {
         if (value.equals(EParameterNameForComponent.PARA_NAME_MAPRED_JT.getName())
                 || value.equals(EParameterNameForComponent.PARA_NAME_RESOURCE_MANAGER.getName())) {
             String jobTrackerURL = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_URL);
-            if (isContextMode(connection, jobTrackerURL)) {
-                return jobTrackerURL;
-            } else {
-                return TalendQuoteUtils.addQuotes(jobTrackerURL);
-            }
+            return getAppropriateValue(connection, jobTrackerURL);
         }
 
         if (value.equals(EParameterNameForComponent.PARA_NAME_USE_YARN.getName())) {
@@ -1343,20 +1286,12 @@ public class RepositoryToComponentProperty {
 
         if (value.equals("MAPRED_JOB_TRACKER") || value.equals("MAPRED_RESOURCE_MANAGER")) {
             String mapredJobTracker = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_URL);
-            if (mapredJobTracker == null) {
-                return mapredJobTracker;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(mapredJobTracker));
-            }
+            return getAppropriateValue(connection, mapredJobTracker);
         }
 
         if (value.equals("NAMENODE_PRINCIPAL")) {
             String nameNodePrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_NAME_NODE_PRINCIPAL);
-            if (nameNodePrincipal == null) {
-                return nameNodePrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(nameNodePrincipal));
-            }
+            return getAppropriateValue(connection, nameNodePrincipal);
         }
 
         /**
@@ -1366,36 +1301,20 @@ public class RepositoryToComponentProperty {
          */
         if (value.equals("JOBTRACKER_PRINCIPAL") || value.equals("RESOURCEMANAGER_PRINCIPAL")) {
             String jtOrRmPrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_TRACKER_PRINCIPAL);
-            if (jtOrRmPrincipal == null) {
-                return jtOrRmPrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(jtOrRmPrincipal));
-            }
+            return getAppropriateValue(connection, jtOrRmPrincipal);
         }
 
         if (value.equals("JOBHISTORY_PRINCIPAL")) {
             String jobHistoryPrincipal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOB_HISTORY_PRINCIPAL);
-            if (jobHistoryPrincipal == null) {
-                return jobHistoryPrincipal;
-            } else {
-                return TalendQuoteUtils.addQuotesIfNotExist(StringUtils.trimToNull(jobHistoryPrincipal));
-            }
+            return getAppropriateValue(connection, jobHistoryPrincipal);
         }
 
         if (value.equals("ZOOKEEPER_QUORUM")) {
-            if (isContextMode(connection, connection.getServerName())) {
-                return connection.getServerName();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getServerName());
-            }
+            return getAppropriateValue(connection, connection.getServerName());
         }
 
         if (value.equals("ZOOKEEPER_CLIENT_PORT")) {
-            if (isContextMode(connection, connection.getPort())) {
-                return connection.getPort();
-            } else {
-                return TalendQuoteUtils.addQuotes(connection.getPort());
-            }
+            return getAppropriateValue(connection, connection.getPort());
         }
 
         if (value.equals("COLUMN_MAPPING")) { //$NON-NLS-1$
@@ -1403,24 +1322,28 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("HIVE_PRINCIPAL")) {
-            return TalendQuoteUtils
-                    .addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_HIVEPRINCIPLA));
+            String hivePrincipal = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_HIVEPRINCIPLA);
+            return getAppropriateValue(connection, hivePrincipal);
         }
 
         if (value.equals("METASTORE_JDBC_URL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_METASTOREURL));
+            String metadataUrl = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_METASTOREURL);
+            return getAppropriateValue(connection, metadataUrl);
         }
 
         if (value.equals("METASTORE_CLASSNAME")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_DRIVERCLASS));
+            String metastoreDC = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_DRIVERCLASS);
+            return getAppropriateValue(connection, metastoreDC);
         }
 
         if (value.equals("METASTORE_USERNAME")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_USERNAME));
+            String metastoreUser = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_USERNAME);
+            return getAppropriateValue(connection, metastoreUser);
         }
 
         if (value.equals("METASTORE_PASSWORD")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_PASSWORD));
+            String metastorePwd = connection.getParameters().get(ConnParameterKeys.HIVE_AUTHENTICATION_PASSWORD);
+            return getAppropriateValue(connection, metastorePwd);
         }
 
         if (value.equals("USE_KEYTAB")) {
@@ -1433,15 +1356,18 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("PRINCIPAL") || value.equals("KEYTAB_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB_PRINCIPAL));
+            String principal = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB_PRINCIPAL);
+            return getAppropriateValue(connection, principal);
         }
 
         if (value.equals("KEYTAB_PATH")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB));
+            String keytab = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_KEYTAB);
+            return getAppropriateValue(connection, keytab);
         }
 
         if (value.equals("IMPALA_PRINCIPAL")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA));
+            String principal = connection.getParameters().get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA);
+            return getAppropriateValue(connection, principal);
         }
 
         if (value.equals("IMPALA_VERSION")) {
@@ -1461,8 +1387,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("RESOURCEMANAGER_SCHEDULER_ADDRESS")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(
-                    ConnParameterKeys.CONN_PARA_KEY_RESOURCEMANAGER_SCHEDULER_ADDRESS));
+            String rmSchAdr = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_RESOURCEMANAGER_SCHEDULER_ADDRESS);
+            return getAppropriateValue(connection, rmSchAdr);
         }
 
         if (value.equals("SET_JOBHISTORY_ADDRESS")) {
@@ -1470,7 +1396,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("JOBHISTORY_ADDRESS")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOBHISTORY_ADDRESS));
+            String jhAddr = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_JOBHISTORY_ADDRESS);
+            return getAppropriateValue(connection, jhAddr);
         }
 
         if (value.equals("SET_STAGING_DIRECTORY")) {
@@ -1478,7 +1405,8 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("STAGING_DIRECTORY")) {
-            return TalendQuoteUtils.addQuotes(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_STAGING_DIRECTORY));
+            String stageDir = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_STAGING_DIRECTORY);
+            return getAppropriateValue(connection, stageDir);
         }
 
         if (value.equals("USE_DATANODE_HOSTNAME")) {
@@ -1490,6 +1418,14 @@ public class RepositoryToComponentProperty {
 
     }
 
+    private static String getAppropriateValue(Connection connection, String rawValue) {
+        if (isContextMode(connection, rawValue)) {
+            return rawValue;
+        } else {
+            return TalendQuoteUtils.addQuotesIfNotExist(rawValue);
+        }
+    }
+
     /**
      * DOC nrousseau Comment method "getDatabaseValue".
      * 
@@ -2694,4 +2630,13 @@ public class RepositoryToComponentProperty {
         }
         return value;
     }
+
+    public static boolean isGenericRepositoryValue(Connection connection, String paramName) {
+        for (IDragAndDropServiceHandler handler : DragAndDropManager.getHandlers()) {
+            if (handler.canHandle(connection)) {
+                return handler.isGenericRepositoryValue(connection, paramName);
+            }
+        }
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EParameterFieldType.java

@@ -98,6 +98,8 @@ public enum EParameterFieldType {
     NAME_SELECTION_AREA, // ycbai added for generic wizard
     NAME_SELECTION_REFERENCE, // ycbai added for generic wizard
     COMPONENT_REFERENCE, // ycbai added for generic wizard
+    HIDDEN_TEXT, // hcyi added for generic wizard/component properties
+    SCHEMA_REFERENCE, // hcyi added for component properties
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -64,7 +64,7 @@ import org.talend.repository.model.IProxyRepositoryService;
  */
 @SuppressWarnings("unchecked")
 public final class ProcessUtils {
-    
+
     private static boolean isHD;
 
     private static List<IProcess> fakeProcesses = new ArrayList<IProcess>();
@@ -751,6 +751,20 @@ public final class ProcessUtils {
         return false;
     }
 
+    public static boolean isSparkStreming(IProcess process) {
+        if (process instanceof IProcess2) {
+            return false;
+        }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerCoreService.class)) {
+            ITestContainerCoreService testContainerService = (ITestContainerCoreService) GlobalServiceRegister.getDefault()
+                    .getService(ITestContainerCoreService.class);
+            if (testContainerService != null) {
+                return testContainerService.isSparkStreaming(((IProcess2) process).getProperty().getItem());
+            }
+        }
+        return false;
+    }
+
     public static List<String> getTestData(IProcess process, String instanceName) {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService = (ITestContainerProviderService) GlobalServiceRegister
@@ -930,26 +944,26 @@ public final class ProcessUtils {
 
         return false;
     }
-    
+
     public static boolean isHDInsight() {
         return isHD;
     }
-    
+
     public static void setHDInsight(boolean isHD) {
         ProcessUtils.isHD = isHD;
     }
-    
+
     public static boolean isDistributionExist(ProcessItem processItem) {
         EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
         for (ElementParameterType pt : parameters) {
             if (pt.getName().equals("DISTRIBUTION")) { //$NON-NLS-1$
-                 String value = pt.getValue();
-                 if("MICROSOFT_HD_INSIGHT".equals(value)){ //$NON-NLS-1$
-                     return true;
-                 }
+                String value = pt.getValue();
+                if ("MICROSOFT_HD_INSIGHT".equals(value)) { //$NON-NLS-1$
+                    return true;
+                }
             }
         }
         return false;
     }
-    
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -87,15 +87,16 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static final String PROD_DQ = ProductUtils.PROD_DQ;
 
-    public static final String PROD_CAMEL = ProductUtils.PROD_CAMEL;
-
     public static final String PROD_MDM = ProductUtils.PROD_MDM;
 
+    public final static ERepositoryObjectType GENERAL_FILE = new ERepositoryObjectType("repository.generalfile", "",
+            "GENERAL_FILE", true, 1, new String[] { PROD_DI, PROD_DQ, PROD_MDM }, new String[] {}, false);
+
     public final static ERepositoryObjectType GIT_ROOT = new ERepositoryObjectType("repository.gitroot", "", "GIT_ROOT", true, 1,
-            new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
+            new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SVN_ROOT = new ERepositoryObjectType("repository.svnroot", "", "SVN_ROOT", true, 1,
-            new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
+            new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SVG_BUSINESS_PROCESS = new ERepositoryObjectType("repository.svgBusinessProcess",
             "businessProcessSVG", "SVG_BUSINESS_PROCESS", true, 3, new String[] { PROD_DI }, new String[] {});
@@ -107,7 +108,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
     // "documentations", "DOCUMENTATION", true, 11, new String[] { PROD_DI }, new String[] {});
 
     public final static ERepositoryObjectType METADATA = new ERepositoryObjectType("repository.metadata", "metadata", "METADATA",
-            true, 8, new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
+            true, 80, new String[] { PROD_DI, PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType METADATA_CON_TABLE = new ERepositoryObjectType("repository.metadataTable",
             "METADATA_CON_TABLE", 13, true, true, new String[] { PROD_DI }, new String[] {}, false);
@@ -149,7 +150,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             "MDM_ELEMENT_TYPE", 25, true, true, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType RECYCLE_BIN = new ERepositoryObjectType("repository.recyclebin", "", "RECYCLE_BIN",
-            true, 26, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false);
+            true, 900, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType METADATA_COLUMN = new ERepositoryObjectType("repository.column", "",
             "METADATA_COLUMN", true, 27, new String[] { PROD_DI }, new String[] {}, false);
@@ -184,22 +185,6 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             "REFERENCED_PROJECTS", 51, true, "repository.referencedProjects.alias", new String[] { PROD_DI, PROD_DQ },
             new String[] {}, false);
 
-    public final static ERepositoryObjectType GENERATED = new ERepositoryObjectType("repository.generated", //$NON-NLS-1$
-            "documentations/generated", "GENERATED", true, 52, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOBS = new ERepositoryObjectType(
-            "repository.jobs", "documentations/generated/jobs", //$NON-NLS-1$
-            "JOBS", true, 53, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOB_DOC = new ERepositoryObjectType("repository.jobdoc", //$NON-NLS-1$
-            "documentations/generated/jobs", "JOB_DOC", true, 54, new String[] { PROD_DI }, new String[] {});
-
-    public final static ERepositoryObjectType JOBLETS = new ERepositoryObjectType("repository.joblets", //$NON-NLS-1$
-            "documentations/generated/joblets", "JOBLETS", true, 58, new String[] { PROD_DI }, new String[] {}, false);
-
-    public final static ERepositoryObjectType JOBLET_DOC = new ERepositoryObjectType("repository.jobletdoc", //$NON-NLS-1$
-            "documentations/generated/joblets", "JOBLET_DOC", true, 59, new String[] { PROD_DI }, new String[] {});
-
     public final static ERepositoryObjectType COMPONENTS = new ERepositoryObjectType("repository.components", "components", //$NON-NLS-1$
             "COMPONENTS", true, 61, new String[] { PROD_DI }, new String[] {}, false);
 
@@ -361,10 +346,10 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             true, "repository.systemIndicators.phoneNumberStatistics.alias", new String[] { PROD_DQ }, new String[] {}, false);
 
     public final static ERepositoryObjectType SERVICESOPERATION = new ERepositoryObjectType("repository.servicesOperation", //$NON-NLS-1$
-            "SERVICESOPERATION", 101, true, true, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false); //$NON-NLS-1$
+            "SERVICESOPERATION", 101, true, true, new String[] { PROD_DI }, new String[] {}, false); //$NON-NLS-1$
 
     public final static ERepositoryObjectType SERVICESPORT = new ERepositoryObjectType("repository.servicesPort", "SERVICESPORT", //$NON-NLS-1$ //$NON-NLS-2$
-            102, true, true, new String[] { PROD_DI, PROD_CAMEL }, new String[] {}, false);
+            102, true, true, new String[] { PROD_DI }, new String[] {}, false);
 
     public final static ERepositoryObjectType SYSTEM_INDICATORS_FRAUDDETECTION = new ERepositoryObjectType(
             "repository.systemIndicators.fraudDetection", "TDQ_Libraries/Indicators/System Indicators/Fraud Detection", //$NON-NLS-1$ //$NON-NLS-2$
@@ -436,8 +421,22 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType SQLPATTERNS = ERepositoryObjectType.valueOf("SQLPATTERNS"); //$NON-NLS-1$
 
+    /**
+     * Docs
+     * 
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
     public final static ERepositoryObjectType DOCUMENTATION = ERepositoryObjectType.valueOf("DOCUMENTATION"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType GENERATED = ERepositoryObjectType.valueOf("GENERATED");//$NON-NLS-1$
+
+    public final static ERepositoryObjectType JOB_DOC = ERepositoryObjectType.valueOf("JOB_DOC");//$NON-NLS-1$
+
+    public final static ERepositoryObjectType JOBLET_DOC = ERepositoryObjectType.valueOf("JOBLET_DOC");//$NON-NLS-1$
+
+    /**
+     * Metadatas
+     */
     public final static ERepositoryObjectType METADATA_FILE_POSITIONAL = ERepositoryObjectType
             .valueOf("METADATA_FILE_POSITIONAL"); //$NON-NLS-1$
 
@@ -460,11 +459,6 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType METADATA_MDMCONNECTION = ERepositoryObjectType.valueOf("METADATA_MDMCONNECTION"); //$NON-NLS-1$
 
-    /**
-     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
-     */
-    public final static ERepositoryObjectType PROCESS_MR = ERepositoryObjectType.valueOf("PROCESS_MR"); //$NON-NLS-1$
-
     public final static ERepositoryObjectType METADATA_CONNECTIONS = ERepositoryObjectType.valueOf("METADATA_CONNECTIONS");
 
     public final static ERepositoryObjectType METADATA_FILE_DELIMITED = ERepositoryObjectType.valueOf("METADATA_FILE_DELIMITED");
@@ -482,6 +476,11 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType METADATA_EDIFACT = ERepositoryObjectType.valueOf("METADATA_EDIFACT");
 
+    /**
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
+    public final static ERepositoryObjectType PROCESS_MR = ERepositoryObjectType.valueOf("PROCESS_MR"); //$NON-NLS-1$
+
     /**
      * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
      */
@@ -515,6 +514,18 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      */
     public final static ERepositoryObjectType PROCESS_SPARKSTREAMING = ERepositoryObjectType.valueOf("PROCESS_SPARKSTREAMING");
 
+    public final static ERepositoryObjectType METADATA_SAP_BW_DATASOURCE = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_DATASOURCE"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_DATASTOREOBJECT = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_DATASTOREOBJECT"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_INFOCUBE = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_INFOCUBE"); //$NON-NLS-1$
+
+    public final static ERepositoryObjectType METADATA_SAP_BW_INFOOBJECT = ERepositoryObjectType
+            .valueOf("METADATA_SAP_BW_INFOOBJECT"); //$NON-NLS-1$
+
     private static Map<String, ERepositoryObjectType> typeCacheById = new HashMap<String, ERepositoryObjectType>();
 
     ERepositoryObjectType(String key, String folder, String type, boolean isStaticNode, int ordinal, String[] products,
@@ -610,8 +621,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             // limit with DI items only.
             List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
             for (ERepositoryObjectType currentType : values(ERepositoryObjectType.class)) {
-                if (ArrayUtils.contains(currentType.getProducts(), PROD_DI)
-                        || ArrayUtils.contains(currentType.getProducts(), PROD_CAMEL)) {
+                if (ArrayUtils.contains(currentType.getProducts(), PROD_DI)) {
                     toReturn.add(currentType);
                 }
             }
@@ -707,9 +717,9 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         return type;
     }
 
-    
     /**
      * Getter for typeCacheById.
+     * 
      * @return the typeCacheById
      */
     public static Map<String, ERepositoryObjectType> getTypeCacheById() {
@@ -730,12 +740,12 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static String getFolderName(ERepositoryObjectType type) {
 
-        if (type == GENERATED || type == JOBS || type == JOB_DOC) {
+        if (type == GENERATED || type == JOB_DOC) {
             if ((PluginChecker.isDocumentationPluginLoaded())) {
                 return type.getFolder();
             }
         }
-        if (type == JOBLETS || type == JOBLET_DOC) {
+        if (type == JOBLET_DOC) {
             if (PluginChecker.isJobLetPluginLoaded()) {
                 return type.getFolder();
             }
@@ -843,6 +853,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      * Please use instead the method : public static ERepositoryObjectType getType(Property property)<br>
      * 
      * This method might change to private in the future, since many calls will have low performance.
+     * 
      * @param item
      * @return
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/temp/ECodePart.java

@@ -34,9 +34,11 @@ public enum ECodePart {
     SPARKCODE("sparkcode"), //$NON-NLS-1$
     SPARKCONFIG("sparkconfig"), //$NON-NLS-1$
     SPARKJOBFOOTER("sparkjobfooter"), //$NON-NLS-1$
+    SPARKPOSTPROCESSING("sparkpostprocessing"), //$NON-NLS-1$
     SPARKSTREAMINGCODE("sparkstreamingcode"), //$NON-NLS-1$
     SPARKSTREAMINGCONFIG("sparkstreamingconfig"), //$NON-NLS-1$
-    SPARKSTREAMINGJOBFOOTER("sparkstreamingjobfooter"); //$NON-NLS-1$
+    SPARKSTREAMINGJOBFOOTER("sparkstreamingjobfooter"), //$NON-NLS-1$
+    SPARKSTREAMINGPOSTPROCESSING("sparkstreamingpostprocessing"); //$NON-NLS-1$
 
     private ECodePart(String name) {
         this.name = name;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/update/RepositoryUpdateManager.java

@@ -93,6 +93,7 @@ import org.talend.core.service.IMRProcessService;
 import org.talend.core.service.IMetadataManagmentService;
 import org.talend.core.service.IStormProcessService;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.SAPBWTableHelper;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.runprocess.ItemCacheManager;
 import org.talend.repository.model.IProxyRepositoryFactory;
@@ -1642,6 +1643,11 @@ public abstract class RepositoryUpdateManager {
 
     public static boolean updateMultiSchema(ConnectionItem connItem, List<IMetadataTable> oldMetadataTable,
             Map<String, String> oldTableMap) {
+        return updateMultiSchema(connItem, oldMetadataTable, oldTableMap, null);
+    }
+
+    public static boolean updateMultiSchema(ConnectionItem connItem, List<IMetadataTable> oldMetadataTable,
+            Map<String, String> oldTableMap, String bwTableType) {
         if (connItem == null) {
             return false;
         }
@@ -1653,8 +1659,8 @@ public abstract class RepositoryUpdateManager {
 
         if (!update) {
             if (oldMetadataTable != null) {
-                List<IMetadataTable> newMetadataTable = RepositoryUpdateManager.getConversionMetadataTables(connItem
-                        .getConnection());
+                List<IMetadataTable> newMetadataTable = RepositoryUpdateManager.getConversionMetadataTables(
+                        connItem.getConnection(), bwTableType);
                 update = !RepositoryUpdateManager.sameAsMetadatTable(newMetadataTable, oldMetadataTable, oldTableMap);
                 isAddColumn = isAddColumn(newMetadataTable, oldMetadataTable);
             }
@@ -1665,9 +1671,14 @@ public abstract class RepositoryUpdateManager {
 
             List<IMetadataTable> newMetadataTable = new ArrayList<IMetadataTable>();
             if (coreService != null
-                    && ((connection instanceof DatabaseConnection) || (connection instanceof GenericSchemaConnection))) {
-                Set<org.talend.core.model.metadata.builder.connection.MetadataTable> newTables = ConnectionHelper
-                        .getTables(connection);
+                    && ((connection instanceof DatabaseConnection) || (connection instanceof GenericSchemaConnection))
+                    || (connection instanceof SAPConnection)) {
+                Set<org.talend.core.model.metadata.builder.connection.MetadataTable> newTables = null;
+                if (bwTableType == null) {
+                    newTables = ConnectionHelper.getTables(connection);
+                } else {
+                    newTables = SAPBWTableHelper.getBWTables(connection, bwTableType);
+                }
                 if (newTables != null) {
                     for (org.talend.core.model.metadata.builder.connection.MetadataTable originalTable : newTables) {
                         IMetadataTable conversionTable = coreService.convert(originalTable);
@@ -1783,6 +1794,15 @@ public abstract class RepositoryUpdateManager {
                 String tableId = newTable.getId();
                 if (tableLabel.equals(oldtableLabel)) {
                     isDeleted = false;
+                    String newInnerIOType = newTable.getAdditionalProperties().get(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE);
+                    String oldInnerIOType = oldTable.getAdditionalProperties().get(SAPBWTableHelper.SAP_INFOOBJECT_INNER_TYPE);
+                    if (newInnerIOType != null) {
+                        if (newInnerIOType.equals(oldInnerIOType) && !tableId.equals(oldtableId)) {
+                            prefix = connItem.getProperty().getId() + UpdatesConstants.SEGMENT_LINE;
+                            deletedOrReselectTables.put(prefix + tableLabel, EUpdateResult.RELOAD);
+                        }
+                        continue;
+                    }
                     /* if table name is same but tableId is not same,means table has been deselect and reselect */
                     if (!tableId.equals(oldtableId)) {
                         prefix = connItem.getProperty().getId() + UpdatesConstants.SEGMENT_LINE;
@@ -2197,14 +2217,22 @@ public abstract class RepositoryUpdateManager {
         return repositoryUpdateManager.doWork();
     }
 
-    @SuppressWarnings("unchecked")
     public static List<IMetadataTable> getConversionMetadataTables(Connection conn) {
+        return getConversionMetadataTables(conn, null);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static List<IMetadataTable> getConversionMetadataTables(Connection conn, String bwTableType) {
         if (conn == null) {
             return Collections.emptyList();
         }
         List<IMetadataTable> tables = new ArrayList<IMetadataTable>();
-
-        Set tables2 = ConnectionHelper.getTables(conn);
+        Set tables2 = null;
+        if (bwTableType != null) {
+            tables2 = SAPBWTableHelper.getBWTables(conn, bwTableType);
+        } else {
+            tables2 = ConnectionHelper.getTables(conn);
+        }
         if (tables2 != null) {
             for (org.talend.core.model.metadata.builder.connection.MetadataTable originalTable : (Set<org.talend.core.model.metadata.builder.connection.MetadataTable>) tables2) {
                 if (GlobalServiceRegister.getDefault().isServiceRegistered(IMetadataManagmentService.class)) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/AbstractDragAndDropServiceHandler.java

@@ -56,4 +56,8 @@ public abstract class AbstractDragAndDropServiceHandler implements IDragAndDropS
         }
     }
 
+    @Override
+    public boolean isGenericRepositoryValue(Connection connection, String paramName) {
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/IDragAndDropServiceHandler.java

@@ -109,4 +109,13 @@ public interface IDragAndDropServiceHandler {
      */
     public boolean isValidForDataViewer(Connection connection, IMetadataTable metadataTable);
 
+    /**
+     * 
+     * DOC hcyi Comment method "isGenericRepositoryValue".
+     * 
+     * @param connection
+     * @param paramName
+     * @return
+     */
+    public boolean isGenericRepositoryValue(Connection connection, String paramName);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusConstants.java

@@ -20,6 +20,8 @@ public class NexusConstants {
 
     public static final String SERVICES_SEARCH = "service/local/lucene/search?";//$NON-NLS-1$ 
 
+    public static final String SERVICES_RESOLVE = "service/local/artifact/maven/resolve?";//$NON-NLS-1$ 
+
     public static final String CONTENT_REPOSITORIES = "/content/repositories/";//$NON-NLS-1$ 
 
     public static final String SLASH = "/";//$NON-NLS-1$ 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusServerUtils.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.nexus;
 
+import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.net.Authenticator;
 import java.net.HttpURLConnection;
@@ -25,15 +26,19 @@ import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSocketFactory;
 
+import org.apache.commons.lang.StringUtils;
 import org.dom4j.Document;
 import org.dom4j.Node;
 import org.dom4j.io.SAXReader;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.service.IRemoteService;
 import org.talend.utils.json.JSONException;
@@ -46,6 +51,11 @@ import org.talend.utils.ssl.SSLUtils;
  */
 public class NexusServerUtils {
 
+    /**
+     * 
+     */
+    public static final String ORG_TALEND_DESIGNER_CORE = "org.talend.designer.core"; //$NON-NLS-1$
+
     public static final String KEY_NEXUS_RUL = "nexusUrl";//$NON-NLS-1$ 
 
     public static final String KEY_NEXUS_USER = "username";//$NON-NLS-1$ 
@@ -131,6 +141,9 @@ public class NexusServerUtils {
      * @return
      */
     public static boolean checkConnectionStatus(String nexusUrl, String repositoryId, final String userName, final String password) {
+        if (StringUtils.isEmpty(nexusUrl)) {
+            return false;
+        }
         final Authenticator defaultAuthenticator = NetworkUtil.getDefaultAuthenticator();
         if (userName != null && !"".equals(userName)) {
             Authenticator.setDefault(new Authenticator() {
@@ -155,10 +168,6 @@ public class NexusServerUtils {
 
             URL url = new URL(urlToCheck);
             HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
-            // if (userName != null && !"".equals(userName)) {
-            // urlConnection.setRequestProperty(
-            //                        "Authorization", "Basic " + Base64.encodeBase64((userName + ":" + password).getBytes()));//$NON-NLS-1$ //$NON-NLS-2$
-            // }
             if (urlConnection instanceof HttpsURLConnection) {
                 String userDir = Platform.getInstallLocation().getURL().getPath();
                 final SSLSocketFactory socketFactory = SSLUtils.getSSLContext(userDir).getSocketFactory();
@@ -173,14 +182,17 @@ public class NexusServerUtils {
 
                 });
             }
-            urlConnection.setConnectTimeout(10000);
-            urlConnection.setReadTimeout(10000);
+            IEclipsePreferences node = InstanceScope.INSTANCE.getNode(ORG_TALEND_DESIGNER_CORE);
+            int timeout = node.getInt(ITalendCorePrefConstants.NEXUS_TIMEOUT, 10000);
+
+            urlConnection.setConnectTimeout(timeout);
+            urlConnection.setReadTimeout(timeout);
             status = urlConnection.getResponseCode();
             if (status == CONNECTION_OK) {
                 return true;
             }
         } catch (Exception e) {
-            // do nothing
+            ExceptionHandler.process(e);
         } finally {
             Authenticator.setDefault(defaultAuthenticator);
         }
@@ -302,6 +314,47 @@ public class NexusServerUtils {
         }
     }
 
+    public static String resolveSha1(String nexusUrl, final String userName, final String password, String repositoryId,
+            String groupId, String artifactId, String version) throws Exception {
+        HttpURLConnection urlConnection = null;
+        final Authenticator defaultAuthenticator = NetworkUtil.getDefaultAuthenticator();
+        if (userName != null && !"".equals(userName)) {
+            Authenticator.setDefault(new Authenticator() {
+
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(userName, password.toCharArray());
+                }
+
+            });
+        }
+        try {
+            String service = NexusConstants.SERVICES_RESOLVE + "a=" + artifactId + "&g=" + groupId + "&r=" + repositoryId + "&v="
+                    + version;
+            urlConnection = getHttpURLConnection(nexusUrl, service, userName, password);
+            SAXReader saxReader = new SAXReader();
+
+            InputStream inputStream = urlConnection.getInputStream();
+            Document document = saxReader.read(inputStream);
+
+            Node sha1Node = document.selectSingleNode("/artifact-resolution/data/sha1");
+            String sha1 = null;
+            if (sha1Node != null) {
+                sha1 = sha1Node.getText();
+            }
+            return sha1;
+
+        } catch (FileNotFoundException e) {
+            // jar not existing on remote nexus
+            return null;
+        } finally {
+            Authenticator.setDefault(defaultAuthenticator);
+            if (null != urlConnection) {
+                urlConnection.disconnect();
+            }
+        }
+    }
+
     private static String getSearchQuery(String repositoryId, String groupId, String artifactId, String version, int from,
             int count) {
         String query = "";//$NON-NLS-1$ 
@@ -352,7 +405,9 @@ public class NexusServerUtils {
 
             });
         }
-        urlConnection.setConnectTimeout(10000);
+        IEclipsePreferences node = InstanceScope.INSTANCE.getNode(ORG_TALEND_DESIGNER_CORE);
+        int timeout = node.getInt(ITalendCorePrefConstants.NEXUS_TIMEOUT, 10000);
+        urlConnection.setConnectTimeout(timeout);
         return urlConnection;
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/ITalendCorePrefConstants.java

@@ -181,4 +181,8 @@ public interface ITalendCorePrefConstants {
     public static final String PERFORMANCE_TAC_CONNECTION_TIMEOUT = "PERFORMANCE_TAC_CONNECTION_TIMEOUT"; //$NON-NLS-1$
 
     public static final String PERFORMANCE_JAVA_PROCESS_CODE_FORMATE_TIMEOUT = "PERFORMANCE_JAVA_PROCESS_CODE_FORMATE_TIMEOUT"; //$NON-NLS-1$
+
+    public static final String NEXUS_TIMEOUT = "NEXUS_TIMEOUT"; //$NON-NLS-1$
+
+    public static final String NEXUS_REFRESH_FREQUENCY = "NEXUS_REFRESH_FREQUENCY"; //$NON-NLS-1$
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/PreferenceManipulator.java

@@ -39,6 +39,8 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
 
     private static final String PREF_DELIMITER = "|"; //$NON-NLS-1$
 
+    private static final String LOCAL_MODE = "local";
+
     /** The preference store manipulated. */
     private IPreferenceStore store;
 
@@ -246,6 +248,18 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
         save();
     }
 
+    public int getLastLogonMode(String projectUrl, String projectName, String branch) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
+        int mode = store.getInt(hexKey);
+        return mode;
+    }
+
+    public void setLastLogonMode(String projectUrl, String projectName, String branch, int mode) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
+        store.setValue(hexKey, mode);
+        save();
+    }
+
     public String getLastUser() {
         return store.getString(LAST_USED_USER);
     }
@@ -284,4 +298,15 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
             }
         }
     }
+
+    public void setNewCreateBranchObjectId(String projectUrl, String projectName, String branch, String id) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
+        store.setValue(hexKey, id);
+        save();
+    }
+
+    public String getNewCreateBranchObjectId(String projectUrl, String projectName, String branch) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
+        return store.getString(hexKey);
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenUrlHelper.java

@@ -208,6 +208,9 @@ public class MavenUrlHelper {
         if (parseMvnUrl == null) {
             return null;
         }
+        if (!MavenConstants.DEFAULT_LIB_GROUP_ID.equals(parseMvnUrl.getGroupId())) {
+            return mavenUri; // snapshot url same as maven url
+        }
 
         return generateMvnUrl(parseMvnUrl.getGroupId(), parseMvnUrl.getArtifactId(), parseMvnUrl.getVersion()
                 + MavenConstants.SNAPSHOT, parseMvnUrl.getType(), parseMvnUrl.getClassifier());
@@ -219,6 +222,9 @@ public class MavenUrlHelper {
         if (parseMvnUrl == null) {
             return null;
         }
+        if (!MavenConstants.DEFAULT_LIB_GROUP_ID.equals(parseMvnUrl.getGroupId())) {
+            return snapshotMvnuri; // snapshot url same as maven url
+        }
         String snapshotVerstion = parseMvnUrl.getVersion();
         String verstion = snapshotVerstion.substring(0, snapshotVerstion.indexOf(MavenConstants.SNAPSHOT));
         return generateMvnUrl(parseMvnUrl.getGroupId(), parseMvnUrl.getArtifactId(), verstion, parseMvnUrl.getType(),

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/ComponentServiceWithValueEvaluator.java

@@ -23,10 +23,8 @@ import org.talend.components.api.service.ComponentService;
 import org.talend.components.api.wizard.ComponentWizard;
 import org.talend.components.api.wizard.ComponentWizardDefinition;
 import org.talend.components.api.wizard.WizardImageType;
-import org.talend.daikon.NamedThing;
 import org.talend.daikon.properties.PropertyValueEvaluator;
 import org.talend.daikon.properties.service.Repository;
-import org.talend.daikon.schema.Schema;
 
 /**
  * created by ycbai on 2016年2月4日 Detailled comment
@@ -44,8 +42,9 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
     }
 
     @Override
-    public String storeProperties(ComponentProperties properties, String name, String repositoryLocation, Schema schema) {
-        return service.storeProperties(properties, name, repositoryLocation, schema);
+    public String storeProperties(ComponentProperties properties, String name, String repositoryLocation,
+            String schemaPropertyName) {
+        return service.storeProperties(properties, name, repositoryLocation, schemaPropertyName);
     }
 
     @Override
@@ -90,11 +89,6 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
         return service.beforePropertyPresent(propName, properties);
     }
 
-    @Override
-    public ComponentProperties commitFormValues(ComponentProperties properties, String formName) {
-        return service.commitFormValues(properties, formName);
-    }
-
     @Override
     public Set<String> getAllComponentNames() {
         return service.getAllComponentNames();
@@ -136,7 +130,7 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
     }
 
     @Override
-    public List<ComponentDefinition> getPossibleComponents(ComponentProperties properties) throws Throwable {
+    public List<ComponentDefinition> getPossibleComponents(ComponentProperties... properties) throws Throwable {
         return service.getPossibleComponents(properties);
     }
 
@@ -166,16 +160,23 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
         return service.validateProperty(propName, properties);
     }
 
-    private void setupChildrenPropertiesContextValueEvaluator(ComponentProperties compProperties,
-            PropertyValueEvaluator valueEvaluator) {
-        List<NamedThing> properties = compProperties.getProperties();
-        for (NamedThing property : properties) {
-            if (property instanceof ComponentProperties) {
-                ComponentProperties childComponentProperties = (ComponentProperties) property;
-                childComponentProperties.setValueEvaluator(valueEvaluator);
-                setupChildrenPropertiesContextValueEvaluator(childComponentProperties, valueEvaluator);
-            }
-        }
+    @Deprecated
+    public ComponentProperties commitFormValues(ComponentProperties properties, String formName) {
+        // to delete
+        return null;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see
+     * org.talend.daikon.properties.service.PropertiesService#cancelFormValues(org.talend.daikon.properties.Properties,
+     * java.lang.String)
+     */
+    @Override
+    public ComponentProperties cancelFormValues(ComponentProperties arg0, String arg1) {
+        // to implement
+        return null;
     }
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/IStormProcessService.java

@@ -17,9 +17,13 @@ import java.util.List;
 import org.eclipse.ui.IEditorPart;
 import org.talend.commons.ui.runtime.image.IImage;
 import org.talend.core.IService;
+import org.talend.core.model.components.IComponentsHandler;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.repository.IRepositoryEditorInput;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.repository.model.IRepositoryNode;
 import org.talend.repository.model.nodes.IProjectRepositoryNode;
@@ -51,4 +55,13 @@ public interface IStormProcessService extends IService {
 
     public IImage getStormWizardIcon(String framework);
 
+    public IComponentsHandler getSparkStreamingComponentsHandler();
+
+    public boolean isSparkStreaming(Item item);
+
+    public IRepositoryEditorInput createTestCaseStormProcessEditorInput(ProcessItem processItem, boolean load,
+            String originalJobID, List<INode> testNodes, Boolean lastVersion, Boolean readonly);
+
+    public IProcess2 createBigdataProcess(Item item);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/ITestContainerCoreService.java

@@ -22,4 +22,6 @@ import org.talend.core.model.properties.Item;
 public interface ITestContainerCoreService extends IService {
 
     public boolean isSpark(Item item);
+
+    public boolean isSparkStreaming(Item item);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/branding/IBrandingConfiguration.java

@@ -31,8 +31,6 @@ public interface IBrandingConfiguration extends IActionBarHelper {
 
     public static final String PERSPECTIVE_MDM_ID = ProductUtils.PERSPECTIVE_MDM_ID;
 
-    public static final String PERSPECTIVE_CAMEL_ID = ProductUtils.PERSPECTIVE_CAMEL_ID;
-
     /**
      * 
      * init perspective.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/utils/ProductUtils.java

@@ -26,8 +26,6 @@ public final class ProductUtils {
      */
     public static final String PERSPECTIVE_DI_ID = "org.talend.rcp.perspective"; //$NON-NLS-1$
 
-    public static final String PERSPECTIVE_CAMEL_ID = "org.talend.camel.perspective"; //$NON-NLS-1$
-
     public static final String PERSPECTIVE_DQ_ID = "org.talend.dataprofiler.DataProfilingPerspective"; //$NON-NLS-1$
 
     public static final String PERSPECTIVE_MDM_ID = "org.talend.mdm.perspective"; //$NON-NLS-1$
@@ -54,7 +52,6 @@ public final class ProductUtils {
     static {
         productPerspectiveMap.put(PROD_DI, PERSPECTIVE_DI_ID);
         productPerspectiveMap.put(PROD_DQ, PERSPECTIVE_DQ_ID);
-        productPerspectiveMap.put(PROD_CAMEL, PERSPECTIVE_CAMEL_ID);
         productPerspectiveMap.put(PROD_MDM, PERSPECTIVE_MDM_ID);
         productPerspectiveMap.put(PROD_MAPPING, PERSPECTIVE_MAPPING_ID);
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -161,6 +161,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         syncRoutine(routineItem, copyToTemp, false);
     }
 
+    @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -43,6 +43,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException;
 
+    void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceToUpdate) throws SystemException;
+
     IFile getFile(Item item) throws SystemException;
 
     IFile getRoutinesFile(Item routineItem) throws SystemException;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -118,6 +118,8 @@ public interface IRunProcessService extends IService {
     public void setDelegateService(IRunProcessService delegateService);
 
     public void updateLibraries(Set<ModuleNeeded> jobModuleList, IProcess process);
+    
+    public void updateLibraries(Set<ModuleNeeded> jobModuleList, IProcess process, Set<ModuleNeeded> alreadyRetrievedModules);
 
     public void refreshView();
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryConstants.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.repository.model;
 
+import org.eclipse.core.runtime.Path;
 import org.talend.core.model.repository.ERepositoryObjectType;
 
 /**
@@ -143,14 +144,16 @@ public class RepositoryConstants {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.GENERATED.toString());
+        return ERepositoryObjectType.GENERATED != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.GENERATED.getFolder()).lastSegment());
     }
 
     public static boolean isJobsFolder(String label) {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.JOBS.toString());
+        return ERepositoryObjectType.JOB_DOC != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment());
     }
 
     /**
@@ -163,6 +166,7 @@ public class RepositoryConstants {
         if (label == null) {
             return false;
         }
-        return label.equalsIgnoreCase(ERepositoryObjectType.JOBLETS.toString());
+        return ERepositoryObjectType.JOBLET_DOC != null
+                && label.equalsIgnoreCase(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment());
     }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryNodeUtilities.java

@@ -75,8 +75,8 @@ public class RepositoryNodeUtilities {
         if (node.isBin()) {
             return new Path(""); //$NON-NLS-1$
         }
-        if ((node.getType() == ENodeType.STABLE_SYSTEM_FOLDER && node.getContentType() != ERepositoryObjectType.JOBS && node
-                .getContentType() != ERepositoryObjectType.JOBLETS) || node.getType() == ENodeType.SYSTEM_FOLDER) {
+        if ((node.getType() == ENodeType.STABLE_SYSTEM_FOLDER && node.getContentType() != ERepositoryObjectType.JOB_DOC && node
+                .getContentType() != ERepositoryObjectType.JOBLET_DOC) || node.getType() == ENodeType.SYSTEM_FOLDER) {
             return new Path(""); //$NON-NLS-1$
         }
         if (node.getType() == ENodeType.SIMPLE_FOLDER) {
@@ -87,11 +87,11 @@ public class RepositoryNodeUtilities {
         String label = node.getLabel();
         // checks if node is under Documentations/Generatated/Jobs
         if (node.getType() == ENodeType.STABLE_SYSTEM_FOLDER
-                && (node.getContentType() == ERepositoryObjectType.JOBS || node.getContentType() == ERepositoryObjectType.JOBLETS)) {
+                && (node.getContentType() == ERepositoryObjectType.JOB_DOC || node.getContentType() == ERepositoryObjectType.JOBLET_DOC)) {
 
             Object nodeLabel = node.getProperties(EProperties.LABEL);
 
-            if (nodeLabel == ERepositoryObjectType.JOBS) {
+            if (nodeLabel == ERepositoryObjectType.JOB_DOC) {
                 return new Path(""); //$NON-NLS-1$
             } else {
                 return getPath(node.getParent()).append(label);
@@ -463,7 +463,7 @@ public class RepositoryNodeUtilities {
         if (rootContextType == ERepositoryObjectType.REFERENCED_PROJECTS || rootNode.getType() == ENodeType.REFERENCED_PROJECT) {
             expandParentNode(view, rootNode);
         }
-        if (rootContextType != null) {
+        if (rootContextType != null && curType != null) {
 
             ERepositoryObjectType tmpType = null;
             if (curType == ERepositoryObjectType.METADATA_CON_TABLE || curType == ERepositoryObjectType.METADATA_CON_VIEW
@@ -490,8 +490,8 @@ public class RepositoryNodeUtilities {
                 tmpType = ERepositoryObjectType.METADATA;
             } else if (curType == ERepositoryObjectType.ROUTINES || curType == ERepositoryObjectType.SNIPPETS) {
                 tmpType = ERepositoryObjectType.ROUTINES;
-            } else if (curType == ERepositoryObjectType.DOCUMENTATION || curType == ERepositoryObjectType.JOB_DOC
-                    || curType == ERepositoryObjectType.JOBLET_DOC) {
+            } else if (curType.equals(ERepositoryObjectType.DOCUMENTATION) || curType.equals(ERepositoryObjectType.JOB_DOC)
+                    || curType.equals(ERepositoryObjectType.JOBLET_DOC)) {
                 tmpType = ERepositoryObjectType.DOCUMENTATION;
             }
 

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/view/Contexts.java

@@ -42,7 +42,7 @@ public class Contexts {
             refreshView(cxtView);
         }
     }
-    
+
     public static void switchToCurContextsView(final IWorkbenchPart part) {
         final AbstractContextView cxtView = getViewWithPerspectiveIDs();
         Display.getDefault().asyncExec(new Runnable() {
@@ -60,7 +60,6 @@ public class Contexts {
     private static AbstractContextView getViewWithPerspectiveIDs() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         return getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
     }
 
@@ -91,7 +90,6 @@ public class Contexts {
     public static void refreshContextsView() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         AbstractContextView cxtView = getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
         if (cxtView != null) {
             refreshView(cxtView);
@@ -189,7 +187,6 @@ public class Contexts {
     public static void clearAll() {
         Set<String> perspIDs = new HashSet<String>();
         perspIDs.add(PERSPECTIVE_DI_ID);
-        perspIDs.add(IBrandingConfiguration.PERSPECTIVE_CAMEL_ID);
         AbstractContextView cxtView = getView(perspIDs, AbstractContextView.CTX_ID_DESIGNER);
         refreshView(cxtView);
         if (cxtView != null) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/images/CoreImageProvider.java

@@ -29,6 +29,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.ui.component.ComponentsFactoryProvider;
 import org.talend.designer.core.IDesignerCoreService;
+import org.talend.utils.string.MD5;
 
 /**
  * amaumont class global comment. Detailled comment <br/>
@@ -80,17 +81,19 @@ public class CoreImageProvider {
         return getIcon(itemType);
     }
 
-    private static Map<ImageDescriptor, Image> componentCachedImages = new HashMap<ImageDescriptor, Image>();
+    private static Map<String, Image> componentCachedImages = new HashMap<String, Image>();
 
-    public static Image getComponentImageFromDesc(ImageDescriptor imageDescriptor) {
-        Image image = null;
-        image = componentCachedImages.get(imageDescriptor);
-        if (image == null || image.isDisposed()) {
-            image = imageDescriptor.createImage();
-            componentCachedImages.put(imageDescriptor, image);
-        }
-        return image;
-    }
+	public static Image getComponentImageFromDesc(
+			ImageDescriptor imageDescriptor) {
+		String md5Desc = MD5.getMD5(imageDescriptor.getImageData().data);
+		Image image = componentCachedImages.get(md5Desc);
+
+		if (image == null || image.isDisposed()) {
+			image = imageDescriptor.createImage();
+			componentCachedImages.put(md5Desc, image);
+		}
+		return image;
+	}
 
     public static Image getComponentIcon(IComponent component, ICON_SIZE iconSize) {
         if (component != null && iconSize != null) {
@@ -130,21 +133,24 @@ public class CoreImageProvider {
         if (name != null && !name.equals("")) { //$NON-NLS-1$
             for (IComponent component : ComponentsFactoryProvider.getInstance().getComponents()) {
                 if (name.equals(component.getName())) {
-                    Image image = componentCachedImages.get(component.getIcon16());
+                	String md5Desc16 = MD5.getMD5(component.getIcon16().getImageData().data);
+                    Image image = componentCachedImages.get(md5Desc16);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon16());
-                    image = componentCachedImages.get(component.getIcon24());
+                    componentCachedImages.remove(md5Desc16);
+                    String md5Desc24 = MD5.getMD5(component.getIcon24().getImageData().data);
+                    image = componentCachedImages.get(md5Desc24);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon24());
-                    image = componentCachedImages.get(component.getIcon32());
+                    componentCachedImages.remove(md5Desc24);
+                    String md5Desc32 = MD5.getMD5(component.getIcon32().getImageData().data);
+                    image = componentCachedImages.get(md5Desc32);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon32());
+                    componentCachedImages.remove(md5Desc32);
                 }
             }
         }

main/plugins/org.talend.core/src/main/java/org/talend/core/model/repository/documentation/DocumentationHelper.java

@@ -18,6 +18,7 @@ import java.util.List;
 
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Path;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.CorePlugin;
@@ -124,10 +125,12 @@ public class DocumentationHelper {
                     && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.JOBLET.toString());
             boolean isNotGenerated = ERepositoryObjectType.GENERATED != null
                     && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.GENERATED.toString());
-            boolean isNotJobs = ERepositoryObjectType.JOBS != null
-                    && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.JOBS.toString());
-            boolean isNotJoblets = ERepositoryObjectType.JOBLETS != null
-                    && !node.getProperties(EProperties.LABEL).toString().equals(ERepositoryObjectType.JOBLETS.toString());
+            boolean isNotJobs = ERepositoryObjectType.JOB_DOC != null
+                    && !node.getProperties(EProperties.LABEL).toString()
+                            .equals(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment());
+            boolean isNotJoblets = ERepositoryObjectType.JOBLET_DOC != null
+                    && !node.getProperties(EProperties.LABEL).toString()
+                            .equals(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment());
 
             if (isNotProcess && isNotJoblet && isNotGenerated && isNotJobs && isNotJoblets) {
                 folderName = node.getProperties(EProperties.LABEL).toString();
@@ -344,7 +347,7 @@ public class DocumentationHelper {
 
             // Goto the Node "Jobs";
             for (IRepositoryNode subNode : node.getChildren()) {
-                if (subNode.getContentType() != ERepositoryObjectType.JOBS) {
+                if (subNode.getContentType() != ERepositoryObjectType.JOB_DOC) {
                     continue;
                 }
 

main/plugins/org.talend.core/src/main/java/org/talend/core/model/repository/documentation/generation/JobHTMLScriptsManager.java

@@ -20,6 +20,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Path;
 import org.talend.core.model.genhtml.HTMLDocUtils;
 import org.talend.core.model.genhtml.IHTMLDocConstants;
 import org.talend.core.model.properties.Item;
@@ -144,12 +145,13 @@ public class JobHTMLScriptsManager implements IDocumentationManager {
 
         String subFolder = null;
 
-        if (targetPath.endsWith(ERepositoryObjectType.GENERATED.toString().toLowerCase())) {
+        if (ERepositoryObjectType.GENERATED != null
+                && targetPath.endsWith(new Path(ERepositoryObjectType.GENERATED.getFolder()).lastSegment().toLowerCase())) {
 
-            if (item instanceof JobDocumentationItem) {
-                subFolder = ERepositoryObjectType.JOBS.toString().toLowerCase();
-            } else if (item instanceof JobletDocumentationItem) {
-                subFolder = ERepositoryObjectType.JOBLETS.toString().toLowerCase();
+            if (item instanceof JobDocumentationItem && ERepositoryObjectType.JOB_DOC != null) {
+                subFolder = new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment().toLowerCase();
+            } else if (item instanceof JobletDocumentationItem && ERepositoryObjectType.JOBLET_DOC != null) {
+                subFolder = new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment().toLowerCase();
             } else {
                 subFolder = ""; //$NON-NLS-1$
             }
@@ -158,8 +160,8 @@ public class JobHTMLScriptsManager implements IDocumentationManager {
         }
 
         // Used for generating/updating all jobs' documentaiton only.
-        if (targetPath.endsWith(ERepositoryObjectType.JOBS.toString().toLowerCase())
-                || targetPath.endsWith(ERepositoryObjectType.JOBLETS.toString().toLowerCase())) {
+        if (targetPath.endsWith(new Path(ERepositoryObjectType.JOB_DOC.getFolder()).lastSegment().toLowerCase())
+                || targetPath.endsWith(new Path(ERepositoryObjectType.JOBLET_DOC.getFolder()).lastSegment().toLowerCase())) {
             targetPath = targetPath + IPath.SEPARATOR + jobPath + IPath.SEPARATOR + jobName;
         }
         String version = ""; //$NON-NLS-1$

main/plugins/org.talend.core/src/main/java/org/talend/core/services/ICreateXtextProcessService.java

@@ -15,6 +15,7 @@ package org.talend.core.services;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.IService;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 import org.talend.designer.joblet.model.JobletProcess;
 
@@ -30,6 +31,8 @@ public interface ICreateXtextProcessService extends IService {
 
     // create the processType with a .job file created by metalanguage.
     public void addProcess(String path);
+    
+    public  ERepositoryObjectType createProcessItem();
 
     public String convertJobtoScript(ProcessType processType);
 

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -127,6 +127,8 @@ public class ProcessorUtilities {
     private static final int GENERATED_WITH_TRACES = 2;
 
     private static final String COMMA = ";"; //$NON-NLS-1$
+    
+    private static final Set<ModuleNeeded> retrievedJarsForCurrentBuild = new HashSet<ModuleNeeded>();
 
     public static void addOpenEditor(IEditorPart editor) {
         openedEditors.add(editor);
@@ -368,6 +370,7 @@ public class ProcessorUtilities {
 
             // this cache only keep the last main job's generation, so clear it since we regenerate a new job.
             LastGenerationInfo.getInstance().getLastGeneratedjobs().clear();
+            retrievedJarsForCurrentBuild.clear();
 
             // if it's the father, reset the processMap to ensure to have a good
             // code generation
@@ -458,7 +461,7 @@ public class ProcessorUtilities {
             LastGenerationInfo.getInstance().setModulesNeededPerJob(jobInfo.getJobId(), jobInfo.getJobVersion(), neededLibraries);
 
             // must install the needed libraries before generate codes with poms.
-            CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess);
+            CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess, retrievedJarsForCurrentBuild);
         }
         resetRunJobComponentParameterForContextApply(jobInfo, currentProcess, selectedContextName);
 
@@ -592,6 +595,7 @@ public class ProcessorUtilities {
                 processor.syntaxCheck();
             }
             needContextInCurrentGeneration = true;
+            retrievedJarsForCurrentBuild.clear();
             codeModified = false;
         }
     }
@@ -702,7 +706,7 @@ public class ProcessorUtilities {
 
                 // this cache only keep the last main job's generation, so clear it since we regenerate a new job.
                 LastGenerationInfo.getInstance().getLastGeneratedjobs().clear();
-
+                retrievedJarsForCurrentBuild.clear();
                 // if it's the father, reset the processMap to ensure to have a good
                 // code generation
                 ItemCacheManager.clearCache();
@@ -801,7 +805,7 @@ public class ProcessorUtilities {
                         neededLibraries);
 
                 // must install the needed libraries before generate codes with poms.
-                CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess);
+                CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess, retrievedJarsForCurrentBuild);
             }
             resetRunJobComponentParameterForContextApply(jobInfo, currentProcess, selectedContextName);
 

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/talendlib/TalendLibsServerManager.java

@@ -251,4 +251,10 @@ public class TalendLibsServerManager {
             String groupIdToSearch, String artifactId, String versionToSearch) throws Exception {
         return NexusServerUtils.search(nexusUrl, userName, password, repositoryId, groupIdToSearch, artifactId, versionToSearch);
     }
+
+    public String resolveSha1(String nexusUrl, String userName, String password, String repositoryId,
+            String groupId, String artifactId, String version) throws Exception {
+        return NexusServerUtils.resolveSha1(nexusUrl, userName, password, repositoryId, groupId, artifactId, version);
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenCodesTemplatePom.java

@@ -20,6 +20,7 @@ import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
 import org.eclipse.core.resources.IFile;
 import org.talend.core.model.general.ModuleNeeded;
+import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
 import org.talend.designer.maven.utils.PomUtil;
 
@@ -63,7 +64,10 @@ public abstract class AbstractMavenCodesTemplatePom extends AbstractMavenGeneral
             }
 
             for (ModuleNeeded module : runningModules) {
-                Dependency dependency = PomUtil.createModuleDependency(module.getModuleName());
+                Dependency dependency = null;
+                if (module.getMavenUriSnapshot() != null && !module.getMavenUriSnapshot().isEmpty() && module.getStatus() != ELibraryInstallStatus.NOT_INSTALLED) {
+                    dependency = PomUtil.createModuleDependency(module.getMavenUriSnapshot());
+                }
                 if (dependency != null) {
                     existedDependencies.add(dependency);
                 }

main/plugins/org.talend.libraries.apache/.classpath

@@ -1,12 +1,12 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
-	<classpathentry exported="true" kind="lib" path="lib/ant.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/apache-mime4j-0.6.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/castor-1.0.3.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/xerces-2.6.2.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/xmlbeans-2.3.0.jar"/>
-	<classpathentry kind="output" path="bin"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry exported="true" kind="lib" path="lib/ant.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/apache-mime4j-0.6.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/castor-1.0.3.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/xerces-2.6.2.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/xmlbeans-2.3.0.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

main/plugins/org.talend.libraries.apache/META-INF/MANIFEST.MF

@@ -12,7 +12,9 @@ Bundle-ClassPath: .,
  lib/xmlbeans-2.3.0.jar
 Bundle-Vendor: .Talend SA.
 Require-Bundle: org.apache.log4j;resolution:=optional,
- org.talend.libraries.apache.common;visibility:=reexport
+ org.talend.libraries.apache.common;visibility:=reexport,
+ jackson-core-asl;bundle-version="1.9.13",
+ jackson-mapper-asl;bundle-version="1.9.13"
 Export-Package: 
  org.apache.james.mime4j,
  org.apache.james.mime4j.codec,

main/plugins/org.talend.libraries.jackson/META-INF/MANIFEST.MF

@@ -38,34 +38,7 @@ Export-Package: com.fasterxml.jackson.databind;version="2.5.3",
  net.sf.json.regexp,
  net.sf.json.test,
  net.sf.json.util,
- net.sf.json.xml,
- org.codehaus.jackson,
- org.codehaus.jackson.annotate,
- org.codehaus.jackson.format,
- org.codehaus.jackson.impl,
- org.codehaus.jackson.io,
- org.codehaus.jackson.map,
- org.codehaus.jackson.map.annotate,
- org.codehaus.jackson.map.deser,
- org.codehaus.jackson.map.deser.impl,
- org.codehaus.jackson.map.deser.std,
- org.codehaus.jackson.map.exc,
- org.codehaus.jackson.map.ext,
- org.codehaus.jackson.map.impl,
- org.codehaus.jackson.map.introspect,
- org.codehaus.jackson.map.jsontype,
- org.codehaus.jackson.map.jsontype.impl,
- org.codehaus.jackson.map.module,
- org.codehaus.jackson.map.ser,
- org.codehaus.jackson.map.ser.impl,
- org.codehaus.jackson.map.ser.std,
- org.codehaus.jackson.map.type,
- org.codehaus.jackson.map.util,
- org.codehaus.jackson.node,
- org.codehaus.jackson.schema,
- org.codehaus.jackson.sym,
- org.codehaus.jackson.type,
- org.codehaus.jackson.util
+ net.sf.json.xml
 Require-Bundle: org.apache.commons.lang,
  org.apache.commons.logging,
  org.apache.commons.collections,

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/wizards/AcceptModuleLicensesWizardPage.java

@@ -292,6 +292,10 @@ public class AcceptModuleLicensesWizardPage extends WizardPage {
                     }
                     Map<String, License> typeToLicense = new HashMap<String, License>();
                     for (ModuleToInstall moduleToInstall : mti) {
+                        // no need accept license if it is from custom nexus
+                        if (moduleToInstall.isFromCustomNexus()) {
+                            continue;
+                        }
                         String licenseType = moduleToInstall.getLicenseType();
                         if (licenseType == null) {
                             licenseType = UNKNOWN_LICENSE;

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/DownloadModuleRunnable.java

@@ -31,6 +31,8 @@ import org.ops4j.pax.url.mvn.Handler;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
+import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
+import org.talend.core.model.general.ModuleStatusProvider;
 import org.talend.core.model.general.ModuleToInstall;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.librariesmanager.ui.LibManagerUiPlugin;
@@ -78,7 +80,7 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
         SubMonitor subMonitor = SubMonitor
                 .convert(
                         monitor,
-                        Messages.getString("ExternalModulesInstallDialog.downloading2") + " (" + toDownload.size() + ")", toDownload.size() + 1); //$NON-NLS-1$
+                        Messages.getString("ExternalModulesInstallDialog.downloading2") + " (" + toDownload.size() + ")", toDownload.size()); //$NON-NLS-1$
 
         Map<String, String> customUriToAdd = new HashMap<String, String>();
         // TUP-3135 : stop to try to download at the first timeout.
@@ -89,20 +91,27 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
                 boolean accepted;
                 try {
                     // check license
-                    boolean isLicenseAccepted = LibManagerUiPlugin.getDefault().getPreferenceStore()
-                            .getBoolean(module.getLicenseType());
+                    boolean isLicenseAccepted = module.isFromCustomNexus()
+                            || LibManagerUiPlugin.getDefault().getPreferenceStore().getBoolean(module.getLicenseType());
                     accepted = isLicenseAccepted;
                     if (!accepted) {
                         subMonitor.worked(1);
                         continue;
                     }
-                    NexusDownloadHelperWithProgress downloader = new NexusDownloadHelperWithProgress();
+                    NexusDownloadHelperWithProgress downloader = new NexusDownloadHelperWithProgress(module);
                     if (!module.getMavenUris().isEmpty()) {
                         for (String mvnUri : module.getMavenUris()) {
                             downloader.download(new URL(null, mvnUri, new Handler()), null, subMonitor.newChild(1));
+                            // update module status
+                            final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
+                            statusMap.put(MavenUrlHelper.generateSnapshotMavenUri(mvnUri), ELibraryInstallStatus.INSTALLED);
                         }
                     } else {
                         downloader.download(new URL(null, module.getMavenUri(), new Handler()), null, subMonitor.newChild(1));
+                        // update module status
+                        final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
+                        statusMap.put(MavenUrlHelper.generateSnapshotMavenUri(module.getMavenUri()),
+                                ELibraryInstallStatus.INSTALLED);
                     }
 
                     // deploy to index as snapshot
@@ -129,14 +138,16 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
             ILibraryManagerService libraryManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
                     .getService(ILibraryManagerService.class);
             libraryManagerService.forceListUpdate();
-            LibManagerUiPlugin.getDefault().getLibrariesService().resetModulesNeeded();
         }
-        subMonitor.worked(1);
     }
 
     protected boolean hasLicensesToAccept() {
         if (toDownload != null && toDownload.size() > 0) {
             for (ModuleToInstall module : toDownload) {
+                // no need accept license if it is from custom nexus
+                if (module.isFromCustomNexus()) {
+                    continue;
+                }
                 String licenseType = module.getLicenseType();
                 if (licenseType != null) {
                     boolean isLicenseAccepted = LibManagerUiPlugin.getDefault().getPreferenceStore()

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/RemoteModulesHelper.java

@@ -26,6 +26,8 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -104,129 +106,202 @@ public class RemoteModulesHelper {
 
         @Override
         public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
-            final Set<String> mavenUriSet = contextMap.keySet();
-            Map<String, MavenArtifact> requiredArtifacts = new HashMap<String, MavenArtifact>();
+            final Set<String> mavenUrisTofetch = new HashSet<String>(contextMap.keySet());
+            monitor.beginTask(Messages.getString("RemoteModulesHelper.fetch.module.info"), mavenUrisTofetch.size() * 10 + 10);//$NON-NLS-1$
+            // fetch modules from local nexus first
+            final NexusServerBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
+            if (customNexusServer != null) {
+                if (addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, localCache)) {
+                    if (collectModulesWithJarName) {
+                        collectModulesWithJarName(toInstall);
+                    }
+                    monitor.done();
+                    return;
+                }
+                // search from local nexus
+                searchFromLocalNexus(monitor);
+                // check again after search
+                if (addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, localCache)) {
+                    if (collectModulesWithJarName) {
+                        collectModulesWithJarName(toInstall);
+                    }
+                    monitor.done();
+                    return;
+                }
+            }
+
+            if (addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, remoteCache)) {
+                if (collectModulesWithJarName) {
+                    collectModulesWithJarName(toInstall);
+                }
+                monitor.done();
+                return;
+            }
+
             Set<String> unavailableModules = new HashSet<String>();
-            monitor.beginTask(Messages.getString("RemoteModulesHelper.fetch.module.info"), mavenUriSet.size() * 10 + 10);//$NON-NLS-1$
             // if the network is not valid, all jars are not available.
             boolean networkValid = NetworkUtil.isNetworkValid();
             if (!networkValid) {
-                unavailableModules.addAll(mavenUriSet);
                 if (!alreadyWarnedAboutConnectionIssue) {
                     log.warn("failed to connect to internet");
                     alreadyWarnedAboutConnectionIssue = true;
                 }// else already warned so do nothing
-            }
-            if (networkValid) {
-                try {
-                    int index = 0;
-                    int limit = 100;
-                    final Iterator<String> iterator = mavenUriSet.iterator();
-                    while (index < mavenUriSet.size()) {
-                        // get block of 100 jars
-                        String jarsToCheck = "";
-                        while (index < limit && index < mavenUriSet.size()) {
-                            index++;
-                            String uriToCheck = iterator.next();
-                            final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(uriToCheck);
-                            if (parseMvnUrl != null) {
-                                requiredArtifacts.put(uriToCheck, parseMvnUrl);
-                                jarsToCheck += parseMvnUrl.getArtifactId();
-                                if (index < limit && index < mavenUriSet.size()) {
-                                    jarsToCheck += ",";
-                                }
-                            }
-                        }
-                        limit += 100;
-                        TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
-                        NexusServerBean nexusServer = manager.getLibrariesNexusServer();
-
-                        List<MavenArtifact> searchResults = manager.search(nexusServer.getServer(), nexusServer.getUserName(),
-                                nexusServer.getPassword(), nexusServer.getRepositoryId(), MavenConstants.DEFAULT_LIB_GROUP_ID,
-                                jarsToCheck, null);
-                        monitor.worked(10);
-                        for (MavenArtifact artifact : searchResults) {
-                            String artifactId = artifact.getArtifactId();
-                            String packageName = artifact.getType();
-                            if (packageName == null) {
-                                packageName = MavenConstants.TYPE_JAR;
-                            }
-                            String version = artifact.getVersion();
-                            String description = artifact.getDescription();
-                            String license = artifact.getLicense();
-                            String license_url = artifact.getLicenseUrl();
-                            String packaging = artifact.getType();
-                            String url = null;
-                            if (artifact.getUrl() != null && !"".equals(artifact.getUrl())) {
-                                url = artifact.getUrl();
-                            }
-                            ModuleToInstall m = new ModuleToInstall();
-                            m.setName(artifactId + "." + packageName);
-                            String mvnUri = MavenUrlHelper.generateMvnUrl(artifact.getGroupId(), artifactId, version, packaging,
-                                    artifact.getClassifier());
-                            m.setMavenUri(mvnUri);
-                            m.setLicenseType(license);
-                            m.setLicenseUrl(license_url);
-                            m.setDescription(description);
-                            m.setUrl_description(url);
-                            m.setUrl_download(url);
-                            if (artifact.getType() == null
-                                    || "".equals(artifact.getType()) || MavenConstants.PACKAGING_POM.equals(artifact.getType())) { //$NON-NLS-1$
-                                m.setDistribution(MavenConstants.DOWNLOAD_MANUAL);
-                            } else {
-                                m.setDistribution(artifact.getType());
-                            }
-
-                            cache.put(mvnUri, m);
-
-                            monitor.worked(10);
-                        }
-
-                    }
-
-                } catch (Exception e1) {
-                    ExceptionHandler.process(e1);
-                }
-                for (String uri : requiredArtifacts.keySet()) {
-                    String mvnUri = uri;
-                    // incase the package is not set int mavenuri
-                    final MavenArtifact artifact = requiredArtifacts.get(mvnUri);
-                    if (artifact.getType() == null) {
-                        artifact.setType(MavenConstants.TYPE_JAR);
-                    }
-                    mvnUri = MavenUrlHelper.generateMvnUrl(artifact.getGroupId(), artifact.getArtifactId(),
-                            artifact.getVersion(), artifact.getType(), artifact.getClassifier());
-                    ModuleToInstall moduleToInstall = cache.get(mvnUri);
-                    if (moduleToInstall != null) {
-                        setContext(moduleToInstall, uri, contextMap);
-                        toInstall.add(moduleToInstall);
-                    } else {
-                        String artifactId = artifact.getArtifactId();
-                        String type = artifact.getType();
-                        if (type == null) {
-                            type = MavenConstants.TYPE_JAR;
-                        }
-                        String name = artifactId + "." + type;
-                        unavailableModules.add(uri);
-                        ExceptionHandler.log("The download URL for " + name + " is not available");//$NON-NLS-1$//$NON-NLS-2$
-                        if (CommonsPlugin.isDebugMode()) {
-                            appendToLogFile(name + "\n");
-                        }
-                    }
-                    monitor.worked(10);
-                }
+            } else {
+                searchFromRemoteNexus(mavenUrisTofetch, monitor);
+                addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, remoteCache);
             }
 
+            unavailableModules.addAll(mavenUrisTofetch);
             addUnavailableModulesToToBeInstalledModules(unavailableModules, toInstall, contextMap);
 
             if (collectModulesWithJarName) {
                 collectModulesWithJarName(toInstall);
             }
+            Collections.sort(toInstall, new Comparator<ModuleToInstall>() {
 
+                @Override
+                public int compare(ModuleToInstall o1, ModuleToInstall o2) {
+                    return o1.getName().compareTo(o2.getName());
+                }
+            });
             monitor.done();
         }
     }
 
+    private void searchFromLocalNexus(IProgressMonitor monitor) {
+        try {
+            TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+            NexusServerBean customServer = manager.getCustomNexusServer();
+
+            List<MavenArtifact> searchResults = manager.search(customServer.getServer(), customServer.getUserName(),
+                    customServer.getPassword(), customServer.getRepositoryId(), MavenConstants.DEFAULT_LIB_GROUP_ID, null, null);
+            monitor.worked(10);
+            addModulesToCache(searchResults, localCache);
+
+        } catch (Exception e1) {
+            ExceptionHandler.process(e1);
+        }
+    }
+
+    private void searchFromRemoteNexus(Set<String> mavenUristoSearch, IProgressMonitor monitor) {
+        try {
+            int index = 0;
+            int limit = 100;
+            final Iterator<String> iterator = mavenUristoSearch.iterator();
+            while (index < mavenUristoSearch.size()) {
+                // get block of 100 jars
+                String jarsToCheck = "";
+                while (index < limit && index < mavenUristoSearch.size()) {
+                    index++;
+                    String uriToCheck = iterator.next();
+                    final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(uriToCheck);
+                    if (parseMvnUrl != null) {
+                        jarsToCheck += parseMvnUrl.getArtifactId();
+                        if (index < limit && index < mavenUristoSearch.size()) {
+                            jarsToCheck += ",";
+                        }
+                    }
+                }
+                limit += 100;
+                TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+                NexusServerBean nexusServer = manager.getLibrariesNexusServer();
+
+                List<MavenArtifact> searchResults = manager.search(nexusServer.getServer(), nexusServer.getUserName(),
+                        nexusServer.getPassword(), nexusServer.getRepositoryId(), MavenConstants.DEFAULT_LIB_GROUP_ID,
+                        jarsToCheck, null);
+                monitor.worked(10);
+                addModulesToCache(searchResults, remoteCache);
+            }
+
+        } catch (Exception e1) {
+            ExceptionHandler.process(e1);
+        }
+    }
+
+    private void addModulesToCache(List<MavenArtifact> searchResults, Map<String, ModuleToInstall> theCache) {
+        for (MavenArtifact artifact : searchResults) {
+            String artifactId = artifact.getArtifactId();
+            String packageName = artifact.getType();
+            if (packageName == null) {
+                packageName = MavenConstants.TYPE_JAR;
+            }
+            String version = artifact.getVersion();
+            if (version.endsWith(MavenConstants.SNAPSHOT)) {
+                version = version.substring(0, version.indexOf(MavenConstants.SNAPSHOT));
+            }
+            String description = artifact.getDescription();
+            String license = artifact.getLicense();
+            String license_url = artifact.getLicenseUrl();
+            String packaging = artifact.getType();
+            String url = null;
+            if (artifact.getUrl() != null && !"".equals(artifact.getUrl())) {
+                url = artifact.getUrl();
+            }
+            ModuleToInstall m = new ModuleToInstall();
+            m.setName(artifactId + "." + packageName);
+            String mvnUri = MavenUrlHelper.generateMvnUrl(artifact.getGroupId(), artifactId, version, packaging,
+                    artifact.getClassifier());
+            m.setMavenUri(mvnUri);
+            m.setLicenseType(license);
+            m.setLicenseUrl(license_url);
+            m.setDescription(description);
+            m.setUrl_description(url);
+            m.setUrl_download(url);
+            if (artifact.getType() == null
+                    || "".equals(artifact.getType()) || MavenConstants.PACKAGING_POM.equals(artifact.getType())) { //$NON-NLS-1$
+                m.setDistribution(MavenConstants.DOWNLOAD_MANUAL);
+            } else {
+                m.setDistribution(artifact.getType());
+            }
+            if (theCache == localCache) {
+                m.setFromCustomNexus(true);
+            }
+            theCache.put(mvnUri, m);
+        }
+    }
+
+    /**
+     * 
+     * DOC wchen Comment method "addCachedModulesToToBeInstallModules".
+     * 
+     * @param toInstall
+     * @param mavenUrisTofetch
+     * @param contextMap
+     * @param theCache
+     * @return if find all modules form the cache ,return true;
+     */
+    private boolean addCachedModulesToToBeInstallModules(List<ModuleToInstall> toInstall, Set<String> mavenUrisTofetch,
+            Map<String, List<ModuleNeeded>> contextMap, Map<String, ModuleToInstall> theCache) {
+        if (theCache.isEmpty()) {
+            return false;
+        }
+        final Iterator<String> iterator = mavenUrisTofetch.iterator();
+        while (iterator.hasNext()) {
+            String mvnUri = iterator.next();
+            ModuleToInstall moduleToInstall = null;
+            moduleToInstall = theCache.get(mvnUri);
+            if (moduleToInstall != null) {
+                List<ModuleNeeded> moduleContext = contextMap.get(mvnUri);
+                setContext(moduleToInstall, mvnUri, contextMap);
+                if (moduleContext != null && moduleContext.size() > 0) {
+                    for (ModuleNeeded needed : moduleContext) {
+                        if (moduleToInstall.getName().equals(needed.getModuleName())) {
+                            moduleToInstall.setRequired(needed.isRequired());
+                        }
+                    }
+                }
+
+                toInstall.add(moduleToInstall);
+                iterator.remove();
+            }
+        }
+
+        if (mavenUrisTofetch.isEmpty()) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * DOC sgandon Comment method "addUnavailableModulesToToBeInstalledModules".
      * 
@@ -254,14 +329,17 @@ public class RemoteModulesHelper {
     private ModuleToInstall createUnavailableModuleToInstall(String mvnUri, Map<String, List<ModuleNeeded>> contextMap) {
         ModuleToInstall m = new ModuleToInstall();
         m.setDistribution(MavenConstants.DOWNLOAD_MANUAL);
+        m.setMavenUri(mvnUri);
         setContext(m, mvnUri, contextMap);
+        String name = null;
         if (contextMap != null) {
             final List<ModuleNeeded> neededModules = contextMap.get(mvnUri);
-            m.setName(neededModules.get(0).getModuleName());
+            name = neededModules.get(0).getModuleName();
+            m.setName(name);
             m.setDescription(getFirstDescription(neededModules));
         } else {
             final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mvnUri);
-            String name = parseMvnUrl.getArtifactId();
+            name = parseMvnUrl.getArtifactId();
             String type = parseMvnUrl.getType();
             if (type == null) {
                 type = MavenConstants.TYPE_JAR;
@@ -269,6 +347,10 @@ public class RemoteModulesHelper {
             m.setName(name + "." + type);
 
         }
+        ExceptionHandler.log("The download URL for " + name + " is not available");//$NON-NLS-1$//$NON-NLS-2$
+        if (CommonsPlugin.isDebugMode()) {
+            appendToLogFile(name + "\n");
+        }
 
         return m;
     }
@@ -353,7 +435,9 @@ public class RemoteModulesHelper {
     /**
      * key : mvnuri(without SANPSHOT in version) value : ModuleToInstall
      */
-    private Map<String, ModuleToInstall> cache = new HashMap<String, ModuleToInstall>();
+    private Map<String, ModuleToInstall> remoteCache = new HashMap<String, ModuleToInstall>();
+
+    private Map<String, ModuleToInstall> localCache = new HashMap<String, ModuleToInstall>();
 
     private RemoteModulesHelper() {
     }
@@ -462,6 +546,7 @@ public class RemoteModulesHelper {
             List<ModuleToInstall> toInstall, boolean collectModulesWithJarName) {
         Map<String, List<ModuleNeeded>> contextMap = new HashMap<String, List<ModuleNeeded>>();
 
+        // collect mvnuri and modules incase many modules have the same mvnuri
         final Iterator<ModuleNeeded> iterator = neededModules.iterator();
         while (iterator.hasNext()) {
             ModuleNeeded module = iterator.next();
@@ -484,6 +569,7 @@ public class RemoteModulesHelper {
                     for (String snapshotUri : configuredSnapshotMvnUris) {
                         String uri = MavenUrlHelper.generateUriFromSnapshot(snapshotUri);
                         if (uri != null) {
+                            uri = addTypeForMavenUri(uri, module.getModuleName());
                             ModuleNeeded newModule = new ModuleNeeded(null, module.getModuleName(), null, true);
                             newModule.setMavenUri(uri);
                             if (!contextMap.keySet().contains(uri)) {
@@ -498,6 +584,7 @@ public class RemoteModulesHelper {
                 }
             }
             if (mvnUri != null) {
+                mvnUri = addTypeForMavenUri(mvnUri, module.getModuleName());
                 if (!contextMap.keySet().contains(mvnUri)) {
                     List<ModuleNeeded> modules = new ArrayList<ModuleNeeded>();
                     modules.add(module);
@@ -507,37 +594,24 @@ public class RemoteModulesHelper {
                 }
             }
         }
-        Set<String> contextKeys = new HashSet<String>(contextMap.keySet());
-        for (String mvnUri : contextKeys) {
-            ModuleToInstall moduleToInstall = null;
-            moduleToInstall = cache.get(mvnUri);
-            if (moduleToInstall != null) {
-                List<ModuleNeeded> moduleContext = contextMap.get(mvnUri);
-                setContext(moduleToInstall, mvnUri, contextMap);
-                if (moduleContext != null && moduleContext.size() > 0) {
-                    for (ModuleNeeded needed : moduleContext) {
-                        if (moduleToInstall.getName().equals(needed.getModuleName())) {
-                            moduleToInstall.setRequired(needed.isRequired());
-                        }
-                    }
-                }
-                toInstall.add(moduleToInstall);
-                contextMap.remove(mvnUri);
-            }
-        }
-
-        if (contextMap.isEmpty()) {
-            if (collectModulesWithJarName) {
-                collectModulesWithJarName(toInstall);
-            }
-            return null;
-        }
-
         // fetch the jars which are not in cache.
         return createRemoteModuleFetchRunnable(contextMap, toInstall, collectModulesWithJarName);
 
     }
 
+    private String addTypeForMavenUri(String uri, String moduleName) {
+        // make sure that mvn uri have the package
+        final String[] split = uri.split("/");
+        if (split.length < 4) {
+            String type = MavenConstants.TYPE_JAR;
+            if (moduleName.lastIndexOf(".") != -1) {
+                type = moduleName.substring(moduleName.lastIndexOf(".") + 1, moduleName.length());
+            }
+            uri = uri + "/" + type;
+        }
+        return uri;
+    }
+
     public RemoteModulesFetchRunnable getNotInstalledModulesRunnable(List<ModuleNeeded> neededModules,
             List<ModuleToInstall> toInstall) {
         return getNotInstalledModulesRunnable(neededModules, toInstall, false);

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/nexus/NexusDownloadHelperWithProgress.java

@@ -12,8 +12,17 @@
 // ============================================================================
 package org.talend.librariesmanager.utils.nexus;
 
+import java.io.File;
+import java.net.URL;
+
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.ops4j.pax.url.mvn.MavenResolver;
 import org.talend.core.download.DownloadHelperWithProgress;
 import org.talend.core.download.IDownloadHelper;
+import org.talend.core.model.general.ModuleToInstall;
+import org.talend.core.nexus.NexusServerBean;
+import org.talend.core.runtime.maven.MavenUrlHelper;
+import org.talend.designer.maven.talendlib.TalendLibsServerManager;
 
 /**
  * created by wchen on Apr 24, 2015 Detailled comment
@@ -21,6 +30,35 @@ import org.talend.core.download.IDownloadHelper;
  */
 public class NexusDownloadHelperWithProgress extends DownloadHelperWithProgress {
 
+    private ModuleToInstall toInstall;
+
+    public NexusDownloadHelperWithProgress(ModuleToInstall toInstall) {
+        this.toInstall = toInstall;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.talend.core.download.DownloadHelperWithProgress#download(java.net.URL, java.io.File,
+     * org.eclipse.core.runtime.IProgressMonitor)
+     */
+    @Override
+    public void download(URL componentUrl, File destination, IProgressMonitor progressMonitor) throws Exception {
+        File resolved = null;
+        if (toInstall.isFromCustomNexus()) {
+            final NexusServerBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
+            if (customNexusServer != null) {
+                progressMonitor.subTask(toInstall.getName() + " from " + customNexusServer.getServer());
+                final MavenResolver mavenResolver = TalendLibsServerManager.getInstance().getMavenResolver();
+                resolved = mavenResolver.resolve(MavenUrlHelper.generateSnapshotMavenUri(componentUrl.toExternalForm()));
+            }
+        }
+        if (resolved != null && resolved.exists()) {
+            return;
+        }
+        super.download(componentUrl, destination, progressMonitor);
+    }
+
     /*
      * (non-Javadoc)
      * 

main/plugins/org.talend.librariesmanager/resources/java/routines/system/ComponentRuntimeContainerBase.java

@@ -1,43 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package routines.system;
-
-import java.util.Date;
-import java.util.Map;
-
-import org.talend.components.api.runtime.ComponentDynamicHolder;
-import org.talend.components.api.runtime.ComponentRuntimeContainer;
-
-public class ComponentRuntimeContainerBase implements ComponentRuntimeContainer {
-
-	@Override
-	public ComponentDynamicHolder createDynamicHolder() {
-		throw new RuntimeException("This must be subclassed");
-	}
-
-	@Override
-	public String formatDate(Date date, String pattern) {
-		return FormatterUtils.format(date, pattern);
-	}
-
-	@Override
-	public Map<String, Object> getGlobalMap() {
-		throw new RuntimeException("This must be subclassed");
-	}
-
-	@Override
-	public String getCurrentComponentName() {
-		throw new RuntimeException("This must be subclassed");
-	}
-
-}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/maven/ArtifactsDeployer.java

@@ -77,6 +77,13 @@ public class ArtifactsDeployer {
             deployToLocalMaven(jarSourceAndMavenUri.get(mavenUri), mavenUri);
         }
     }
+    
+    public void deployToLocalMaven(Map<String, String> jarSourceAndMavenUri, boolean updateRemoteJar) throws Exception {
+        for (String mavenUri : jarSourceAndMavenUri.keySet()) {
+            deployToLocalMaven(jarSourceAndMavenUri.get(mavenUri), mavenUri, updateRemoteJar);
+        }
+    }
+
 
     public void deployToLocalMaven(String path, String mavenUri, boolean toRemoteNexus) throws Exception {
         MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mavenUri);
@@ -173,6 +180,7 @@ public class ArtifactsDeployer {
         }
     }
 
+
     // private void install(String path, MavenArtifact artifact) {
     // StringBuffer command = new StringBuffer();
     // // mvn -Dfile=E:\studio_code\.metadata\aaabbbb\lib\java\ojdbc6.jar -DgroupId=org.talend.libraries

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -12,21 +12,29 @@
 // ============================================================================
 package org.talend.librariesmanager.model.service;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Collection;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import javax.xml.bind.DatatypeConverter;
+
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -34,6 +42,9 @@ import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.ConfigurationScope;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.emf.common.util.EMap;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.FrameworkUtil;
@@ -58,6 +69,8 @@ import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.general.ModuleStatusProvider;
 import org.talend.core.nexus.NexusServerBean;
+import org.talend.core.nexus.NexusServerUtils;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
@@ -103,13 +116,14 @@ public class LocalLibraryManager implements ILibraryManagerService {
 
     boolean listToUpdate;
 
-    private ArtifactsDeployer deployer = new ArtifactsDeployer();
+    private ArtifactsDeployer deployer;
 
     /**
      * DOC nrousseau LocalLibraryManager constructor comment.
      */
     public LocalLibraryManager() {
         super();
+        deployer = new ArtifactsDeployer();
     }
 
     @Override
@@ -139,7 +153,7 @@ public class LocalLibraryManager implements ILibraryManagerService {
         if (file == null || !file.exists()) {
             return;
         }
-        deployFile(file, null, monitorWrap);
+        deployFile(file, null, true, monitorWrap);
         // deploy to configuration/lib/java if tac still use the svn lib
         try {
             if (isSvnLibSetup()) {
@@ -167,7 +181,7 @@ public class LocalLibraryManager implements ILibraryManagerService {
      * @param mavenUri snaopshot mvn uri
      * @param monitorWrap
      */
-    private void deployFile(File file, String snapshotMavenUri, IProgressMonitor... monitorWrap) {
+    private void deployFile(File file, String snapshotMavenUri, boolean updateRemoteJar, IProgressMonitor... monitorWrap) {
         try {
             listToUpdate = true;
             if (file.isDirectory()) {
@@ -216,7 +230,7 @@ public class LocalLibraryManager implements ILibraryManagerService {
                     sourceAndMavenUri.put(snapshotMavenUri, file.getAbsolutePath());
                 }
 
-                deployer.deployToLocalMaven(sourceAndMavenUri);
+                deployer.deployToLocalMaven(sourceAndMavenUri, updateRemoteJar);
             }
 
         } catch (IOException e) {
@@ -243,7 +257,7 @@ public class LocalLibraryManager implements ILibraryManagerService {
                 return;
             }
             // deploy to maven
-            deployFile(file, null, monitorWrap);
+            deployFile(file, null, false, monitorWrap);
             // deploy to configuration/lib/java if tac still use the svn lib
             try {
                 if (isSvnLibSetup()) {
@@ -321,11 +335,10 @@ public class LocalLibraryManager implements ILibraryManagerService {
             if (jarFile == null) {
                 jarFile = getJarFile(jarNeeded);
             }
-            // retreive form custom nexus server automatically
+            // retrieve form custom nexus server automatically
             TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
             final NexusServerBean customNexusServer = manager.getCustomNexusServer();
             if (customNexusServer != null) {
-                // TODO????? Resolver all versions if the mavenUri is null
                 Set<String> toResolve = new HashSet<String>();
                 if (snapshotMvnUri != null) {
                     toResolve.add(snapshotMvnUri);
@@ -343,26 +356,11 @@ public class LocalLibraryManager implements ILibraryManagerService {
                     }
                 }
                 for (String uri : toResolve) {
-                    MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
-                    final List<MavenArtifact> searchResults = manager.search(customNexusServer.getServer(),
-                            customNexusServer.getUserName(), customNexusServer.getPassword(),
-                            customNexusServer.getRepositoryId(), artifact.getGroupId(), artifact.getArtifactId(),
-                            artifact.getVersion());
-                    if (!searchResults.isEmpty()) {
-                        File resolvedFile = null;
-                        for (MavenArtifact result : searchResults) {
-                            if (jarNeeded.equals(result.getArtifactId() + "." + result.getType())) {
-                                resolvedFile = manager.getMavenResolver().resolve(uri);
-                                break;
-                            }
-                        }
-                        if (resolvedFile != null) {
-                            // reset module status
-                            final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
-                            statusMap.put(uri, ELibraryInstallStatus.INSTALLED);
-                            jarFile = resolvedFile;
-                            // update installed path
-                            mavenJarInstalled.put(uri, jarFile.getAbsolutePath());
+                    if (isResolveAllowed(uri)) {
+                        File resolvedJar = resolveJar(manager, customNexusServer, uri);
+                        if (resolvedJar != null) {
+                            jarFile = resolvedJar;
+                            break;
                         }
                     }
                 }
@@ -420,6 +418,143 @@ public class LocalLibraryManager implements ILibraryManagerService {
 
     }
 
+    /**
+     * DOC nrousseau Comment method "resolveJar".
+     * @param jarNeeded
+     * @param jarFile
+     * @param manager
+     * @param customNexusServer
+     * @param uri
+     * @return
+     * @throws Exception
+     * @throws IOException
+     */
+    public File resolveJar(TalendLibsServerManager manager,
+            final NexusServerBean customNexusServer, String uri) throws Exception, IOException {
+        MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
+        String remoteSha1 = manager.resolveSha1(customNexusServer.getServer(), customNexusServer.getUserName(),
+                customNexusServer.getPassword(), customNexusServer.getRepositoryId(), artifact.getGroupId(),
+                artifact.getArtifactId(), artifact.getVersion());
+        File resolvedFile = null;
+        if (remoteSha1 != null) {
+            String localFilePath = getJarPathFromMaven(uri);
+            if (localFilePath != null) {
+                File localFile = new File(localFilePath); 
+                FileInputStream fis = new FileInputStream(localFile);
+                String localSha1 = DigestUtils.shaHex(fis);
+                fis.close();
+                if (!StringUtils.equalsIgnoreCase(remoteSha1, localSha1)) {
+                    org.talend.utils.io.FilesUtils.deleteFolder(localFile.getParentFile(), true);
+                    resolvedFile = manager.getMavenResolver().resolve(uri);
+                }
+            } else {
+                resolvedFile = manager.getMavenResolver().resolve(uri);
+            }
+        }
+        if (resolvedFile != null) {
+            // reset module status
+            final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
+            statusMap.put(uri, ELibraryInstallStatus.INSTALLED);
+            // update installed path
+            mavenJarInstalled.put(uri, resolvedFile.getAbsolutePath());
+        }
+        updateLastResolveDate(uri);
+        return resolvedFile;
+    }
+
+    private Map<String, Date> lastResolveDate;
+
+    private String LAST_UPDATE_KEY = "lastUpdate"; //$NON-NLS-1$
+
+    public long daysBetween(final Calendar startDate, final Calendar endDate) {
+        // assert: startDate must be before endDate
+        int MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
+        long endInstant = endDate.getTimeInMillis();
+        int presumedDays = (int) ((endInstant - startDate.getTimeInMillis()) / MILLIS_IN_DAY);
+        Calendar cursor = (Calendar) startDate.clone();
+        cursor.add(Calendar.DAY_OF_YEAR, presumedDays);
+        long instant = cursor.getTimeInMillis();
+        if (instant == endInstant)
+            return presumedDays;
+
+        final int step = instant < endInstant ? 1 : -1;
+        do {
+            cursor.add(Calendar.DAY_OF_MONTH, step);
+            presumedDays += step;
+        } while (cursor.getTimeInMillis() <= endInstant);
+        return presumedDays -1;
+    }
+
+    /**
+     * DOC nrousseau Comment method "isResolveAllowed".
+     * 
+     * @param uri
+     * @return
+     */
+    public boolean isResolveAllowed(String uri) {
+        IEclipsePreferences node = InstanceScope.INSTANCE.getNode(NexusServerUtils.ORG_TALEND_DESIGNER_CORE);
+        int refreshTime = node.getInt(ITalendCorePrefConstants.NEXUS_REFRESH_FREQUENCY, 0);
+        if (refreshTime == 0) {
+            return true;
+        }
+        if (refreshTime == -1) {
+            return false;
+        }
+
+        if (lastResolveDate == null) {
+            lastResolveDate = new HashMap<String, Date>();
+            IEclipsePreferences prefSetting = ConfigurationScope.INSTANCE.getNode("org.talend.librariesmanager");
+            String lastUpdate = prefSetting.get(LAST_UPDATE_KEY, null);
+            if (lastUpdate != null) {
+                byte[] lastUpdateStream = DatatypeConverter.parseHexBinary(lastUpdate);
+                ByteArrayInputStream bais = new ByteArrayInputStream(lastUpdateStream);
+                ObjectInputStream ois;
+                try {
+                    ois = new ObjectInputStream(bais);
+                    lastResolveDate = (HashMap) ois.readObject();
+                    ois.close();
+                    bais.close();
+                } catch (Exception e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        }
+        Date lastDate = lastResolveDate.get(uri);
+        if (lastDate == null) {
+            return true;
+        }
+        Calendar before = Calendar.getInstance();
+        before.setTime(lastDate);
+        Calendar after = Calendar.getInstance();
+        after.setTime(new Date());
+        long days = daysBetween(before, after);
+        if (days >= refreshTime) {
+            return true;
+        }
+        return false;
+    }
+
+    public void updateLastResolveDate(String uri) {
+        if (lastResolveDate == null) {
+            lastResolveDate = new HashMap<String, Date>();
+        }
+        lastResolveDate.put(uri, new Date());
+        IEclipsePreferences prefSetting = ConfigurationScope.INSTANCE.getNode("org.talend.librariesmanager");
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos;
+        try {
+            oos = new ObjectOutputStream(baos);
+            oos.writeObject(lastResolveDate);
+            oos.close();
+            String lastUpdate = DatatypeConverter.printHexBinary(baos.toByteArray());
+            prefSetting.put(LAST_UPDATE_KEY, lastUpdate);
+            prefSetting.flush();
+            baos.close();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public boolean retrieve(Collection<String> jarsNeeded, String pathToStore, boolean showDialog,
             IProgressMonitor... monitorWrap) {
@@ -738,7 +873,7 @@ public class LocalLibraryManager implements ILibraryManagerService {
                     ExceptionHandler.log("missing jar:" + module.getModuleName());
                 }
                 if (fileToDeploy != null) {
-                    deployFile(fileToDeploy, mavenUri, monitorWrap);
+                    deployFile(fileToDeploy, mavenUri, false, monitorWrap);
                 }
             }
         }
@@ -826,6 +961,10 @@ public class LocalLibraryManager implements ILibraryManagerService {
 
     private boolean checkJarInstalledInMaven(String mvnUri) {
         if (mavenJarInstalled.containsKey(mvnUri)) {
+            if (!new File(mavenJarInstalled.get(mvnUri)).exists()) {
+                mavenJarInstalled.remove(mvnUri);
+                return false;
+            }
             return mavenJarInstalled.get(mvnUri) != null;
         }
         String artifactPath = PomUtil.getAbsArtifactPath(MavenUrlHelper.parseMvnUrl(mvnUri));

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/DqRepositoryViewService.java

@@ -88,7 +88,8 @@ public final class DqRepositoryViewService {
     private static final String REPLACEMENT_CHARS = "_"; //$NON-NLS-1$
 
     public static final String[] TABLE_TYPES = new String[] { ETableTypes.TABLETYPE_TABLE.getName(),
-            ETableTypes.EXTERNAL_TABLE.getName(), ETableTypes.MANAGED_TABLE.getName(), ETableTypes.INDEX_TABLE.getName() };
+            ETableTypes.EXTERNAL_TABLE.getName(), ETableTypes.MANAGED_TABLE.getName(), ETableTypes.INDEX_TABLE.getName(),
+            ETableTypes.SYSTEM_TABLE.getName() };
 
     public static final String[] VIEW_TYPES = new String[] { ETableTypes.TABLETYPE_VIEW.getName(),
             ETableTypes.VIRTUAL_VIEW.getName() };

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataFromDataBase.java

@@ -92,7 +92,8 @@ public class ExtractMetaDataFromDataBase {
         EXTERNAL_TABLE("EXTERNAL_TABLE"), //$NON-NLS-1$ // for hive
         MANAGED_TABLE("MANAGED_TABLE"), //$NON-NLS-1$ // for hive
         INDEX_TABLE("INDEX_TABLE"), //$NON-NLS-1$ // for hive
-        VIRTUAL_VIEW("VIRTUAL_VIEW"); //$NON-NLS-1$ // for hive
+        VIRTUAL_VIEW("VIRTUAL_VIEW"), //$NON-NLS-1$ // for hive
+        SYSTEM_TABLE("SYSTEM TABLE"); //$NON-NLS-1$ //added for amazon redshift TDQ-11558 by msjian 
 
         private final String name;
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/dburl/SupportDBUrlStore.java

@@ -95,6 +95,7 @@ public final class SupportDBUrlStore {
         supportDBUrlMap.put(SupportDBUrlType.VERTICA.getDBKey(), SupportDBUrlType.VERTICA);
         supportDBUrlMap.put(SupportDBUrlType.VERTICA2.getDBKey(), SupportDBUrlType.VERTICA2);
         supportDBUrlMap.put(SupportDBUrlType.IMPALA.getDBKey(), SupportDBUrlType.IMPALA);
+        supportDBUrlMap.put(SupportDBUrlType.REDSHIFT.getDBKey(), SupportDBUrlType.REDSHIFT);
 
         // MOD mzhao bug 12313, 2010-04-02 There is not dbType in prv files before 4.0 release, here use driver class
         // name
@@ -130,6 +131,7 @@ public final class SupportDBUrlStore {
         supportDiverNameDBUrlMap.put(SupportDBUrlType.HIVEDEFAULTURL.getDbDriver(), SupportDBUrlType.HIVEDEFAULTURL);
         supportDiverNameDBUrlMap.put(SupportDBUrlType.VERTICA.getDbDriver(), SupportDBUrlType.VERTICA);
         supportDiverNameDBUrlMap.put(SupportDBUrlType.VERTICA2.getDbDriver(), SupportDBUrlType.VERTICA2);
+        supportDiverNameDBUrlMap.put(SupportDBUrlType.REDSHIFT.getDbDriver(), SupportDBUrlType.REDSHIFT);
         // ~12313
     }
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/dburl/SupportDBUrlType.java

@@ -170,7 +170,7 @@ public enum SupportDBUrlType {
               null,
               "org.exist.xmldb.DatabaseImpl", //$NON-NLS-1$
               null,
-              "eXist"),
+              "eXist"), //$NON-NLS-1$
     MDM("MDM", //$NON-NLS-1$
         "localhost", //$NON-NLS-1$
         "8080", //$NON-NLS-1$
@@ -178,7 +178,7 @@ public enum SupportDBUrlType {
         null,
         "", //$NON-NLS-1$
         null,
-        "MDM"),
+        "MDM"), //$NON-NLS-1$
     DELIMITEDFILE("DELIMITEDFILE", //$NON-NLS-1$
                   "", //$NON-NLS-1$
                   "", //$NON-NLS-1$
@@ -186,8 +186,8 @@ public enum SupportDBUrlType {
                   null,
                   "", //$NON-NLS-1$
                   null,
-                  "DELIMITEDFILE"),
-    JAVADEFAULTURL("Java", "localhost", null, "dbname", null, null, null, "Java"),
+                  "DELIMITEDFILE"), //$NON-NLS-1$
+    JAVADEFAULTURL("Java", "localhost", null, "dbname", null, null, null, "Java"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
     ACCESS("ACCESS", //$NON-NLS-1$
            null,
            null,
@@ -214,12 +214,12 @@ public enum SupportDBUrlType {
            "Impala"), //$NON-NLS-1$
     REDSHIFT("Redshift", //$NON-NLS-1$
              "localhost", //$NON-NLS-1$
-             "10000", //$NON-NLS-1$
+             "5439", //$NON-NLS-1$
              null,
              null,
-             "", //$NON-NLS-1$
+             "com.paraccel.Driver", //$NON-NLS-1$
              null,
-             "Redshift"), //$NON-NLS-1$
+             "ParAccel"), //$NON-NLS-1$
     VERTICA("Vertica", //$NON-NLS-1$
             null,
             "5433", //$NON-NLS-1$

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/utils/MetadataConnectionUtils.java

@@ -647,7 +647,6 @@ public class MetadataConnectionUtils {
      * @return
      */
     public static List<String> getTDQSupportDBTemplate() {
-
         try {
             if (GlobalServiceRegister.getDefault().isDQDriverServiceRegistered(IDriverService.class)) {
                 IDriverService driverService = (IDriverService) GlobalServiceRegister.getDefault().getDQDriverService(
@@ -670,17 +669,9 @@ public class MetadataConnectionUtils {
         if (conn == null) {
             return false;
         }
-        try {
-            if (GlobalServiceRegister.getDefault().isDQDriverServiceRegistered(IDriverService.class)) {
-                IDriverService driverService = (IDriverService) GlobalServiceRegister.getDefault().getDQDriverService(
-                        IDriverService.class);
-                if (conn instanceof DatabaseConnection) {
-                    String databaseType = ((DatabaseConnection) conn).getDatabaseType();
-                    return driverService.getTDQSupportDBTemplate().contains(databaseType);
-                }
-            }
-        } catch (Exception e) {
-            log.error(e, e);
+        if (conn instanceof DatabaseConnection) {
+            String databaseType = ((DatabaseConnection) conn).getDatabaseType();
+            return getTDQSupportDBTemplate().contains(databaseType);
         }
 
         return false;

main/plugins/org.talend.model/model/TalendFile.ecore

@@ -638,6 +638,8 @@
       </eAnnotations>
     </eStructuralFeatures>
     <eStructuralFeatures xsi:type="ecore:EAttribute" name="type" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
+    <eStructuralFeatures xsi:type="ecore:EReference" name="additionalProperties" upperBound="-1"
+        eType="#//AdditionalFieldMap" containment="true"/>
   </eClassifiers>
   <eClassifiers xsi:type="ecore:EClass" name="NodeType">
     <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
@@ -956,6 +958,20 @@
         eType="#//ScreenshotsMap" transient="true" containment="true"/>
     <eStructuralFeatures xsi:type="ecore:EReference" name="routinesDependencies" upperBound="-1"
         eType="#//ItemInforType" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="jobType" eType="ecore:EDataType http://www.eclipse.org/emf/2003/XMLType#//String">
+      <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+        <details key="kind" value="attribute"/>
+        <details key="name" value="name"/>
+        <details key="namespace" value="##targetNamespace"/>
+      </eAnnotations>
+    </eStructuralFeatures>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="framework" eType="ecore:EDataType http://www.eclipse.org/emf/2003/XMLType#//String">
+      <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+        <details key="kind" value="attribute"/>
+        <details key="name" value="name"/>
+        <details key="namespace" value="##targetNamespace"/>
+      </eAnnotations>
+    </eStructuralFeatures>
   </eClassifiers>
   <eClassifiers xsi:type="ecore:EClass" name="RequiredType">
     <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">

main/plugins/org.talend.model/model/TalendFile.genmodel

@@ -108,6 +108,7 @@
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//MetadataType/name"/>
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//MetadataType/source"/>
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//MetadataType/type"/>
+      <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference TalendFile.ecore#//MetadataType/additionalProperties"/>
     </genClasses>
     <genClasses ecoreClass="TalendFile.ecore#//NodeType">
       <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference TalendFile.ecore#//NodeType/elementParameter"/>
@@ -160,6 +161,8 @@
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//ProcessType/screenshot"/>
       <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference TalendFile.ecore#//ProcessType/screenshots"/>
       <genFeatures notify="false" createChild="false" propertySortChoices="true" ecoreFeature="ecore:EReference TalendFile.ecore#//ProcessType/routinesDependencies"/>
+      <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//ProcessType/jobType"/>
+      <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//ProcessType/framework"/>
     </genClasses>
     <genClasses ecoreClass="TalendFile.ecore#//RequiredType">
       <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference TalendFile.ecore#//RequiredType/job"/>

main/plugins/org.talend.model/model/metadata.ecore

@@ -363,6 +363,14 @@
     <eStructuralFeatures xsi:type="ecore:EAttribute" name="jcoVersion" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
     <eStructuralFeatures xsi:type="ecore:EReference" name="additionalProperties" upperBound="-1"
         eType="#//AdditionalConnectionProperty" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EReference" name="BWDataSources" upperBound="-1"
+        eType="#//SAPBWTable" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EReference" name="BWDataStoreObjects" upperBound="-1"
+        eType="#//SAPBWTable" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EReference" name="BWInfoCubes" upperBound="-1"
+        eType="#//SAPBWTable" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EReference" name="BWInfoObjects" upperBound="-1"
+        eType="#//SAPBWTable" containment="true"/>
   </eClassifiers>
   <eClassifiers xsi:type="ecore:EClass" name="SAPFunctionUnit" eSuperTypes="#//AbstractMetadataObject">
     <eOperations name="setDocument">
@@ -844,6 +852,16 @@
     <eStructuralFeatures xsi:type="ecore:EAttribute" name="propertyName" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
     <eStructuralFeatures xsi:type="ecore:EAttribute" name="Value" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
   </eClassifiers>
+  <eClassifiers xsi:type="ecore:EClass" name="SAPBWTable" eSuperTypes="#//SAPTable">
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="modelType" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="active" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EBoolean"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="sourceSystemName" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="infoAreaName" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="innerIOType" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
+  </eClassifiers>
+  <eClassifiers xsi:type="ecore:EClass" name="SAPBWTableField" eSuperTypes="#//SAPTableField">
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="logicalName" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
+  </eClassifiers>
   <eSubpackages name="cwm" nsURI="http://www.talend.org/cwm/2010" nsPrefix="cwm">
     <eSubpackages name="relational" nsURI="http://www.talend.org/cwm/resource/relational/2010"
         nsPrefix="relational">