TCOMP-28 Avro representation changes

https://jira.talendforge.org/browse/TBD-3228

main/plugins/org.talend.core.repository/plugin.xml

@@ -17,6 +17,7 @@
    <extension-point id="repository_review_filter" name="repository_review_filter" schema="schema/repository_review_filter.exsd"/>
    <extension-point id="repository_context" name="Repository_context" schema="schema/repository_context.exsd"/>
    <extension-point id="document_generator" name="document_generator" schema="schema/document_generator.exsd"/>
+   <extension-point id="git_content_service" name="git_content_service" schema="schema/git_content_service.exsd"/>
 
    <extension
          point="org.talend.core.runtime.service">

main/plugins/org.talend.core.repository/schema/git_content_service.exsd

@@ -0,0 +1,102 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!-- Schema file written by PDE -->
+<schema targetNamespace="org.talend.core.repository" xmlns="http://www.w3.org/2001/XMLSchema">
+<annotation>
+      <appinfo>
+         <meta.schema plugin="org.talend.core.repository" id="git_content_service" name="git_content_service"/>
+      </appinfo>
+      <documentation>
+         [Enter description of this extension point.]
+      </documentation>
+   </annotation>
+
+   <element name="extension">
+      <annotation>
+         <appinfo>
+            <meta.element />
+         </appinfo>
+      </annotation>
+      <complexType>
+         <sequence>
+            <element ref="service" minOccurs="1" maxOccurs="unbounded"/>
+         </sequence>
+         <attribute name="point" type="string" use="required">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="id" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="name" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+               <appinfo>
+                  <meta.attribute translatable="true"/>
+               </appinfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <element name="service">
+      <complexType>
+         <attribute name="class" type="string">
+            <annotation>
+               <documentation>
+                  
+               </documentation>
+               <appinfo>
+                  <meta.attribute kind="java" basedOn=":org.talend.core.repository.services.IGitRepositoryService"/>
+               </appinfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="since"/>
+      </appinfo>
+      <documentation>
+         [Enter the first release in which this extension point appears.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="examples"/>
+      </appinfo>
+      <documentation>
+         [Enter extension point usage example here.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="apiinfo"/>
+      </appinfo>
+      <documentation>
+         [Enter API information here.]
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appinfo>
+         <meta.section type="implementation"/>
+      </appinfo>
+      <documentation>
+         [Enter information about supplied implementation of this extension point.]
+      </documentation>
+   </annotation>
+
+
+</schema>

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/provider/GitContentServiceProviderManager.java

@@ -0,0 +1,49 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2015 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.repository.model.provider;
+
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IExtension;
+import org.eclipse.core.runtime.IExtensionPoint;
+import org.eclipse.core.runtime.IExtensionRegistry;
+import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.repository.services.IGitContentService;
+
+public class GitContentServiceProviderManager {
+
+    public static IGitContentService getGitContentService() {
+        IGitContentService service = null;
+        IExtensionRegistry extensionRegistry = Platform.getExtensionRegistry();
+        IExtensionPoint extensionPoint = extensionRegistry.getExtensionPoint("org.talend.core.repository.git_content_service"); //$NON-NLS-1$
+        if (extensionPoint != null) {
+            IExtension[] extensions = extensionPoint.getExtensions();
+            for (IExtension extension : extensions) {
+                IConfigurationElement[] configurationElements = extension.getConfigurationElements();
+                for (IConfigurationElement configurationElement : configurationElements) {
+                    try {
+                        Object object = configurationElement.createExecutableExtension("class"); //$NON-NLS-1$
+                        if (object instanceof IGitContentService) {
+                            service = (IGitContentService) object;
+                            break;
+                        }
+                    } catch (CoreException e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+            }
+        }
+        return service;
+    }
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/services/IGitContentService.java

@@ -0,0 +1,12 @@
+package org.talend.core.repository.services;
+
+public interface IGitContentService {
+
+    public abstract void setMenu(Object object);
+
+    public abstract boolean isGIT();
+
+    public abstract void createDropdownCombo(Object composite);
+
+    public abstract void configureCombo(String descriptor);
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CopyObjectAction.java

@@ -426,7 +426,8 @@ public class CopyObjectAction {
             }
         }
         if (sourceNode.getObjectType() != ERepositoryObjectType.PROCESS
-                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_MR) {
+                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_MR
+                && sourceNode.getObjectType() != ERepositoryObjectType.PROCESS_STORM) {
             return false;
         }
         if (!(newItem instanceof ProcessItem)) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/view/RepositoryLabelProvider.java

@@ -60,6 +60,7 @@ import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.repository.model.IRepositoryNode.ENodeType;
 import org.talend.repository.model.RepositoryNode;
 import org.talend.repository.ui.views.IRepositoryView;
+import org.talend.utils.string.MD5;
 
 /**
  * Label provider for the repository view. <code>DEBUG</code> boolean field specify if details (such as objects ids)
@@ -297,11 +298,12 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
 
             ImageDescriptor imageDesc = ImageUtils.createImageFromData(item.getIcon().getInnerContent());
             imageDesc = ImageUtils.scale(imageDesc, ICON_SIZE.ICON_32);
-
-            image = cachedImages.get(item.getIcon().getInnerContent());
+            String md5Desc = MD5.getMD5(item.getIcon().getInnerContent());
+            image = cachedImages.get(md5Desc);
+           
             if (image == null || image.isDisposed()) {
                 image = imageDesc.createImage();
-                cachedImages.put(item.getIcon().getInnerContent(), image);
+            	cachedImages.put(md5Desc, image);
             } else {
                 // image = imageDesc.createImage();
             }
@@ -309,7 +311,7 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
         return image;
     }
 
-    private static Map<byte[], Image> cachedImages = new HashMap<byte[], Image>();
+    private static Map<String, Image> cachedImages = new HashMap<String, Image>();
 
     @Override
     public Image getImage(Object obj) {

main/plugins/org.talend.core.runtime/META-INF/MANIFEST.MF

@@ -105,12 +105,14 @@ Require-Bundle: org.eclipse.jdt.core,
  org.apache.log4j,
  org.talend.model,
  org.talend.utils;visibility:=reexport,
+ org.talend.libraries.apache,
  org.talend.libraries.dom4j-jaxen,
  org.talend.libraries.csv,
  org.talend.commons.runtime;visibility:=reexport,
  org.talend.commons.ui,
  org.ops4j.pax.url.mvn,
- org.talend.components.api
+ org.talend.components.api,
+ org.talend.libraries.jackson
 Bundle-Activator: org.talend.core.runtime.CoreRuntimePlugin
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,

main/plugins/org.talend.core.runtime/mappings/mapping_Cassandra_datastax.xml

@@ -82,7 +82,6 @@
             <dbToTalendTypes>
                 <dbType type="ascii">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="bigint">
                     <talendType type="id_Long" default="true" />
@@ -110,8 +109,6 @@
                 </dbType>
                 <dbType type="int">
                     <talendType type="id_Integer" default="true" />
-                    <talendType type="id_Short" />
-                    <talendType type="id_Byte"/>
                 </dbType>
                 <dbType type="list">
                     <talendType type="id_List" default="true" />
@@ -124,7 +121,6 @@
                 </dbType>
                 <dbType type="text">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="timestamp">
                     <talendType type="id_Date" default="true" />
@@ -137,7 +133,6 @@
                 </dbType>
                 <dbType type="varchar">
                     <talendType type="id_String" default="true" />
-                    <talendType type="id_Character" />
                 </dbType>
                 <dbType type="varint">
                     <talendType type="id_Object" default="true" />

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConstants.java

@@ -32,6 +32,8 @@ public class HadoopConstants {
     public static final String FRAMEWORK_STORM = "Storm";
 
     public static final String FRAMEWORK_SPARKSTREAMING = "Spark Streaming";
+    
+    public static final String FRAMEWORK_SPARK_STREAMING = "Spark_Streaming";
 
     public static final String FRAMEWORK_MAPREDUCE = "MapReduce";
 
@@ -40,6 +42,10 @@ public class HadoopConstants {
     public static final String JOBTYPEBDSTREAMING = "Big Data Streaming";
 
     public static final String JOBTYPEBDBATCH = "Big Data Batch";
+    
+    public static final String JOBTYPE_BDSTREAMING = "Big_Data_Streaming";
+
+    public static final String JOBTYPE_BDBATCH = "Big_Data_Batch";
 
     public static final String SPARKSTREAMING_TYPE = "SPARKSTREAMING";
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/ComponentCategory.java

@@ -17,6 +17,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.service.IMRProcessService;
+import org.talend.core.service.IStormProcessService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 
 /**
@@ -96,6 +97,16 @@ public enum ComponentCategory {
                 return ComponentCategory.CATEGORY_4_MAPREDUCE;
             }
         }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IStormProcessService.class)) {
+            IStormProcessService stormService = (IStormProcessService) GlobalServiceRegister.getDefault().getService(
+                    IStormProcessService.class);
+            if (stormService.isSparkStreaming(item)) {
+                return ComponentCategory.CATEGORY_4_SPARKSTREAMING;
+            }
+            if (stormService.isStormItem(item)) {
+                return ComponentCategory.CATEGORY_4_STORM;
+            }
+        }
         if (item instanceof ProcessItem || item instanceof JobletProcessItem) {
             return ComponentCategory.CATEGORY_4_DI;
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleToInstall.java

@@ -43,6 +43,8 @@ public class ModuleToInstall {
 
     private String distribution = "";
 
+    private boolean fromCustomNexus;
+
     public String getName() {
         return this.name;
     }
@@ -176,4 +178,16 @@ public class ModuleToInstall {
         return this.mavenUris;
     }
 
+    /**
+     * Sets the fromCustomNexus.
+     * 
+     * @param fromCustomNexus the fromCustomNexus to set
+     */
+    public void setFromCustomNexus(boolean fromCustomNexus) {
+        this.fromCustomNexus = fromCustomNexus;
+    }
+
+    public boolean isFromCustomNexus() {
+        return this.fromCustomNexus;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -158,26 +158,15 @@ public class MetadataTable implements IMetadataTable, Cloneable {
 
     @Override
     public boolean isDynamicSchema() {
-
-        int sizeListColumns = listColumns.size();
-        boolean hasDynamic = false;
-        for (int i = 0; i < sizeListColumns; i++) {
-            if (listColumns.get(i).getTalendType().equals("id_Dynamic")) {
-                hasDynamic = true;
-                break;
-            }
-        }
-        return hasDynamic;
+        return getDynamicColumn() != null;
 
     }
 
     @Override
     public IMetadataColumn getDynamicColumn() {
-        if (isDynamicSchema()) {
-            for (int i = 0; i < listColumns.size(); i++) {
-                if (listColumns.get(i).getTalendType().equals("id_Dynamic")) {
-                    return listColumns.get(i);
-                }
+        for (IMetadataColumn column : listColumns) {
+            if (column.getTalendType().equals("id_Dynamic")) {
+                return column;
             }
         }
         return null;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -24,6 +24,16 @@ import java.util.Set;
 import java.util.Vector;
 import java.util.regex.Pattern;
 
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.LogicalTypes.Decimal;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.SchemaBuilder.BaseFieldTypeBuilder;
+import org.apache.avro.SchemaBuilder.FieldAssembler;
+import org.apache.avro.SchemaBuilder.FieldBuilder;
+import org.apache.avro.SchemaBuilder.PropBuilder;
+import org.apache.avro.SchemaBuilder.RecordBuilder;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.emf.common.util.BasicEList;
@@ -47,6 +57,7 @@ import org.talend.core.language.ECodeLanguage;
 import org.talend.core.language.LanguageManager;
 import org.talend.core.model.metadata.builder.connection.AbstractMetadataObject;
 import org.talend.core.model.metadata.builder.connection.Connection;
+import org.talend.core.model.metadata.builder.connection.ConnectionFactory;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.metadata.builder.connection.QueriesConnection;
 import org.talend.core.model.metadata.builder.connection.Query;
@@ -71,6 +82,9 @@ import org.talend.core.runtime.i18n.Messages;
 import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.utils.KeywordsValidator;
 import org.talend.cwm.helper.ConnectionHelper;
+import org.talend.cwm.helper.TaggedValueHelper;
+import org.talend.daikon.avro.util.AvroUtils;
+import org.talend.daikon.talend6.Talend6SchemaConstants;
 import org.talend.designer.core.model.utils.emf.talendfile.ColumnType;
 import org.talend.designer.core.model.utils.emf.talendfile.MetadataType;
 import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
@@ -1310,7 +1324,6 @@ public final class MetadataToolHelper {
             }
         }
         return null;
-
     }
 
     public static IMetadataTable convert(MetadataTable old) {
@@ -1399,4 +1412,386 @@ public final class MetadataToolHelper {
         return result;
     }
 
+    /**
+     * @return An Avro schema with enriched properties from the incoming metadata table.
+     */
+    public static org.apache.avro.Schema convertToAvro(MetadataTable in) {
+        RecordBuilder<Schema> builder = SchemaBuilder.builder().record(in.getLabel());
+        copyTableProperties(builder, in);
+
+        FieldAssembler<Schema> fa = builder.fields();
+        for (org.talend.core.model.metadata.builder.connection.MetadataColumn column : in.getColumns()) {
+            fa = convertToAvro(fa, column);
+        }
+        return fa.endRecord();
+    }
+
+    /**
+     * Copy all of the information from the MetadataTable in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyTableProperties(PropBuilder<T> builder, MetadataTable in) {
+
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Table-specific properties.
+        if (in.getName() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_NAME, in.getName());
+        }
+        if (in.getTableType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_TABLE_TYPE, in.getTableType());
+        }
+
+        return builder;
+    }
+
+    /**
+     * Build a field into a schema using enriched properties from the incoming column.
+     */
+    private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        FieldBuilder<Schema> fb = fa.name(in.getLabel());
+        copyColumnProperties(fb, in);
+        BaseFieldTypeBuilder<Schema> ftb = in.isNullable() ? fb.type() : fb.type().nullable();
+
+        String defaultValue = in.getInitialValue().getBody();
+
+        String tt = in.getTalendType();
+
+        // Numeric types.
+        if (JavaTypesManager.LONG.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.longType().noDefault()
+                    : ftb.longType().longDefault(Long.parseLong(defaultValue));
+        }
+        if (JavaTypesManager.INTEGER.getId().equals(tt) || JavaTypesManager.SHORT.getId().equals(tt)
+                || JavaTypesManager.BYTE.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.intType().noDefault()
+                    : ftb.intType().intDefault(Integer.parseInt(defaultValue));
+        }
+        if (JavaTypesManager.DOUBLE.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.doubleType().noDefault()
+                    : ftb.doubleType().doubleDefault(Double.parseDouble(defaultValue));
+        }
+        if (JavaTypesManager.FLOAT.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.floatType().noDefault()
+                    : ftb.floatType().floatDefault(Float.parseFloat(defaultValue));
+        }
+        if (JavaTypesManager.BIGDECIMAL.getId().equals(tt)) {
+            // decimal(precision, scale) == column length and precision?
+            Decimal d = LogicalTypes.decimal((int) in.getLength(), (int) in.getPrecision());
+            Schema bigdecimal = d.addToSchema(Schema.create(Schema.Type.BYTES));
+            return fb.type(bigdecimal).withDefault(defaultValue);
+        }
+
+        // Other primitive types that map directly to Avro.
+        if (JavaTypesManager.BOOLEAN.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.booleanType().noDefault()
+                    : ftb.booleanType().booleanDefault(Boolean.parseBoolean(defaultValue));
+        }
+        if (JavaTypesManager.BYTE_ARRAY.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.bytesType().noDefault()
+                    : ftb.bytesType().bytesDefault(defaultValue);
+        }
+        if (JavaTypesManager.DATE.getId().equals(tt)) {
+            // Date is saved as a long, like System.currentTimeMillis()
+            return ftb.longType().noDefault();
+        }
+        // String-ish types.
+        if (JavaTypesManager.STRING.getId().equals(tt) || JavaTypesManager.FILE.getId().equals(tt)
+                || JavaTypesManager.DIRECTORY.getId().equals(tt) || JavaTypesManager.VALUE_LIST.getId().equals(tt)
+                || JavaTypesManager.CHARACTER.getId().equals(tt) || JavaTypesManager.PASSWORD.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.stringType().noDefault()
+                    : ftb.stringType().stringDefault(defaultValue);
+        }
+
+        // Types with unknown elements, store as binary
+        if (JavaTypesManager.OBJECT.getId().equals(tt) || JavaTypesManager.DYNAMIC.getId().equals(tt)) {
+            return defaultValue == null //
+            ? ftb.bytesType().noDefault()
+                    : ftb.bytesType().bytesDefault(defaultValue);
+        }
+        if (JavaTypesManager.LIST.getId().equals(tt)) {
+            return ftb.array().items().bytesType().noDefault();
+        }
+
+        // Can this occur?
+        throw new UnsupportedOperationException("Unrecognized type " + tt); //$NON-NLS-1$
+    }
+
+    /**
+     * Copy all of the information from the IMetadataColumn in the form of key/value properties into an Avro object.
+     * 
+     * @param builder Any Avro builder capable of taking key/value in the form of strings.
+     * @param in The element to copy information from.
+     * @return the instance of the builder passed in.
+     */
+    private static <T extends PropBuilder<T>> PropBuilder<T> copyColumnProperties(PropBuilder<T> builder,
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+        // Properties common to tables and columns.
+        if (in.getId() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_ID, in.getId());
+        }
+        if (in.getComment() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COMMENT, in.getComment());
+        }
+        if (in.getLabel() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_LABEL, in.getLabel());
+        }
+        if (in.isReadOnly()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_IS_READ_ONLY, "true"); //$NON-NLS-1$
+        }
+        for (TaggedValue tv : in.getTaggedValue()) {
+            String additionalTag = tv.getTag();
+            if (tv.getValue() != null) {
+                builder.prop(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES + additionalTag, tv.getValue());
+            }
+        }
+
+        // Column-specific properties.
+        if (in.isKey()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY, "true"); //$NON-NLS-1$
+        }
+        if (in.getType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE, in.getSourceType());
+        }
+        if (in.getTalendType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE, in.getTalendType());
+        }
+        if (in.getPattern() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN, in.getPattern());
+        }
+        if (in.getLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH, String.valueOf((int) in.getLength()));
+        }
+        if (in.getOriginalLength() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH, String.valueOf(in.getOriginalLength()));
+        }
+        if (in.isNullable()) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE, "true"); //$NON-NLS-1$
+        }
+        if (in.getPrecision() >= 0) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION, String.valueOf(in.getPrecision()));
+        }
+        if (in.getInitialValue().getBody() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT, in.getInitialValue().getBody());
+        }
+        if (in.getName() != null) {
+            // keyword fixes?
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME, in.getName());
+        }
+        if (in.getRelatedEntity() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY, in.getRelatedEntity());
+        }
+        if (in.getRelationshipType() != null) {
+            builder.prop(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE, in.getRelationshipType());
+        }
+        return builder;
+    }
+
+    /**
+     * @param in A schema which must be of type {@link Schema.Type#RECORD}.
+     * @return A MetadataTable containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static MetadataTable convertFromAvro(org.apache.avro.Schema in) {
+        MetadataTable table = ConnectionFactory.eINSTANCE.createMetadataTable();
+
+        // Set the defaults values to the name (the only information guaranteed to be present in the schema.)
+        table.setId(in.getName());
+        table.setLabel(in.getName());
+        table.setName(in.getName());
+        table.setSourceName(in.getName());
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            table.setId(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            table.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            table.setLabel(null);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            table.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : in.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
+                table.getTaggedValue().add(tv);
+            }
+        }
+
+        // Table-specific properties.
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_NAME))) {
+            table.setName(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_TABLE_TYPE))) {
+            table.setTableType(prop);
+        }
+
+        // Add the columns.
+        List<org.talend.core.model.metadata.builder.connection.MetadataColumn> columns = new ArrayList<>(in.getFields().size());
+        for (Schema.Field f : in.getFields()) {
+            columns.add(convertFromAvro(f));
+        }
+        table.getColumns().addAll(columns);
+        return table;
+    }
+
+    /**
+     * @param in A field from an incoming schema
+     * @return A MetadataColumn containing all the information from the Schema, including any information included the
+     * schema as JSON property annotations for Talend 6 generated schemas.
+     */
+    public static org.talend.core.model.metadata.builder.connection.MetadataColumn convertFromAvro(Schema.Field field) {
+        org.talend.core.model.metadata.builder.connection.MetadataColumn col = ConnectionFactory.eINSTANCE.createMetadataColumn();
+        Schema in = field.schema();
+
+        // Set the defaults values to the name (the only information guaranteed to be available in every field).
+        col.setId(field.name());
+        col.setLabel(field.name());
+        col.setName(field.name());
+        switch (AvroUtils.unwrapIfNullable(in).getType()) {
+        case ARRAY:
+            col.setSourceType(JavaTypesManager.LIST.getId());
+            break;
+        case BOOLEAN:
+            col.setSourceType(JavaTypesManager.BOOLEAN.getId());
+            break;
+        case BYTES:
+        case FIXED:
+            col.setSourceType(JavaTypesManager.BYTE_ARRAY.getId());
+            break;
+        case DOUBLE:
+            col.setSourceType(JavaTypesManager.DOUBLE.getId());
+            break;
+        case FLOAT:
+            col.setSourceType(JavaTypesManager.FLOAT.getId());
+            break;
+        case INT:
+            col.setSourceType(JavaTypesManager.INTEGER.getId());
+            break;
+        case LONG:
+            col.setSourceType(JavaTypesManager.LONG.getId());
+            break;
+        case ENUM:
+        case STRING:
+            col.setSourceType(JavaTypesManager.STRING.getId());
+            break;
+        case RECORD:
+        case NULL:
+        case MAP:
+        case UNION:
+        default:
+            // Can this occur in the studio?
+            break;
+        }
+        // TODO setSourceType from the field Schema type.
+        col.setNullable(AvroUtils.isNullable(in));
+
+        // Extract any properties that have been saved in the enriched schema.
+
+        // Properties common to tables and columns.
+        String prop;
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_ID))) {
+            col.setId(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COMMENT))) {
+            col.setComment(in.getProp(Talend6SchemaConstants.TALEND6_ID));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_LABEL))) {
+            col.setLabel(null);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_IS_READ_ONLY))) {
+            col.setReadOnly(Boolean.parseBoolean(prop));
+        }
+        for (String key : in.getJsonProps().keySet()) {
+            if (key.startsWith(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES)) {
+                String originalKey = key.substring(Talend6SchemaConstants.TALEND6_ADDITIONAL_PROPERTIES.length());
+                TaggedValue tv = TaggedValueHelper.createTaggedValue(originalKey, in.getProp(key));
+                col.getTaggedValue().add(tv);
+            }
+        }
+
+        // Column-specific properties.
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_KEY))) {
+            col.setKey(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_SOURCE_TYPE))) {
+            col.setSourceType(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_TALEND_TYPE))) {
+            col.setTalendType(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PATTERN))) {
+            col.setPattern(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_LENGTH))) {
+            Long value = Long.parseLong(prop);
+            col.setOriginalLength(value > 0 ? value : -1);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_IS_NULLABLE))) {
+            col.setNullable(Boolean.parseBoolean(prop));
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_PRECISION))) {
+            Long value = Long.parseLong(prop);
+            col.setPrecision(value > 0 ? value : -1);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_DEFAULT))) {
+            col.setDefaultValue(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_ORIGINAL_DB_COLUMN_NAME))) {
+            col.setName(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATED_ENTITY))) {
+            col.setRelatedEntity(prop);
+        }
+        if (null != (prop = in.getProp(Talend6SchemaConstants.TALEND6_COLUMN_RELATIONSHIP_TYPE))) {
+            col.setRelationshipType(prop);
+        }
+
+        // If the source type wasn't set, there is an issue. Can this occur in the studio.
+        if (col.getSourceType() == null) {
+            throw new UnsupportedOperationException("Unrecognized type " + in); //$NON-NLS-1$
+        }
+
+        return col;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -2694,4 +2694,13 @@ public class RepositoryToComponentProperty {
         }
         return value;
     }
+
+    public static boolean isGenericRepositoryValue(Connection connection, String paramName) {
+        for (IDragAndDropServiceHandler handler : DragAndDropManager.getHandlers()) {
+            if (handler.canHandle(connection)) {
+                return handler.isGenericRepositoryValue(connection, paramName);
+            }
+        }
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EParameterFieldType.java

@@ -98,6 +98,7 @@ public enum EParameterFieldType {
     NAME_SELECTION_AREA, // ycbai added for generic wizard
     NAME_SELECTION_REFERENCE, // ycbai added for generic wizard
     COMPONENT_REFERENCE, // ycbai added for generic wizard
+    HIDDEN_TEXT, // hcyi added for generic wizard/component properties
 
     ;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -64,7 +64,7 @@ import org.talend.repository.model.IProxyRepositoryService;
  */
 @SuppressWarnings("unchecked")
 public final class ProcessUtils {
-    
+
     private static boolean isHD;
 
     private static List<IProcess> fakeProcesses = new ArrayList<IProcess>();
@@ -751,6 +751,20 @@ public final class ProcessUtils {
         return false;
     }
 
+    public static boolean isSparkStreming(IProcess process) {
+        if (process instanceof IProcess2) {
+            return false;
+        }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerCoreService.class)) {
+            ITestContainerCoreService testContainerService = (ITestContainerCoreService) GlobalServiceRegister.getDefault()
+                    .getService(ITestContainerCoreService.class);
+            if (testContainerService != null) {
+                return testContainerService.isSparkStreaming(((IProcess2) process).getProperty().getItem());
+            }
+        }
+        return false;
+    }
+
     public static List<String> getTestData(IProcess process, String instanceName) {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService = (ITestContainerProviderService) GlobalServiceRegister
@@ -930,26 +944,26 @@ public final class ProcessUtils {
 
         return false;
     }
-    
+
     public static boolean isHDInsight() {
         return isHD;
     }
-    
+
     public static void setHDInsight(boolean isHD) {
         ProcessUtils.isHD = isHD;
     }
-    
+
     public static boolean isDistributionExist(ProcessItem processItem) {
         EList<ElementParameterType> parameters = processItem.getProcess().getParameters().getElementParameter();
         for (ElementParameterType pt : parameters) {
             if (pt.getName().equals("DISTRIBUTION")) { //$NON-NLS-1$
-                 String value = pt.getValue();
-                 if("MICROSOFT_HD_INSIGHT".equals(value)){ //$NON-NLS-1$
-                     return true;
-                 }
+                String value = pt.getValue();
+                if ("MICROSOFT_HD_INSIGHT".equals(value)) { //$NON-NLS-1$
+                    return true;
+                }
             }
         }
         return false;
     }
-    
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -91,6 +91,9 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public static final String PROD_MDM = ProductUtils.PROD_MDM;
 
+    public final static ERepositoryObjectType GENERAL_FILE = new ERepositoryObjectType("repository.generalfile", "",
+            "GENERAL_FILE", true, 1, new String[] { PROD_DI, PROD_CAMEL, PROD_DQ, PROD_MDM }, new String[] {}, false);
+
     public final static ERepositoryObjectType GIT_ROOT = new ERepositoryObjectType("repository.gitroot", "", "GIT_ROOT", true, 1,
             new String[] { PROD_DI, PROD_CAMEL, PROD_DQ }, new String[] {}, false);
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/AbstractDragAndDropServiceHandler.java

@@ -56,4 +56,8 @@ public abstract class AbstractDragAndDropServiceHandler implements IDragAndDropS
         }
     }
 
+    @Override
+    public boolean isGenericRepositoryValue(Connection connection, String paramName) {
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/IDragAndDropServiceHandler.java

@@ -109,4 +109,13 @@ public interface IDragAndDropServiceHandler {
      */
     public boolean isValidForDataViewer(Connection connection, IMetadataTable metadataTable);
 
+    /**
+     * 
+     * DOC hcyi Comment method "isGenericRepositoryValue".
+     * 
+     * @param connection
+     * @param paramName
+     * @return
+     */
+    public boolean isGenericRepositoryValue(Connection connection, String paramName);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusServerUtils.java

@@ -25,15 +25,19 @@ import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSocketFactory;
 
+import org.apache.commons.lang.StringUtils;
 import org.dom4j.Document;
 import org.dom4j.Node;
 import org.dom4j.io.SAXReader;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.service.IRemoteService;
 import org.talend.utils.json.JSONException;
@@ -46,6 +50,11 @@ import org.talend.utils.ssl.SSLUtils;
  */
 public class NexusServerUtils {
 
+    /**
+     * 
+     */
+    public static final String ORG_TALEND_DESIGNER_CORE = "org.talend.designer.core"; //$NON-NLS-1$
+
     public static final String KEY_NEXUS_RUL = "nexusUrl";//$NON-NLS-1$ 
 
     public static final String KEY_NEXUS_USER = "username";//$NON-NLS-1$ 
@@ -131,6 +140,9 @@ public class NexusServerUtils {
      * @return
      */
     public static boolean checkConnectionStatus(String nexusUrl, String repositoryId, final String userName, final String password) {
+        if (StringUtils.isEmpty(nexusUrl)) {
+            return false;
+        }
         final Authenticator defaultAuthenticator = NetworkUtil.getDefaultAuthenticator();
         if (userName != null && !"".equals(userName)) {
             Authenticator.setDefault(new Authenticator() {
@@ -155,10 +167,6 @@ public class NexusServerUtils {
 
             URL url = new URL(urlToCheck);
             HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
-            // if (userName != null && !"".equals(userName)) {
-            // urlConnection.setRequestProperty(
-            //                        "Authorization", "Basic " + Base64.encodeBase64((userName + ":" + password).getBytes()));//$NON-NLS-1$ //$NON-NLS-2$
-            // }
             if (urlConnection instanceof HttpsURLConnection) {
                 String userDir = Platform.getInstallLocation().getURL().getPath();
                 final SSLSocketFactory socketFactory = SSLUtils.getSSLContext(userDir).getSocketFactory();
@@ -173,14 +181,17 @@ public class NexusServerUtils {
 
                 });
             }
-            urlConnection.setConnectTimeout(10000);
-            urlConnection.setReadTimeout(10000);
+            IEclipsePreferences node = InstanceScope.INSTANCE.getNode(ORG_TALEND_DESIGNER_CORE);
+            int timeout = node.getInt(ITalendCorePrefConstants.NEXUS_TIMEOUT, 10000);
+
+            urlConnection.setConnectTimeout(timeout);
+            urlConnection.setReadTimeout(timeout);
             status = urlConnection.getResponseCode();
             if (status == CONNECTION_OK) {
                 return true;
             }
         } catch (Exception e) {
-            // do nothing
+            ExceptionHandler.process(e);
         } finally {
             Authenticator.setDefault(defaultAuthenticator);
         }
@@ -352,7 +363,9 @@ public class NexusServerUtils {
 
             });
         }
-        urlConnection.setConnectTimeout(10000);
+        IEclipsePreferences node = InstanceScope.INSTANCE.getNode(ORG_TALEND_DESIGNER_CORE);
+        int timeout = node.getInt(ITalendCorePrefConstants.NEXUS_TIMEOUT, 10000);
+        urlConnection.setConnectTimeout(timeout);
         return urlConnection;
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/ITalendCorePrefConstants.java

@@ -181,4 +181,8 @@ public interface ITalendCorePrefConstants {
     public static final String PERFORMANCE_TAC_CONNECTION_TIMEOUT = "PERFORMANCE_TAC_CONNECTION_TIMEOUT"; //$NON-NLS-1$
 
     public static final String PERFORMANCE_JAVA_PROCESS_CODE_FORMATE_TIMEOUT = "PERFORMANCE_JAVA_PROCESS_CODE_FORMATE_TIMEOUT"; //$NON-NLS-1$
+
+    public static final String NEXUS_TIMEOUT = "NEXUS_TIMEOUT"; //$NON-NLS-1$
+
+    public static final String NEXUS_REFRESH_FREQUENCY = "NEXUS_REFRESH_FREQUENCY"; //$NON-NLS-1$
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/PreferenceManipulator.java

@@ -39,6 +39,8 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
 
     private static final String PREF_DELIMITER = "|"; //$NON-NLS-1$
 
+    private static final String LOCAL_MODE = "local";
+
     /** The preference store manipulated. */
     private IPreferenceStore store;
 
@@ -246,6 +248,18 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
         save();
     }
 
+    public int getLastLogonMode(String projectUrl, String projectName, String branch) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
+        int mode = store.getInt(hexKey);
+        return mode;
+    }
+
+    public void setLastLogonMode(String projectUrl, String projectName, String branch, int mode) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + branch).getBytes());
+        store.setValue(hexKey, mode);
+        save();
+    }
+
     public String getLastUser() {
         return store.getString(LAST_USED_USER);
     }
@@ -284,4 +298,15 @@ public final class PreferenceManipulator implements ITalendCorePrefConstants {
             }
         }
     }
+
+    public void setNewCreateBranchObjectId(String projectUrl, String projectName, String branch, String id) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
+        store.setValue(hexKey, id);
+        save();
+    }
+
+    public String getNewCreateBranchObjectId(String projectUrl, String projectName, String branch) {
+        String hexKey = Hex.encodeHexString((projectUrl + projectName + LOCAL_MODE + branch).getBytes());
+        return store.getString(hexKey);
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/ComponentServiceWithValueEvaluator.java

@@ -23,10 +23,8 @@ import org.talend.components.api.service.ComponentService;
 import org.talend.components.api.wizard.ComponentWizard;
 import org.talend.components.api.wizard.ComponentWizardDefinition;
 import org.talend.components.api.wizard.WizardImageType;
-import org.talend.daikon.NamedThing;
 import org.talend.daikon.properties.PropertyValueEvaluator;
 import org.talend.daikon.properties.service.Repository;
-import org.talend.daikon.schema.Schema;
 
 /**
  * created by ycbai on 2016年2月4日 Detailled comment
@@ -44,8 +42,9 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
     }
 
     @Override
-    public String storeProperties(ComponentProperties properties, String name, String repositoryLocation, Schema schema) {
-        return service.storeProperties(properties, name, repositoryLocation, schema);
+    public String storeProperties(ComponentProperties properties, String name, String repositoryLocation,
+            String schemaPropertyName) {
+        return service.storeProperties(properties, name, repositoryLocation, schemaPropertyName);
     }
 
     @Override
@@ -90,11 +89,6 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
         return service.beforePropertyPresent(propName, properties);
     }
 
-    @Override
-    public ComponentProperties commitFormValues(ComponentProperties properties, String formName) {
-        return service.commitFormValues(properties, formName);
-    }
-
     @Override
     public Set<String> getAllComponentNames() {
         return service.getAllComponentNames();
@@ -136,7 +130,7 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
     }
 
     @Override
-    public List<ComponentDefinition> getPossibleComponents(ComponentProperties properties) throws Throwable {
+    public List<ComponentDefinition> getPossibleComponents(ComponentProperties... properties) throws Throwable {
         return service.getPossibleComponents(properties);
     }
 
@@ -166,16 +160,23 @@ public class ComponentServiceWithValueEvaluator implements ComponentService {
         return service.validateProperty(propName, properties);
     }
 
-    private void setupChildrenPropertiesContextValueEvaluator(ComponentProperties compProperties,
-            PropertyValueEvaluator valueEvaluator) {
-        List<NamedThing> properties = compProperties.getProperties();
-        for (NamedThing property : properties) {
-            if (property instanceof ComponentProperties) {
-                ComponentProperties childComponentProperties = (ComponentProperties) property;
-                childComponentProperties.setValueEvaluator(valueEvaluator);
-                setupChildrenPropertiesContextValueEvaluator(childComponentProperties, valueEvaluator);
-            }
-        }
+    @Deprecated
+    public ComponentProperties commitFormValues(ComponentProperties properties, String formName) {
+        // to delete
+        return null;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see
+     * org.talend.daikon.properties.service.PropertiesService#cancelFormValues(org.talend.daikon.properties.Properties,
+     * java.lang.String)
+     */
+    @Override
+    public ComponentProperties cancelFormValues(ComponentProperties arg0, String arg1) {
+        // to implement
+        return null;
     }
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/IStormProcessService.java

@@ -17,9 +17,13 @@ import java.util.List;
 import org.eclipse.ui.IEditorPart;
 import org.talend.commons.ui.runtime.image.IImage;
 import org.talend.core.IService;
+import org.talend.core.model.components.IComponentsHandler;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.repository.IRepositoryEditorInput;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.repository.model.IRepositoryNode;
 import org.talend.repository.model.nodes.IProjectRepositoryNode;
@@ -51,4 +55,13 @@ public interface IStormProcessService extends IService {
 
     public IImage getStormWizardIcon(String framework);
 
+    public IComponentsHandler getSparkStreamingComponentsHandler();
+
+    public boolean isSparkStreaming(Item item);
+
+    public IRepositoryEditorInput createTestCaseStormProcessEditorInput(ProcessItem processItem, boolean load,
+            String originalJobID, List<INode> testNodes, Boolean lastVersion, Boolean readonly);
+
+    public IProcess2 createBigdataProcess(Item item);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/ITestContainerCoreService.java

@@ -22,4 +22,6 @@ import org.talend.core.model.properties.Item;
 public interface ITestContainerCoreService extends IService {
 
     public boolean isSpark(Item item);
+
+    public boolean isSparkStreaming(Item item);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -118,6 +118,8 @@ public interface IRunProcessService extends IService {
     public void setDelegateService(IRunProcessService delegateService);
 
     public void updateLibraries(Set<ModuleNeeded> jobModuleList, IProcess process);
+    
+    public void updateLibraries(Set<ModuleNeeded> jobModuleList, IProcess process, Set<ModuleNeeded> alreadyRetrievedModules);
 
     public void refreshView();
 

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/images/CoreImageProvider.java

@@ -29,6 +29,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.ui.component.ComponentsFactoryProvider;
 import org.talend.designer.core.IDesignerCoreService;
+import org.talend.utils.string.MD5;
 
 /**
  * amaumont class global comment. Detailled comment <br/>
@@ -80,17 +81,19 @@ public class CoreImageProvider {
         return getIcon(itemType);
     }
 
-    private static Map<ImageDescriptor, Image> componentCachedImages = new HashMap<ImageDescriptor, Image>();
+    private static Map<String, Image> componentCachedImages = new HashMap<String, Image>();
 
-    public static Image getComponentImageFromDesc(ImageDescriptor imageDescriptor) {
-        Image image = null;
-        image = componentCachedImages.get(imageDescriptor);
-        if (image == null || image.isDisposed()) {
-            image = imageDescriptor.createImage();
-            componentCachedImages.put(imageDescriptor, image);
-        }
-        return image;
-    }
+	public static Image getComponentImageFromDesc(
+			ImageDescriptor imageDescriptor) {
+		String md5Desc = MD5.getMD5(imageDescriptor.getImageData().data);
+		Image image = componentCachedImages.get(md5Desc);
+
+		if (image == null || image.isDisposed()) {
+			image = imageDescriptor.createImage();
+			componentCachedImages.put(md5Desc, image);
+		}
+		return image;
+	}
 
     public static Image getComponentIcon(IComponent component, ICON_SIZE iconSize) {
         if (component != null && iconSize != null) {
@@ -130,21 +133,24 @@ public class CoreImageProvider {
         if (name != null && !name.equals("")) { //$NON-NLS-1$
             for (IComponent component : ComponentsFactoryProvider.getInstance().getComponents()) {
                 if (name.equals(component.getName())) {
-                    Image image = componentCachedImages.get(component.getIcon16());
+                	String md5Desc16 = MD5.getMD5(component.getIcon16().getImageData().data);
+                    Image image = componentCachedImages.get(md5Desc16);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon16());
-                    image = componentCachedImages.get(component.getIcon24());
+                    componentCachedImages.remove(md5Desc16);
+                    String md5Desc24 = MD5.getMD5(component.getIcon24().getImageData().data);
+                    image = componentCachedImages.get(md5Desc24);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon24());
-                    image = componentCachedImages.get(component.getIcon32());
+                    componentCachedImages.remove(md5Desc24);
+                    String md5Desc32 = MD5.getMD5(component.getIcon32().getImageData().data);
+                    image = componentCachedImages.get(md5Desc32);
                     if (image != null && !image.isDisposed()) {
                         image.dispose();
                     }
-                    componentCachedImages.remove(component.getIcon32());
+                    componentCachedImages.remove(md5Desc32);
                 }
             }
         }

main/plugins/org.talend.core/src/main/java/org/talend/core/services/ICreateXtextProcessService.java

@@ -15,6 +15,7 @@ package org.talend.core.services;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.IService;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 import org.talend.designer.joblet.model.JobletProcess;
 
@@ -30,6 +31,8 @@ public interface ICreateXtextProcessService extends IService {
 
     // create the processType with a .job file created by metalanguage.
     public void addProcess(String path);
+    
+    public  ERepositoryObjectType createProcessItem();
 
     public String convertJobtoScript(ProcessType processType);
 

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -127,6 +127,8 @@ public class ProcessorUtilities {
     private static final int GENERATED_WITH_TRACES = 2;
 
     private static final String COMMA = ";"; //$NON-NLS-1$
+    
+    private static final Set<ModuleNeeded> retrievedJarsForCurrentBuild = new HashSet<ModuleNeeded>();
 
     public static void addOpenEditor(IEditorPart editor) {
         openedEditors.add(editor);
@@ -368,6 +370,7 @@ public class ProcessorUtilities {
 
             // this cache only keep the last main job's generation, so clear it since we regenerate a new job.
             LastGenerationInfo.getInstance().getLastGeneratedjobs().clear();
+            retrievedJarsForCurrentBuild.clear();
 
             // if it's the father, reset the processMap to ensure to have a good
             // code generation
@@ -458,7 +461,7 @@ public class ProcessorUtilities {
             LastGenerationInfo.getInstance().setModulesNeededPerJob(jobInfo.getJobId(), jobInfo.getJobVersion(), neededLibraries);
 
             // must install the needed libraries before generate codes with poms.
-            CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess);
+            CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess, retrievedJarsForCurrentBuild);
         }
         resetRunJobComponentParameterForContextApply(jobInfo, currentProcess, selectedContextName);
 
@@ -592,6 +595,7 @@ public class ProcessorUtilities {
                 processor.syntaxCheck();
             }
             needContextInCurrentGeneration = true;
+            retrievedJarsForCurrentBuild.clear();
             codeModified = false;
         }
     }
@@ -702,7 +706,7 @@ public class ProcessorUtilities {
 
                 // this cache only keep the last main job's generation, so clear it since we regenerate a new job.
                 LastGenerationInfo.getInstance().getLastGeneratedjobs().clear();
-
+                retrievedJarsForCurrentBuild.clear();
                 // if it's the father, reset the processMap to ensure to have a good
                 // code generation
                 ItemCacheManager.clearCache();
@@ -801,7 +805,7 @@ public class ProcessorUtilities {
                         neededLibraries);
 
                 // must install the needed libraries before generate codes with poms.
-                CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess);
+                CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess, retrievedJarsForCurrentBuild);
             }
             resetRunJobComponentParameterForContextApply(jobInfo, currentProcess, selectedContextName);
 

main/plugins/org.talend.libraries.apache/.classpath

@@ -1,12 +1,13 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
-	<classpathentry exported="true" kind="lib" path="lib/ant.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/apache-mime4j-0.6.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/castor-1.0.3.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/xerces-2.6.2.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/xmlbeans-2.3.0.jar"/>
-	<classpathentry kind="output" path="bin"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry exported="true" kind="lib" path="lib/avro-1.8.0.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry exported="true" kind="lib" path="lib/ant.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/apache-mime4j-0.6.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/castor-1.0.3.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/xerces-2.6.2.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/xmlbeans-2.3.0.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

main/plugins/org.talend.libraries.apache/META-INF/MANIFEST.MF

@@ -9,11 +9,23 @@ Bundle-ClassPath: .,
  lib/castor-1.0.3.jar,
  lib/jakarta-oro-2.0.8.jar,
  lib/xerces-2.6.2.jar,
- lib/xmlbeans-2.3.0.jar
+ lib/xmlbeans-2.3.0.jar,
+ lib/avro-1.8.0.jar
 Bundle-Vendor: .Talend SA.
 Require-Bundle: org.apache.log4j;resolution:=optional,
- org.talend.libraries.apache.common;visibility:=reexport
+ org.talend.libraries.apache.common;visibility:=reexport,
+ jackson-core-asl;bundle-version="1.9.13",
+ jackson-mapper-asl;bundle-version="1.9.13"
 Export-Package: 
+ org.apache.avro;version="1.8.0",
+ org.apache.avro.data;version="1.8.0",
+ org.apache.avro.file;version="1.8.0",
+ org.apache.avro.generic;version="1.8.0",
+ org.apache.avro.io;version="1.8.0",
+ org.apache.avro.io.parsing;version="1.8.0",
+ org.apache.avro.reflect;version="1.8.0",
+ org.apache.avro.specific;version="1.8.0",
+ org.apache.avro.util;version="1.8.0",
  org.apache.james.mime4j,
  org.apache.james.mime4j.codec,
  org.apache.james.mime4j.descriptor,

main/plugins/org.talend.libraries.apache/build.properties

@@ -2,7 +2,8 @@ output.. = bin/
 bin.includes = META-INF/,\
                .,\
                licences/,\
-               lib/
+               lib/,\
+               lib/avro-1.8.0.jar
 bin.excludes = libs_not_used/commons-betwixt-0.8.jar,\
                libs_not_used/commons-digester-1.7.jar,\
                libs_not_used/commons-discovery-0.2.jar,\

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/wizards/AcceptModuleLicensesWizardPage.java

@@ -292,6 +292,10 @@ public class AcceptModuleLicensesWizardPage extends WizardPage {
                     }
                     Map<String, License> typeToLicense = new HashMap<String, License>();
                     for (ModuleToInstall moduleToInstall : mti) {
+                        // no need accept license if it is from custom nexus
+                        if (moduleToInstall.isFromCustomNexus()) {
+                            continue;
+                        }
                         String licenseType = moduleToInstall.getLicenseType();
                         if (licenseType == null) {
                             licenseType = UNKNOWN_LICENSE;

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/DownloadModuleRunnable.java

@@ -31,6 +31,8 @@ import org.ops4j.pax.url.mvn.Handler;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
+import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
+import org.talend.core.model.general.ModuleStatusProvider;
 import org.talend.core.model.general.ModuleToInstall;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.librariesmanager.ui.LibManagerUiPlugin;
@@ -78,7 +80,7 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
         SubMonitor subMonitor = SubMonitor
                 .convert(
                         monitor,
-                        Messages.getString("ExternalModulesInstallDialog.downloading2") + " (" + toDownload.size() + ")", toDownload.size() + 1); //$NON-NLS-1$
+                        Messages.getString("ExternalModulesInstallDialog.downloading2") + " (" + toDownload.size() + ")", toDownload.size()); //$NON-NLS-1$
 
         Map<String, String> customUriToAdd = new HashMap<String, String>();
         // TUP-3135 : stop to try to download at the first timeout.
@@ -89,20 +91,27 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
                 boolean accepted;
                 try {
                     // check license
-                    boolean isLicenseAccepted = LibManagerUiPlugin.getDefault().getPreferenceStore()
-                            .getBoolean(module.getLicenseType());
+                    boolean isLicenseAccepted = module.isFromCustomNexus()
+                            || LibManagerUiPlugin.getDefault().getPreferenceStore().getBoolean(module.getLicenseType());
                     accepted = isLicenseAccepted;
                     if (!accepted) {
                         subMonitor.worked(1);
                         continue;
                     }
-                    NexusDownloadHelperWithProgress downloader = new NexusDownloadHelperWithProgress();
+                    NexusDownloadHelperWithProgress downloader = new NexusDownloadHelperWithProgress(module);
                     if (!module.getMavenUris().isEmpty()) {
                         for (String mvnUri : module.getMavenUris()) {
                             downloader.download(new URL(null, mvnUri, new Handler()), null, subMonitor.newChild(1));
+                            // update module status
+                            final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
+                            statusMap.put(MavenUrlHelper.generateSnapshotMavenUri(mvnUri), ELibraryInstallStatus.INSTALLED);
                         }
                     } else {
                         downloader.download(new URL(null, module.getMavenUri(), new Handler()), null, subMonitor.newChild(1));
+                        // update module status
+                        final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
+                        statusMap.put(MavenUrlHelper.generateSnapshotMavenUri(module.getMavenUri()),
+                                ELibraryInstallStatus.INSTALLED);
                     }
 
                     // deploy to index as snapshot
@@ -129,14 +138,16 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
             ILibraryManagerService libraryManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
                     .getService(ILibraryManagerService.class);
             libraryManagerService.forceListUpdate();
-            LibManagerUiPlugin.getDefault().getLibrariesService().resetModulesNeeded();
         }
-        subMonitor.worked(1);
     }
 
     protected boolean hasLicensesToAccept() {
         if (toDownload != null && toDownload.size() > 0) {
             for (ModuleToInstall module : toDownload) {
+                // no need accept license if it is from custom nexus
+                if (module.isFromCustomNexus()) {
+                    continue;
+                }
                 String licenseType = module.getLicenseType();
                 if (licenseType != null) {
                     boolean isLicenseAccepted = LibManagerUiPlugin.getDefault().getPreferenceStore()

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/RemoteModulesHelper.java

@@ -26,6 +26,8 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -104,129 +106,202 @@ public class RemoteModulesHelper {
 
         @Override
         public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
-            final Set<String> mavenUriSet = contextMap.keySet();
-            Map<String, MavenArtifact> requiredArtifacts = new HashMap<String, MavenArtifact>();
+            final Set<String> mavenUrisTofetch = new HashSet<String>(contextMap.keySet());
+            monitor.beginTask(Messages.getString("RemoteModulesHelper.fetch.module.info"), mavenUrisTofetch.size() * 10 + 10);//$NON-NLS-1$
+            // fetch modules from local nexus first
+            final NexusServerBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
+            if (customNexusServer != null) {
+                if (addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, localCache)) {
+                    if (collectModulesWithJarName) {
+                        collectModulesWithJarName(toInstall);
+                    }
+                    monitor.done();
+                    return;
+                }
+                // search from local nexus
+                searchFromLocalNexus(monitor);
+                // check again after search
+                if (addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, localCache)) {
+                    if (collectModulesWithJarName) {
+                        collectModulesWithJarName(toInstall);
+                    }
+                    monitor.done();
+                    return;
+                }
+            }
+
+            if (addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, remoteCache)) {
+                if (collectModulesWithJarName) {
+                    collectModulesWithJarName(toInstall);
+                }
+                monitor.done();
+                return;
+            }
+
             Set<String> unavailableModules = new HashSet<String>();
-            monitor.beginTask(Messages.getString("RemoteModulesHelper.fetch.module.info"), mavenUriSet.size() * 10 + 10);//$NON-NLS-1$
             // if the network is not valid, all jars are not available.
             boolean networkValid = NetworkUtil.isNetworkValid();
             if (!networkValid) {
-                unavailableModules.addAll(mavenUriSet);
                 if (!alreadyWarnedAboutConnectionIssue) {
                     log.warn("failed to connect to internet");
                     alreadyWarnedAboutConnectionIssue = true;
                 }// else already warned so do nothing
-            }
-            if (networkValid) {
-                try {
-                    int index = 0;
-                    int limit = 100;
-                    final Iterator<String> iterator = mavenUriSet.iterator();
-                    while (index < mavenUriSet.size()) {
-                        // get block of 100 jars
-                        String jarsToCheck = "";
-                        while (index < limit && index < mavenUriSet.size()) {
-                            index++;
-                            String uriToCheck = iterator.next();
-                            final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(uriToCheck);
-                            if (parseMvnUrl != null) {
-                                requiredArtifacts.put(uriToCheck, parseMvnUrl);
-                                jarsToCheck += parseMvnUrl.getArtifactId();
-                                if (index < limit && index < mavenUriSet.size()) {
-                                    jarsToCheck += ",";
-                                }
-                            }
-                        }
-                        limit += 100;
-                        TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
-                        NexusServerBean nexusServer = manager.getLibrariesNexusServer();
-
-                        List<MavenArtifact> searchResults = manager.search(nexusServer.getServer(), nexusServer.getUserName(),
-                                nexusServer.getPassword(), nexusServer.getRepositoryId(), MavenConstants.DEFAULT_LIB_GROUP_ID,
-                                jarsToCheck, null);
-                        monitor.worked(10);
-                        for (MavenArtifact artifact : searchResults) {
-                            String artifactId = artifact.getArtifactId();
-                            String packageName = artifact.getType();
-                            if (packageName == null) {
-                                packageName = MavenConstants.TYPE_JAR;
-                            }
-                            String version = artifact.getVersion();
-                            String description = artifact.getDescription();
-                            String license = artifact.getLicense();
-                            String license_url = artifact.getLicenseUrl();
-                            String packaging = artifact.getType();
-                            String url = null;
-                            if (artifact.getUrl() != null && !"".equals(artifact.getUrl())) {
-                                url = artifact.getUrl();
-                            }
-                            ModuleToInstall m = new ModuleToInstall();
-                            m.setName(artifactId + "." + packageName);
-                            String mvnUri = MavenUrlHelper.generateMvnUrl(artifact.getGroupId(), artifactId, version, packaging,
-                                    artifact.getClassifier());
-                            m.setMavenUri(mvnUri);
-                            m.setLicenseType(license);
-                            m.setLicenseUrl(license_url);
-                            m.setDescription(description);
-                            m.setUrl_description(url);
-                            m.setUrl_download(url);
-                            if (artifact.getType() == null
-                                    || "".equals(artifact.getType()) || MavenConstants.PACKAGING_POM.equals(artifact.getType())) { //$NON-NLS-1$
-                                m.setDistribution(MavenConstants.DOWNLOAD_MANUAL);
-                            } else {
-                                m.setDistribution(artifact.getType());
-                            }
-
-                            cache.put(mvnUri, m);
-
-                            monitor.worked(10);
-                        }
-
-                    }
-
-                } catch (Exception e1) {
-                    ExceptionHandler.process(e1);
-                }
-                for (String uri : requiredArtifacts.keySet()) {
-                    String mvnUri = uri;
-                    // incase the package is not set int mavenuri
-                    final MavenArtifact artifact = requiredArtifacts.get(mvnUri);
-                    if (artifact.getType() == null) {
-                        artifact.setType(MavenConstants.TYPE_JAR);
-                    }
-                    mvnUri = MavenUrlHelper.generateMvnUrl(artifact.getGroupId(), artifact.getArtifactId(),
-                            artifact.getVersion(), artifact.getType(), artifact.getClassifier());
-                    ModuleToInstall moduleToInstall = cache.get(mvnUri);
-                    if (moduleToInstall != null) {
-                        setContext(moduleToInstall, uri, contextMap);
-                        toInstall.add(moduleToInstall);
-                    } else {
-                        String artifactId = artifact.getArtifactId();
-                        String type = artifact.getType();
-                        if (type == null) {
-                            type = MavenConstants.TYPE_JAR;
-                        }
-                        String name = artifactId + "." + type;
-                        unavailableModules.add(uri);
-                        ExceptionHandler.log("The download URL for " + name + " is not available");//$NON-NLS-1$//$NON-NLS-2$
-                        if (CommonsPlugin.isDebugMode()) {
-                            appendToLogFile(name + "\n");
-                        }
-                    }
-                    monitor.worked(10);
-                }
+            } else {
+                searchFromRemoteNexus(mavenUrisTofetch, monitor);
+                addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, remoteCache);
             }
 
+            unavailableModules.addAll(mavenUrisTofetch);
             addUnavailableModulesToToBeInstalledModules(unavailableModules, toInstall, contextMap);
 
             if (collectModulesWithJarName) {
                 collectModulesWithJarName(toInstall);
             }
+            Collections.sort(toInstall, new Comparator<ModuleToInstall>() {
 
+                @Override
+                public int compare(ModuleToInstall o1, ModuleToInstall o2) {
+                    return o1.getName().compareTo(o2.getName());
+                }
+            });
             monitor.done();
         }
     }
 
+    private void searchFromLocalNexus(IProgressMonitor monitor) {
+        try {
+            TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+            NexusServerBean customServer = manager.getCustomNexusServer();
+
+            List<MavenArtifact> searchResults = manager.search(customServer.getServer(), customServer.getUserName(),
+                    customServer.getPassword(), customServer.getRepositoryId(), MavenConstants.DEFAULT_LIB_GROUP_ID, null, null);
+            monitor.worked(10);
+            addModulesToCache(searchResults, localCache);
+
+        } catch (Exception e1) {
+            ExceptionHandler.process(e1);
+        }
+    }
+
+    private void searchFromRemoteNexus(Set<String> mavenUristoSearch, IProgressMonitor monitor) {
+        try {
+            int index = 0;
+            int limit = 100;
+            final Iterator<String> iterator = mavenUristoSearch.iterator();
+            while (index < mavenUristoSearch.size()) {
+                // get block of 100 jars
+                String jarsToCheck = "";
+                while (index < limit && index < mavenUristoSearch.size()) {
+                    index++;
+                    String uriToCheck = iterator.next();
+                    final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(uriToCheck);
+                    if (parseMvnUrl != null) {
+                        jarsToCheck += parseMvnUrl.getArtifactId();
+                        if (index < limit && index < mavenUristoSearch.size()) {
+                            jarsToCheck += ",";
+                        }
+                    }
+                }
+                limit += 100;
+                TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+                NexusServerBean nexusServer = manager.getLibrariesNexusServer();
+
+                List<MavenArtifact> searchResults = manager.search(nexusServer.getServer(), nexusServer.getUserName(),
+                        nexusServer.getPassword(), nexusServer.getRepositoryId(), MavenConstants.DEFAULT_LIB_GROUP_ID,
+                        jarsToCheck, null);
+                monitor.worked(10);
+                addModulesToCache(searchResults, remoteCache);
+            }
+
+        } catch (Exception e1) {
+            ExceptionHandler.process(e1);
+        }
+    }
+
+    private void addModulesToCache(List<MavenArtifact> searchResults, Map<String, ModuleToInstall> theCache) {
+        for (MavenArtifact artifact : searchResults) {
+            String artifactId = artifact.getArtifactId();
+            String packageName = artifact.getType();
+            if (packageName == null) {
+                packageName = MavenConstants.TYPE_JAR;
+            }
+            String version = artifact.getVersion();
+            if (version.endsWith(MavenConstants.SNAPSHOT)) {
+                version = version.substring(0, version.indexOf(MavenConstants.SNAPSHOT));
+            }
+            String description = artifact.getDescription();
+            String license = artifact.getLicense();
+            String license_url = artifact.getLicenseUrl();
+            String packaging = artifact.getType();
+            String url = null;
+            if (artifact.getUrl() != null && !"".equals(artifact.getUrl())) {
+                url = artifact.getUrl();
+            }
+            ModuleToInstall m = new ModuleToInstall();
+            m.setName(artifactId + "." + packageName);
+            String mvnUri = MavenUrlHelper.generateMvnUrl(artifact.getGroupId(), artifactId, version, packaging,
+                    artifact.getClassifier());
+            m.setMavenUri(mvnUri);
+            m.setLicenseType(license);
+            m.setLicenseUrl(license_url);
+            m.setDescription(description);
+            m.setUrl_description(url);
+            m.setUrl_download(url);
+            if (artifact.getType() == null
+                    || "".equals(artifact.getType()) || MavenConstants.PACKAGING_POM.equals(artifact.getType())) { //$NON-NLS-1$
+                m.setDistribution(MavenConstants.DOWNLOAD_MANUAL);
+            } else {
+                m.setDistribution(artifact.getType());
+            }
+            if (theCache == localCache) {
+                m.setFromCustomNexus(true);
+            }
+            theCache.put(mvnUri, m);
+        }
+    }
+
+    /**
+     * 
+     * DOC wchen Comment method "addCachedModulesToToBeInstallModules".
+     * 
+     * @param toInstall
+     * @param mavenUrisTofetch
+     * @param contextMap
+     * @param theCache
+     * @return if find all modules form the cache ,return true;
+     */
+    private boolean addCachedModulesToToBeInstallModules(List<ModuleToInstall> toInstall, Set<String> mavenUrisTofetch,
+            Map<String, List<ModuleNeeded>> contextMap, Map<String, ModuleToInstall> theCache) {
+        if (theCache.isEmpty()) {
+            return false;
+        }
+        final Iterator<String> iterator = mavenUrisTofetch.iterator();
+        while (iterator.hasNext()) {
+            String mvnUri = iterator.next();
+            ModuleToInstall moduleToInstall = null;
+            moduleToInstall = theCache.get(mvnUri);
+            if (moduleToInstall != null) {
+                List<ModuleNeeded> moduleContext = contextMap.get(mvnUri);
+                setContext(moduleToInstall, mvnUri, contextMap);
+                if (moduleContext != null && moduleContext.size() > 0) {
+                    for (ModuleNeeded needed : moduleContext) {
+                        if (moduleToInstall.getName().equals(needed.getModuleName())) {
+                            moduleToInstall.setRequired(needed.isRequired());
+                        }
+                    }
+                }
+
+                toInstall.add(moduleToInstall);
+                iterator.remove();
+            }
+        }
+
+        if (mavenUrisTofetch.isEmpty()) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * DOC sgandon Comment method "addUnavailableModulesToToBeInstalledModules".
      * 
@@ -254,14 +329,17 @@ public class RemoteModulesHelper {
     private ModuleToInstall createUnavailableModuleToInstall(String mvnUri, Map<String, List<ModuleNeeded>> contextMap) {
         ModuleToInstall m = new ModuleToInstall();
         m.setDistribution(MavenConstants.DOWNLOAD_MANUAL);
+        m.setMavenUri(mvnUri);
         setContext(m, mvnUri, contextMap);
+        String name = null;
         if (contextMap != null) {
             final List<ModuleNeeded> neededModules = contextMap.get(mvnUri);
-            m.setName(neededModules.get(0).getModuleName());
+            name = neededModules.get(0).getModuleName();
+            m.setName(name);
             m.setDescription(getFirstDescription(neededModules));
         } else {
             final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mvnUri);
-            String name = parseMvnUrl.getArtifactId();
+            name = parseMvnUrl.getArtifactId();
             String type = parseMvnUrl.getType();
             if (type == null) {
                 type = MavenConstants.TYPE_JAR;
@@ -269,6 +347,10 @@ public class RemoteModulesHelper {
             m.setName(name + "." + type);
 
         }
+        ExceptionHandler.log("The download URL for " + name + " is not available");//$NON-NLS-1$//$NON-NLS-2$
+        if (CommonsPlugin.isDebugMode()) {
+            appendToLogFile(name + "\n");
+        }
 
         return m;
     }
@@ -353,7 +435,9 @@ public class RemoteModulesHelper {
     /**
      * key : mvnuri(without SANPSHOT in version) value : ModuleToInstall
      */
-    private Map<String, ModuleToInstall> cache = new HashMap<String, ModuleToInstall>();
+    private Map<String, ModuleToInstall> remoteCache = new HashMap<String, ModuleToInstall>();
+
+    private Map<String, ModuleToInstall> localCache = new HashMap<String, ModuleToInstall>();
 
     private RemoteModulesHelper() {
     }
@@ -462,6 +546,7 @@ public class RemoteModulesHelper {
             List<ModuleToInstall> toInstall, boolean collectModulesWithJarName) {
         Map<String, List<ModuleNeeded>> contextMap = new HashMap<String, List<ModuleNeeded>>();
 
+        // collect mvnuri and modules incase many modules have the same mvnuri
         final Iterator<ModuleNeeded> iterator = neededModules.iterator();
         while (iterator.hasNext()) {
             ModuleNeeded module = iterator.next();
@@ -484,6 +569,7 @@ public class RemoteModulesHelper {
                     for (String snapshotUri : configuredSnapshotMvnUris) {
                         String uri = MavenUrlHelper.generateUriFromSnapshot(snapshotUri);
                         if (uri != null) {
+                            uri = addTypeForMavenUri(uri, module.getModuleName());
                             ModuleNeeded newModule = new ModuleNeeded(null, module.getModuleName(), null, true);
                             newModule.setMavenUri(uri);
                             if (!contextMap.keySet().contains(uri)) {
@@ -498,6 +584,7 @@ public class RemoteModulesHelper {
                 }
             }
             if (mvnUri != null) {
+                mvnUri = addTypeForMavenUri(mvnUri, module.getModuleName());
                 if (!contextMap.keySet().contains(mvnUri)) {
                     List<ModuleNeeded> modules = new ArrayList<ModuleNeeded>();
                     modules.add(module);
@@ -507,37 +594,24 @@ public class RemoteModulesHelper {
                 }
             }
         }
-        Set<String> contextKeys = new HashSet<String>(contextMap.keySet());
-        for (String mvnUri : contextKeys) {
-            ModuleToInstall moduleToInstall = null;
-            moduleToInstall = cache.get(mvnUri);
-            if (moduleToInstall != null) {
-                List<ModuleNeeded> moduleContext = contextMap.get(mvnUri);
-                setContext(moduleToInstall, mvnUri, contextMap);
-                if (moduleContext != null && moduleContext.size() > 0) {
-                    for (ModuleNeeded needed : moduleContext) {
-                        if (moduleToInstall.getName().equals(needed.getModuleName())) {
-                            moduleToInstall.setRequired(needed.isRequired());
-                        }
-                    }
-                }
-                toInstall.add(moduleToInstall);
-                contextMap.remove(mvnUri);
-            }
-        }
-
-        if (contextMap.isEmpty()) {
-            if (collectModulesWithJarName) {
-                collectModulesWithJarName(toInstall);
-            }
-            return null;
-        }
-
         // fetch the jars which are not in cache.
         return createRemoteModuleFetchRunnable(contextMap, toInstall, collectModulesWithJarName);
 
     }
 
+    private String addTypeForMavenUri(String uri, String moduleName) {
+        // make sure that mvn uri have the package
+        final String[] split = uri.split("/");
+        if (split.length < 4) {
+            String type = MavenConstants.TYPE_JAR;
+            if (moduleName.lastIndexOf(".") != -1) {
+                type = moduleName.substring(moduleName.lastIndexOf(".") + 1, moduleName.length());
+            }
+            uri = uri + "/" + type;
+        }
+        return uri;
+    }
+
     public RemoteModulesFetchRunnable getNotInstalledModulesRunnable(List<ModuleNeeded> neededModules,
             List<ModuleToInstall> toInstall) {
         return getNotInstalledModulesRunnable(neededModules, toInstall, false);

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/nexus/NexusDownloadHelperWithProgress.java

@@ -12,8 +12,17 @@
 // ============================================================================
 package org.talend.librariesmanager.utils.nexus;
 
+import java.io.File;
+import java.net.URL;
+
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.ops4j.pax.url.mvn.MavenResolver;
 import org.talend.core.download.DownloadHelperWithProgress;
 import org.talend.core.download.IDownloadHelper;
+import org.talend.core.model.general.ModuleToInstall;
+import org.talend.core.nexus.NexusServerBean;
+import org.talend.core.runtime.maven.MavenUrlHelper;
+import org.talend.designer.maven.talendlib.TalendLibsServerManager;
 
 /**
  * created by wchen on Apr 24, 2015 Detailled comment
@@ -21,6 +30,35 @@ import org.talend.core.download.IDownloadHelper;
  */
 public class NexusDownloadHelperWithProgress extends DownloadHelperWithProgress {
 
+    private ModuleToInstall toInstall;
+
+    public NexusDownloadHelperWithProgress(ModuleToInstall toInstall) {
+        this.toInstall = toInstall;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.talend.core.download.DownloadHelperWithProgress#download(java.net.URL, java.io.File,
+     * org.eclipse.core.runtime.IProgressMonitor)
+     */
+    @Override
+    public void download(URL componentUrl, File destination, IProgressMonitor progressMonitor) throws Exception {
+        File resolved = null;
+        if (toInstall.isFromCustomNexus()) {
+            final NexusServerBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
+            if (customNexusServer != null) {
+                progressMonitor.subTask(toInstall.getName() + " from " + customNexusServer.getServer());
+                final MavenResolver mavenResolver = TalendLibsServerManager.getInstance().getMavenResolver();
+                resolved = mavenResolver.resolve(MavenUrlHelper.generateSnapshotMavenUri(componentUrl.toExternalForm()));
+            }
+        }
+        if (resolved != null && resolved.exists()) {
+            return;
+        }
+        super.download(componentUrl, destination, progressMonitor);
+    }
+
     /*
      * (non-Javadoc)
      * 

main/plugins/org.talend.librariesmanager/resources/java/routines/system/ComponentRuntimeContainerBase.java

@@ -1,43 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package routines.system;
-
-import java.util.Date;
-import java.util.Map;
-
-import org.talend.components.api.runtime.ComponentDynamicHolder;
-import org.talend.components.api.runtime.ComponentRuntimeContainer;
-
-public class ComponentRuntimeContainerBase implements ComponentRuntimeContainer {
-
-	@Override
-	public ComponentDynamicHolder createDynamicHolder() {
-		throw new RuntimeException("This must be subclassed");
-	}
-
-	@Override
-	public String formatDate(Date date, String pattern) {
-		return FormatterUtils.format(date, pattern);
-	}
-
-	@Override
-	public Map<String, Object> getGlobalMap() {
-		throw new RuntimeException("This must be subclassed");
-	}
-
-	@Override
-	public String getCurrentComponentName() {
-		throw new RuntimeException("This must be subclassed");
-	}
-
-}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -12,21 +12,29 @@
 // ============================================================================
 package org.talend.librariesmanager.model.service;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Collection;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import javax.xml.bind.DatatypeConverter;
+
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -34,10 +42,14 @@ import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.ConfigurationScope;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.emf.common.util.EMap;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.FrameworkUtil;
 import org.osgi.framework.ServiceReference;
+import org.osgi.service.prefs.BackingStoreException;
 import org.talend.commons.CommonsPlugin;
 import org.talend.commons.exception.CommonExceptionHandler;
 import org.talend.commons.exception.ExceptionHandler;
@@ -58,6 +70,8 @@ import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.general.ModuleStatusProvider;
 import org.talend.core.nexus.NexusServerBean;
+import org.talend.core.nexus.NexusServerUtils;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
@@ -321,11 +335,10 @@ public class LocalLibraryManager implements ILibraryManagerService {
             if (jarFile == null) {
                 jarFile = getJarFile(jarNeeded);
             }
-            // retreive form custom nexus server automatically
+            // retrieve form custom nexus server automatically
             TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
             final NexusServerBean customNexusServer = manager.getCustomNexusServer();
             if (customNexusServer != null) {
-                // TODO????? Resolver all versions if the mavenUri is null
                 Set<String> toResolve = new HashSet<String>();
                 if (snapshotMvnUri != null) {
                     toResolve.add(snapshotMvnUri);
@@ -343,27 +356,30 @@ public class LocalLibraryManager implements ILibraryManagerService {
                     }
                 }
                 for (String uri : toResolve) {
-                    MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
-                    final List<MavenArtifact> searchResults = manager.search(customNexusServer.getServer(),
-                            customNexusServer.getUserName(), customNexusServer.getPassword(),
-                            customNexusServer.getRepositoryId(), artifact.getGroupId(), artifact.getArtifactId(),
-                            artifact.getVersion());
-                    if (!searchResults.isEmpty()) {
-                        File resolvedFile = null;
-                        for (MavenArtifact result : searchResults) {
-                            if (jarNeeded.equals(result.getArtifactId() + "." + result.getType())) {
-                                resolvedFile = manager.getMavenResolver().resolve(uri);
-                                break;
+                    if (isResolveAllowed(uri)) {
+                        MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
+                        final List<MavenArtifact> searchResults = manager.search(customNexusServer.getServer(),
+                                customNexusServer.getUserName(), customNexusServer.getPassword(),
+                                customNexusServer.getRepositoryId(), artifact.getGroupId(), artifact.getArtifactId(),
+                                artifact.getVersion());
+                        if (!searchResults.isEmpty()) {
+                            File resolvedFile = null;
+                            for (MavenArtifact result : searchResults) {
+                                if (jarNeeded.equals(result.getArtifactId() + "." + result.getType())) {
+                                    resolvedFile = manager.getMavenResolver().resolve(uri);
+                                    break;
+                                }
+                            }
+                            if (resolvedFile != null) {
+                                // reset module status
+                                final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
+                                statusMap.put(uri, ELibraryInstallStatus.INSTALLED);
+                                jarFile = resolvedFile;
+                                // update installed path
+                                mavenJarInstalled.put(uri, jarFile.getAbsolutePath());
                             }
                         }
-                        if (resolvedFile != null) {
-                            // reset module status
-                            final Map<String, ELibraryInstallStatus> statusMap = ModuleStatusProvider.getStatusMap();
-                            statusMap.put(uri, ELibraryInstallStatus.INSTALLED);
-                            jarFile = resolvedFile;
-                            // update installed path
-                            mavenJarInstalled.put(uri, jarFile.getAbsolutePath());
-                        }
+                        updateLastResolveDate(uri);
                     }
                 }
             }
@@ -420,6 +436,99 @@ public class LocalLibraryManager implements ILibraryManagerService {
 
     }
 
+    private Map<String, Date> lastResolveDate;
+
+    private String LAST_UPDATE_KEY = "lastUpdate"; //$NON-NLS-1$
+
+    public long daysBetween(final Calendar startDate, final Calendar endDate) {
+        // assert: startDate must be before endDate
+        int MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
+        long endInstant = endDate.getTimeInMillis();
+        int presumedDays = (int) ((endInstant - startDate.getTimeInMillis()) / MILLIS_IN_DAY);
+        Calendar cursor = (Calendar) startDate.clone();
+        cursor.add(Calendar.DAY_OF_YEAR, presumedDays);
+        long instant = cursor.getTimeInMillis();
+        if (instant == endInstant)
+            return presumedDays;
+
+        final int step = instant < endInstant ? 1 : -1;
+        do {
+            cursor.add(Calendar.DAY_OF_MONTH, step);
+            presumedDays += step;
+        } while (cursor.getTimeInMillis() <= endInstant);
+        return presumedDays -1;
+    }
+
+    /**
+     * DOC nrousseau Comment method "isResolveAllowed".
+     * 
+     * @param uri
+     * @return
+     */
+    public boolean isResolveAllowed(String uri) {
+        IEclipsePreferences node = InstanceScope.INSTANCE.getNode(NexusServerUtils.ORG_TALEND_DESIGNER_CORE);
+        int refreshTime = node.getInt(ITalendCorePrefConstants.NEXUS_REFRESH_FREQUENCY, 0);
+        if (refreshTime == 0) {
+            return true;
+        }
+        if (refreshTime == -1) {
+            return false;
+        }
+
+        if (lastResolveDate == null) {
+            lastResolveDate = new HashMap<String, Date>();
+            IEclipsePreferences prefSetting = ConfigurationScope.INSTANCE.getNode("org.talend.librariesmanager");
+            String lastUpdate = prefSetting.get(LAST_UPDATE_KEY, null);
+            if (lastUpdate != null) {
+                byte[] lastUpdateStream = DatatypeConverter.parseHexBinary(lastUpdate);
+                ByteArrayInputStream bais = new ByteArrayInputStream(lastUpdateStream);
+                ObjectInputStream ois;
+                try {
+                    ois = new ObjectInputStream(bais);
+                    lastResolveDate = (HashMap) ois.readObject();
+                    ois.close();
+                    bais.close();
+                } catch (Exception e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        }
+        Date lastDate = lastResolveDate.get(uri);
+        if (lastDate == null) {
+            return true;
+        }
+        Calendar before = Calendar.getInstance();
+        before.setTime(lastDate);
+        Calendar after = Calendar.getInstance();
+        after.setTime(new Date());
+        long days = daysBetween(before, after);
+        if (days >= refreshTime) {
+            return true;
+        }
+        return false;
+    }
+
+    public void updateLastResolveDate(String uri) {
+        if (lastResolveDate == null) {
+            lastResolveDate = new HashMap<String, Date>();
+        }
+        lastResolveDate.put(uri, new Date());
+        IEclipsePreferences prefSetting = ConfigurationScope.INSTANCE.getNode("org.talend.librariesmanager");
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos;
+        try {
+            oos = new ObjectOutputStream(baos);
+            oos.writeObject(lastResolveDate);
+            oos.close();
+            String lastUpdate = DatatypeConverter.printHexBinary(baos.toByteArray());
+            prefSetting.put(LAST_UPDATE_KEY, lastUpdate);
+            prefSetting.flush();
+            baos.close();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public boolean retrieve(Collection<String> jarsNeeded, String pathToStore, boolean showDialog,
             IProgressMonitor... monitorWrap) {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/DqRepositoryViewService.java

@@ -88,7 +88,8 @@ public final class DqRepositoryViewService {
     private static final String REPLACEMENT_CHARS = "_"; //$NON-NLS-1$
 
     public static final String[] TABLE_TYPES = new String[] { ETableTypes.TABLETYPE_TABLE.getName(),
-            ETableTypes.EXTERNAL_TABLE.getName(), ETableTypes.MANAGED_TABLE.getName(), ETableTypes.INDEX_TABLE.getName() };
+            ETableTypes.EXTERNAL_TABLE.getName(), ETableTypes.MANAGED_TABLE.getName(), ETableTypes.INDEX_TABLE.getName(),
+            ETableTypes.SYSTEM_TABLE.getName() };
 
     public static final String[] VIEW_TYPES = new String[] { ETableTypes.TABLETYPE_VIEW.getName(),
             ETableTypes.VIRTUAL_VIEW.getName() };

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataFromDataBase.java

@@ -92,7 +92,8 @@ public class ExtractMetaDataFromDataBase {
         EXTERNAL_TABLE("EXTERNAL_TABLE"), //$NON-NLS-1$ // for hive
         MANAGED_TABLE("MANAGED_TABLE"), //$NON-NLS-1$ // for hive
         INDEX_TABLE("INDEX_TABLE"), //$NON-NLS-1$ // for hive
-        VIRTUAL_VIEW("VIRTUAL_VIEW"); //$NON-NLS-1$ // for hive
+        VIRTUAL_VIEW("VIRTUAL_VIEW"), //$NON-NLS-1$ // for hive
+        SYSTEM_TABLE("SYSTEM TABLE"); //$NON-NLS-1$ //added for amazon redshift TDQ-11558 by msjian 
 
         private final String name;
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/dburl/SupportDBUrlStore.java

@@ -95,6 +95,7 @@ public final class SupportDBUrlStore {
         supportDBUrlMap.put(SupportDBUrlType.VERTICA.getDBKey(), SupportDBUrlType.VERTICA);
         supportDBUrlMap.put(SupportDBUrlType.VERTICA2.getDBKey(), SupportDBUrlType.VERTICA2);
         supportDBUrlMap.put(SupportDBUrlType.IMPALA.getDBKey(), SupportDBUrlType.IMPALA);
+        supportDBUrlMap.put(SupportDBUrlType.REDSHIFT.getDBKey(), SupportDBUrlType.REDSHIFT);
 
         // MOD mzhao bug 12313, 2010-04-02 There is not dbType in prv files before 4.0 release, here use driver class
         // name
@@ -130,6 +131,7 @@ public final class SupportDBUrlStore {
         supportDiverNameDBUrlMap.put(SupportDBUrlType.HIVEDEFAULTURL.getDbDriver(), SupportDBUrlType.HIVEDEFAULTURL);
         supportDiverNameDBUrlMap.put(SupportDBUrlType.VERTICA.getDbDriver(), SupportDBUrlType.VERTICA);
         supportDiverNameDBUrlMap.put(SupportDBUrlType.VERTICA2.getDbDriver(), SupportDBUrlType.VERTICA2);
+        supportDiverNameDBUrlMap.put(SupportDBUrlType.REDSHIFT.getDbDriver(), SupportDBUrlType.REDSHIFT);
         // ~12313
     }
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/dburl/SupportDBUrlType.java

@@ -170,7 +170,7 @@ public enum SupportDBUrlType {
               null,
               "org.exist.xmldb.DatabaseImpl", //$NON-NLS-1$
               null,
-              "eXist"),
+              "eXist"), //$NON-NLS-1$
     MDM("MDM", //$NON-NLS-1$
         "localhost", //$NON-NLS-1$
         "8080", //$NON-NLS-1$
@@ -178,7 +178,7 @@ public enum SupportDBUrlType {
         null,
         "", //$NON-NLS-1$
         null,
-        "MDM"),
+        "MDM"), //$NON-NLS-1$
     DELIMITEDFILE("DELIMITEDFILE", //$NON-NLS-1$
                   "", //$NON-NLS-1$
                   "", //$NON-NLS-1$
@@ -186,8 +186,8 @@ public enum SupportDBUrlType {
                   null,
                   "", //$NON-NLS-1$
                   null,
-                  "DELIMITEDFILE"),
-    JAVADEFAULTURL("Java", "localhost", null, "dbname", null, null, null, "Java"),
+                  "DELIMITEDFILE"), //$NON-NLS-1$
+    JAVADEFAULTURL("Java", "localhost", null, "dbname", null, null, null, "Java"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
     ACCESS("ACCESS", //$NON-NLS-1$
            null,
            null,
@@ -214,12 +214,12 @@ public enum SupportDBUrlType {
            "Impala"), //$NON-NLS-1$
     REDSHIFT("Redshift", //$NON-NLS-1$
              "localhost", //$NON-NLS-1$
-             "10000", //$NON-NLS-1$
+             "5439", //$NON-NLS-1$
              null,
              null,
-             "", //$NON-NLS-1$
+             "com.paraccel.Driver", //$NON-NLS-1$
              null,
-             "Redshift"), //$NON-NLS-1$
+             "ParAccel"), //$NON-NLS-1$
     VERTICA("Vertica", //$NON-NLS-1$
             null,
             "5433", //$NON-NLS-1$

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/utils/MetadataConnectionUtils.java

@@ -647,7 +647,6 @@ public class MetadataConnectionUtils {
      * @return
      */
     public static List<String> getTDQSupportDBTemplate() {
-
         try {
             if (GlobalServiceRegister.getDefault().isDQDriverServiceRegistered(IDriverService.class)) {
                 IDriverService driverService = (IDriverService) GlobalServiceRegister.getDefault().getDQDriverService(
@@ -670,17 +669,9 @@ public class MetadataConnectionUtils {
         if (conn == null) {
             return false;
         }
-        try {
-            if (GlobalServiceRegister.getDefault().isDQDriverServiceRegistered(IDriverService.class)) {
-                IDriverService driverService = (IDriverService) GlobalServiceRegister.getDefault().getDQDriverService(
-                        IDriverService.class);
-                if (conn instanceof DatabaseConnection) {
-                    String databaseType = ((DatabaseConnection) conn).getDatabaseType();
-                    return driverService.getTDQSupportDBTemplate().contains(databaseType);
-                }
-            }
-        } catch (Exception e) {
-            log.error(e, e);
+        if (conn instanceof DatabaseConnection) {
+            String databaseType = ((DatabaseConnection) conn).getDatabaseType();
+            return getTDQSupportDBTemplate().contains(databaseType);
         }
 
         return false;

main/plugins/org.talend.model/model/TalendFile.ecore

@@ -956,6 +956,20 @@
         eType="#//ScreenshotsMap" transient="true" containment="true"/>
     <eStructuralFeatures xsi:type="ecore:EReference" name="routinesDependencies" upperBound="-1"
         eType="#//ItemInforType" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="jobType" eType="ecore:EDataType http://www.eclipse.org/emf/2003/XMLType#//String">
+      <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+        <details key="kind" value="attribute"/>
+        <details key="name" value="name"/>
+        <details key="namespace" value="##targetNamespace"/>
+      </eAnnotations>
+    </eStructuralFeatures>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="framework" eType="ecore:EDataType http://www.eclipse.org/emf/2003/XMLType#//String">
+      <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+        <details key="kind" value="attribute"/>
+        <details key="name" value="name"/>
+        <details key="namespace" value="##targetNamespace"/>
+      </eAnnotations>
+    </eStructuralFeatures>
   </eClassifiers>
   <eClassifiers xsi:type="ecore:EClass" name="RequiredType">
     <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">

main/plugins/org.talend.model/model/TalendFile.genmodel

@@ -160,6 +160,8 @@
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//ProcessType/screenshot"/>
       <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference TalendFile.ecore#//ProcessType/screenshots"/>
       <genFeatures notify="false" createChild="false" propertySortChoices="true" ecoreFeature="ecore:EReference TalendFile.ecore#//ProcessType/routinesDependencies"/>
+      <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//ProcessType/jobType"/>
+      <genFeatures createChild="false" ecoreFeature="ecore:EAttribute TalendFile.ecore#//ProcessType/framework"/>
     </genClasses>
     <genClasses ecoreClass="TalendFile.ecore#//RequiredType">
       <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference TalendFile.ecore#//RequiredType/job"/>

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/ProcessType.java

@@ -39,6 +39,8 @@ import org.eclipse.emf.ecore.EObject;
  *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getScreenshot <em>Screenshot</em>}</li>
  *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getScreenshots <em>Screenshots</em>}</li>
  *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getRoutinesDependencies <em>Routines Dependencies</em>}</li>
+ *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getJobType <em>Job Type</em>}</li>
+ *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getFramework <em>Framework</em>}</li>
  * </ul>
  * </p>
  *
@@ -272,6 +274,60 @@ public interface ProcessType extends EObject {
      */
     EList getRoutinesDependencies();
 
+    /**
+     * Returns the value of the '<em><b>Job Type</b></em>' attribute.
+     * <!-- begin-user-doc -->
+     * <p>
+     * If the meaning of the '<em>Job Type</em>' attribute isn't clear,
+     * there really should be more of a description here...
+     * </p>
+     * <!-- end-user-doc -->
+     * @return the value of the '<em>Job Type</em>' attribute.
+     * @see #setJobType(String)
+     * @see org.talend.designer.core.model.utils.emf.talendfile.TalendFilePackage#getProcessType_JobType()
+     * @model dataType="org.eclipse.emf.ecore.xml.type.String"
+     *        extendedMetaData="kind='attribute' name='name' namespace='##targetNamespace'"
+     * @generated
+     */
+    String getJobType();
+
+    /**
+     * Sets the value of the '{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getJobType <em>Job Type</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @param value the new value of the '<em>Job Type</em>' attribute.
+     * @see #getJobType()
+     * @generated
+     */
+    void setJobType(String value);
+
+    /**
+     * Returns the value of the '<em><b>Framework</b></em>' attribute.
+     * <!-- begin-user-doc -->
+     * <p>
+     * If the meaning of the '<em>Framework</em>' attribute isn't clear,
+     * there really should be more of a description here...
+     * </p>
+     * <!-- end-user-doc -->
+     * @return the value of the '<em>Framework</em>' attribute.
+     * @see #setFramework(String)
+     * @see org.talend.designer.core.model.utils.emf.talendfile.TalendFilePackage#getProcessType_Framework()
+     * @model dataType="org.eclipse.emf.ecore.xml.type.String"
+     *        extendedMetaData="kind='attribute' name='name' namespace='##targetNamespace'"
+     * @generated
+     */
+    String getFramework();
+
+    /**
+     * Sets the value of the '{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getFramework <em>Framework</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @param value the new value of the '<em>Framework</em>' attribute.
+     * @see #getFramework()
+     * @generated
+     */
+    void setFramework(String value);
+
     /**
      * Returns the value of the '<em><b>Logs</b></em>' containment reference.
      * <!-- begin-user-doc -->

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/TalendFilePackage.java

@@ -1472,6 +1472,24 @@ public interface TalendFilePackage extends EPackage {
      */
     int PROCESS_TYPE__ROUTINES_DEPENDENCIES = 19;
 
+    /**
+     * The feature id for the '<em><b>Job Type</b></em>' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     * @ordered
+     */
+    int PROCESS_TYPE__JOB_TYPE = 20;
+
+    /**
+     * The feature id for the '<em><b>Framework</b></em>' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     * @ordered
+     */
+    int PROCESS_TYPE__FRAMEWORK = 21;
+
     /**
      * The number of structural features of the '<em>Process Type</em>' class.
      * <!-- begin-user-doc -->
@@ -1479,7 +1497,7 @@ public interface TalendFilePackage extends EPackage {
      * @generated
      * @ordered
      */
-    int PROCESS_TYPE_FEATURE_COUNT = 20;
+    int PROCESS_TYPE_FEATURE_COUNT = 22;
 
 
     /**
@@ -3150,6 +3168,28 @@ public interface TalendFilePackage extends EPackage {
      */
     EReference getProcessType_RoutinesDependencies();
 
+    /**
+     * Returns the meta object for the attribute '{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getJobType <em>Job Type</em>}'.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @return the meta object for the attribute '<em>Job Type</em>'.
+     * @see org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getJobType()
+     * @see #getProcessType()
+     * @generated
+     */
+    EAttribute getProcessType_JobType();
+
+    /**
+     * Returns the meta object for the attribute '{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getFramework <em>Framework</em>}'.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @return the meta object for the attribute '<em>Framework</em>'.
+     * @see org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getFramework()
+     * @see #getProcessType()
+     * @generated
+     */
+    EAttribute getProcessType_Framework();
+
     /**
      * Returns the meta object for the containment reference '{@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType#getLogs <em>Logs</em>}'.
      * <!-- begin-user-doc -->
@@ -4550,6 +4590,22 @@ public interface TalendFilePackage extends EPackage {
          */
         EReference PROCESS_TYPE__ROUTINES_DEPENDENCIES = eINSTANCE.getProcessType_RoutinesDependencies();
 
+        /**
+         * The meta object literal for the '<em><b>Job Type</b></em>' attribute feature.
+         * <!-- begin-user-doc -->
+         * <!-- end-user-doc -->
+         * @generated
+         */
+        EAttribute PROCESS_TYPE__JOB_TYPE = eINSTANCE.getProcessType_JobType();
+
+        /**
+         * The meta object literal for the '<em><b>Framework</b></em>' attribute feature.
+         * <!-- begin-user-doc -->
+         * <!-- end-user-doc -->
+         * @generated
+         */
+        EAttribute PROCESS_TYPE__FRAMEWORK = eINSTANCE.getProcessType_Framework();
+
         /**
          * The meta object literal for the '<em><b>Logs</b></em>' containment reference feature.
          * <!-- begin-user-doc -->

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/impl/ProcessTypeImpl.java

@@ -60,6 +60,8 @@ import org.talend.designer.core.model.utils.emf.talendfile.TalendFilePackage;
  *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.impl.ProcessTypeImpl#getScreenshot <em>Screenshot</em>}</li>
  *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.impl.ProcessTypeImpl#getScreenshots <em>Screenshots</em>}</li>
  *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.impl.ProcessTypeImpl#getRoutinesDependencies <em>Routines Dependencies</em>}</li>
+ *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.impl.ProcessTypeImpl#getJobType <em>Job Type</em>}</li>
+ *   <li>{@link org.talend.designer.core.model.utils.emf.talendfile.impl.ProcessTypeImpl#getFramework <em>Framework</em>}</li>
  * </ul>
  * </p>
  *
@@ -366,6 +368,46 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
      */
     protected EList routinesDependencies;
 
+    /**
+     * The default value of the '{@link #getJobType() <em>Job Type</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #getJobType()
+     * @generated
+     * @ordered
+     */
+    protected static final String JOB_TYPE_EDEFAULT = null;
+
+    /**
+     * The cached value of the '{@link #getJobType() <em>Job Type</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #getJobType()
+     * @generated
+     * @ordered
+     */
+    protected String jobType = JOB_TYPE_EDEFAULT;
+
+    /**
+     * The default value of the '{@link #getFramework() <em>Framework</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #getFramework()
+     * @generated
+     * @ordered
+     */
+    protected static final String FRAMEWORK_EDEFAULT = null;
+
+    /**
+     * The cached value of the '{@link #getFramework() <em>Framework</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #getFramework()
+     * @generated
+     * @ordered
+     */
+    protected String framework = FRAMEWORK_EDEFAULT;
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -584,6 +626,48 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
         return routinesDependencies;
     }
 
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public String getJobType() {
+        return jobType;
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public void setJobType(String newJobType) {
+        String oldJobType = jobType;
+        jobType = newJobType;
+        if (eNotificationRequired())
+            eNotify(new ENotificationImpl(this, Notification.SET, TalendFilePackage.PROCESS_TYPE__JOB_TYPE, oldJobType, jobType));
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public String getFramework() {
+        return framework;
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public void setFramework(String newFramework) {
+        String oldFramework = framework;
+        framework = newFramework;
+        if (eNotificationRequired())
+            eNotify(new ENotificationImpl(this, Notification.SET, TalendFilePackage.PROCESS_TYPE__FRAMEWORK, oldFramework, framework));
+    }
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -853,6 +937,10 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
                 else return getScreenshots().map();
             case TalendFilePackage.PROCESS_TYPE__ROUTINES_DEPENDENCIES:
                 return getRoutinesDependencies();
+            case TalendFilePackage.PROCESS_TYPE__JOB_TYPE:
+                return getJobType();
+            case TalendFilePackage.PROCESS_TYPE__FRAMEWORK:
+                return getFramework();
         }
         return super.eGet(featureID, resolve, coreType);
     }
@@ -929,6 +1017,12 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
                 getRoutinesDependencies().clear();
                 getRoutinesDependencies().addAll((Collection)newValue);
                 return;
+            case TalendFilePackage.PROCESS_TYPE__JOB_TYPE:
+                setJobType((String)newValue);
+                return;
+            case TalendFilePackage.PROCESS_TYPE__FRAMEWORK:
+                setFramework((String)newValue);
+                return;
         }
         super.eSet(featureID, newValue);
     }
@@ -999,6 +1093,12 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
             case TalendFilePackage.PROCESS_TYPE__ROUTINES_DEPENDENCIES:
                 getRoutinesDependencies().clear();
                 return;
+            case TalendFilePackage.PROCESS_TYPE__JOB_TYPE:
+                setJobType(JOB_TYPE_EDEFAULT);
+                return;
+            case TalendFilePackage.PROCESS_TYPE__FRAMEWORK:
+                setFramework(FRAMEWORK_EDEFAULT);
+                return;
         }
         super.eUnset(featureID);
     }
@@ -1049,6 +1149,10 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
                 return screenshots != null && !screenshots.isEmpty();
             case TalendFilePackage.PROCESS_TYPE__ROUTINES_DEPENDENCIES:
                 return routinesDependencies != null && !routinesDependencies.isEmpty();
+            case TalendFilePackage.PROCESS_TYPE__JOB_TYPE:
+                return JOB_TYPE_EDEFAULT == null ? jobType != null : !JOB_TYPE_EDEFAULT.equals(jobType);
+            case TalendFilePackage.PROCESS_TYPE__FRAMEWORK:
+                return FRAMEWORK_EDEFAULT == null ? framework != null : !FRAMEWORK_EDEFAULT.equals(framework);
         }
         return super.eIsSet(featureID);
     }
@@ -1081,6 +1185,10 @@ public class ProcessTypeImpl extends EObjectImpl implements ProcessType {
         result.append(version);
         result.append(", screenshot: ");
         result.append(screenshot);
+        result.append(", jobType: ");
+        result.append(jobType);
+        result.append(", framework: ");
+        result.append(framework);
         result.append(')');
         return result.toString();
     }

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/impl/TalendFilePackageImpl.java

@@ -1308,6 +1308,24 @@ public class TalendFilePackageImpl extends EPackageImpl implements TalendFilePac
         return (EReference)processTypeEClass.getEStructuralFeatures().get(19);
     }
 
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public EAttribute getProcessType_JobType() {
+        return (EAttribute)processTypeEClass.getEStructuralFeatures().get(20);
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public EAttribute getProcessType_Framework() {
+        return (EAttribute)processTypeEClass.getEStructuralFeatures().get(21);
+    }
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -1720,6 +1738,8 @@ public class TalendFilePackageImpl extends EPackageImpl implements TalendFilePac
         createEAttribute(processTypeEClass, PROCESS_TYPE__SCREENSHOT);
         createEReference(processTypeEClass, PROCESS_TYPE__SCREENSHOTS);
         createEReference(processTypeEClass, PROCESS_TYPE__ROUTINES_DEPENDENCIES);
+        createEAttribute(processTypeEClass, PROCESS_TYPE__JOB_TYPE);
+        createEAttribute(processTypeEClass, PROCESS_TYPE__FRAMEWORK);
 
         requiredTypeEClass = createEClass(REQUIRED_TYPE);
         createEReference(requiredTypeEClass, REQUIRED_TYPE__JOB);
@@ -1931,6 +1951,8 @@ public class TalendFilePackageImpl extends EPackageImpl implements TalendFilePac
         initEAttribute(getProcessType_Screenshot(), theXMLTypePackage.getBase64Binary(), "screenshot", null, 0, 1, ProcessType.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, !IS_UNSETTABLE, !IS_ID, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
         initEReference(getProcessType_Screenshots(), this.getScreenshotsMap(), null, "screenshots", null, 0, -1, ProcessType.class, IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, IS_COMPOSITE, !IS_RESOLVE_PROXIES, !IS_UNSETTABLE, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
         initEReference(getProcessType_RoutinesDependencies(), this.getItemInforType(), null, "routinesDependencies", null, 0, -1, ProcessType.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, IS_COMPOSITE, !IS_RESOLVE_PROXIES, !IS_UNSETTABLE, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
+        initEAttribute(getProcessType_JobType(), theXMLTypePackage.getString(), "jobType", null, 0, 1, ProcessType.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, !IS_UNSETTABLE, !IS_ID, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
+        initEAttribute(getProcessType_Framework(), theXMLTypePackage.getString(), "framework", null, 0, 1, ProcessType.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, !IS_UNSETTABLE, !IS_ID, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
 
         initEClass(requiredTypeEClass, RequiredType.class, "RequiredType", !IS_ABSTRACT, !IS_INTERFACE, IS_GENERATED_INSTANCE_CLASS);
         initEReference(getRequiredType_Job(), this.getJobType(), null, "job", null, 0, -1, RequiredType.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, IS_COMPOSITE, !IS_RESOLVE_PROXIES, !IS_UNSETTABLE, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
@@ -2965,6 +2987,22 @@ public class TalendFilePackageImpl extends EPackageImpl implements TalendFilePac
              "name", "Subjob",
              "namespace", "##targetNamespace"
            });	
+        addAnnotation
+          (getProcessType_JobType(), 
+           source, 
+           new String[] {
+             "kind", "attribute",
+             "name", "name",
+             "namespace", "##targetNamespace"
+           });	
+        addAnnotation
+          (getProcessType_Framework(), 
+           source, 
+           new String[] {
+             "kind", "attribute",
+             "name", "name",
+             "namespace", "##targetNamespace"
+           });	
         addAnnotation
           (requiredTypeEClass, 
            source, 

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/AbstractExternalDataValidator.java

@@ -0,0 +1,17 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.AbstractExternalData}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface AbstractExternalDataValidator {
+    boolean validate();
+
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/AdditionalFieldMapValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link java.util.Map.Entry}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface AdditionalFieldMapValidator {
+    boolean validate();
+
+    boolean validateTypedKey(String value);
+    boolean validateTypedValue(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ColumnTypeValidator.java

@@ -0,0 +1,34 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EMap;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ColumnType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ColumnTypeValidator {
+    boolean validate();
+
+    boolean validateComment(String value);
+    boolean validateDefaultValue(String value);
+    boolean validateKey(boolean value);
+    boolean validateLength(int value);
+    boolean validateName(String value);
+    boolean validateNullable(boolean value);
+    boolean validateOriginalDbColumnName(String value);
+    boolean validatePattern(String value);
+    boolean validatePrecision(int value);
+    boolean validateSourceType(String value);
+    boolean validateType(String value);
+    boolean validateRelatedEntity(String value);
+    boolean validateRelationshipType(String value);
+    boolean validateOriginalLength(int value);
+    boolean validateAdditionalField(EMap value);
+    boolean validateUsefulColumn(boolean value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ConnectionTypeValidator.java

@@ -0,0 +1,29 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ConnectionType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ConnectionTypeValidator {
+    boolean validate();
+
+    boolean validateElementParameter(EList value);
+    boolean validateConnectorName(String value);
+    boolean validateLabel(String value);
+    boolean validateLineStyle(int value);
+    boolean validateMergeOrder(int value);
+    boolean validateMetaname(String value);
+    boolean validateOffsetLabelX(int value);
+    boolean validateOffsetLabelY(int value);
+    boolean validateOutputId(int value);
+    boolean validateSource(String value);
+    boolean validateTarget(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ContextParameterTypeValidator.java

@@ -0,0 +1,24 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ContextParameterTypeValidator {
+    boolean validate();
+
+    boolean validateComment(String value);
+    boolean validateName(String value);
+    boolean validatePrompt(String value);
+    boolean validatePromptNeeded(boolean value);
+    boolean validateRepositoryContextId(String value);
+    boolean validateType(String value);
+    boolean validateValue(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ContextTypeValidator.java

@@ -0,0 +1,21 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ContextType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ContextTypeValidator {
+    boolean validate();
+
+    boolean validateContextParameter(EList value);
+    boolean validateConfirmationNeeded(boolean value);
+    boolean validateName(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/DocumentRootValidator.java

@@ -0,0 +1,42 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EMap;
+
+import org.eclipse.emf.ecore.util.FeatureMap;
+
+import org.talend.designer.core.model.utils.emf.talendfile.ConnectionType;
+import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
+import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
+import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
+import org.talend.designer.core.model.utils.emf.talendfile.NoteType;
+import org.talend.designer.core.model.utils.emf.talendfile.ParametersType;
+import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
+import org.talend.designer.core.model.utils.emf.talendfile.RequiredType;
+import org.talend.designer.core.model.utils.emf.talendfile.SubjobType;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.DocumentRoot}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface DocumentRootValidator {
+    boolean validate();
+
+    boolean validateMixed(FeatureMap value);
+    boolean validateXMLNSPrefixMap(EMap value);
+    boolean validateXSISchemaLocation(EMap value);
+    boolean validateConnection(ConnectionType value);
+    boolean validateContext(ContextType value);
+    boolean validateElementParameter(ElementParameterType value);
+    boolean validateNode(NodeType value);
+    boolean validateNote(NoteType value);
+    boolean validateParameters(ParametersType value);
+    boolean validateProcess(ProcessType value);
+    boolean validateRequired(RequiredType value);
+    boolean validateSubjob(SubjobType value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ElementParameterTypeValidator.java

@@ -0,0 +1,24 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ElementParameterTypeValidator {
+    boolean validate();
+
+    boolean validateElementValue(EList value);
+    boolean validateField(String value);
+    boolean validateName(String value);
+    boolean validateValue(String value);
+    boolean validateContextMode(boolean value);
+    boolean validateShow(boolean value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ElementValueTypeValidator.java

@@ -0,0 +1,20 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ElementValueType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ElementValueTypeValidator {
+    boolean validate();
+
+    boolean validateElementRef(String value);
+    boolean validateValue(String value);
+    boolean validateType(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ItemInforTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ItemInforType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ItemInforTypeValidator {
+    boolean validate();
+
+    boolean validateIdOrName(String value);
+    boolean validateSystem(boolean value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/JobTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.JobType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface JobTypeValidator {
+    boolean validate();
+
+    boolean validateContext(String value);
+    boolean validateName(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/LogToDatabaseTypeValidator.java

@@ -0,0 +1,20 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.LogToDatabaseType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface LogToDatabaseTypeValidator {
+    boolean validate();
+
+    boolean validateDatabase(String value);
+    boolean validateLevel(int value);
+    boolean validateSelected(boolean value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/LogToFileTypeValidator.java

@@ -0,0 +1,20 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.LogToFileType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface LogToFileTypeValidator {
+    boolean validate();
+
+    boolean validateFilename(String value);
+    boolean validateLevel(int value);
+    boolean validateSelected(boolean value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/LogToStdOutTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.LogToStdOutType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface LogToStdOutTypeValidator {
+    boolean validate();
+
+    boolean validateLevel(int value);
+    boolean validateSelected(boolean value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/LogsTypeValidator.java

@@ -0,0 +1,23 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.talend.designer.core.model.utils.emf.talendfile.LogToDatabaseType;
+import org.talend.designer.core.model.utils.emf.talendfile.LogToFileType;
+import org.talend.designer.core.model.utils.emf.talendfile.LogToStdOutType;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.LogsType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface LogsTypeValidator {
+    boolean validate();
+
+    boolean validateLogToFile(LogToFileType value);
+    boolean validateLogToDatabase(LogToDatabaseType value);
+    boolean validateLogToStdOut(LogToStdOutType value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/MetadataTypeValidator.java

@@ -0,0 +1,25 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.MetadataType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface MetadataTypeValidator {
+    boolean validate();
+
+    boolean validateColumn(EList value);
+    boolean validateComment(String value);
+    boolean validateConnector(String value);
+    boolean validateLabel(String value);
+    boolean validateName(String value);
+    boolean validateSource(String value);
+    boolean validateType(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/NodeContainerTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.NodeContainerType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface NodeContainerTypeValidator {
+    boolean validate();
+
+    boolean validateElementParameter(EList value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/NodeTypeValidator.java

@@ -0,0 +1,36 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+import org.talend.designer.core.model.utils.emf.talendfile.AbstractExternalData;
+import org.talend.designer.core.model.utils.emf.talendfile.NodeContainerType;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.NodeType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface NodeTypeValidator {
+    boolean validate();
+
+    boolean validateElementParameter(EList value);
+    boolean validateMetadata(EList value);
+    boolean validateBinaryData(byte[] value);
+    boolean validateStringData(String value);
+    boolean validateComponentName(String value);
+    boolean validateComponentVersion(String value);
+    boolean validateOffsetLabelX(int value);
+    boolean validateOffsetLabelY(int value);
+    boolean validatePosX(int value);
+    boolean validatePosY(int value);
+    boolean validateSizeX(int value);
+    boolean validateSizeY(int value);
+    boolean validateScreenshot(byte[] value);
+    boolean validateNodeData(AbstractExternalData value);
+    boolean validateNodeContainer(NodeContainerType value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/NoteTypeValidator.java

@@ -0,0 +1,25 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.NoteType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface NoteTypeValidator {
+    boolean validate();
+
+    boolean validateOpaque(boolean value);
+    boolean validatePosX(int value);
+    boolean validatePosY(int value);
+    boolean validateSizeHeight(int value);
+    boolean validateSizeWidth(int value);
+    boolean validateText(String value);
+    boolean validateElementParameter(EList value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ParametersTypeValidator.java

@@ -0,0 +1,20 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ParametersType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ParametersTypeValidator {
+    boolean validate();
+
+    boolean validateElementParameter(EList value);
+    boolean validateRoutinesParameter(EList value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ProcessTypeValidator.java

@@ -0,0 +1,45 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+import org.eclipse.emf.common.util.EMap;
+
+import org.talend.designer.core.model.utils.emf.talendfile.LogsType;
+import org.talend.designer.core.model.utils.emf.talendfile.ParametersType;
+import org.talend.designer.core.model.utils.emf.talendfile.RequiredType;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.ProcessType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ProcessTypeValidator {
+    boolean validate();
+
+    boolean validateDescription(String value);
+    boolean validateRequired(RequiredType value);
+    boolean validateContext(EList value);
+    boolean validateParameters(ParametersType value);
+    boolean validateNode(EList value);
+    boolean validateConnection(EList value);
+    boolean validateNote(EList value);
+    boolean validateLogs(LogsType value);
+    boolean validateAuthor(String value);
+    boolean validateComment(String value);
+    boolean validateDefaultContext(String value);
+    boolean validateName(String value);
+    boolean validatePurpose(String value);
+    boolean validateRepositoryContextId(String value);
+    boolean validateStatus(String value);
+    boolean validateVersion(String value);
+    boolean validateSubjob(EList value);
+    boolean validateScreenshot(byte[] value);
+    boolean validateScreenshots(EMap value);
+    boolean validateRoutinesDependencies(EList value);
+    boolean validateJobType(String value);
+    boolean validateFramework(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/RequiredTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.RequiredType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface RequiredTypeValidator {
+    boolean validate();
+
+    boolean validateJob(EList value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/RoutinesParameterTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.RoutinesParameterType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface RoutinesParameterTypeValidator {
+    boolean validate();
+
+    boolean validateId(String value);
+    boolean validateName(String value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/ScreenshotsMapValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+
+/**
+ * A sample validator interface for {@link java.util.Map.Entry}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface ScreenshotsMapValidator {
+    boolean validate();
+
+    boolean validateTypedKey(String value);
+    boolean validateTypedValue(byte[] value);
+}

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/validation/SubjobTypeValidator.java

@@ -0,0 +1,19 @@
+/**
+ *
+ * $Id$
+ */
+package org.talend.designer.core.model.utils.emf.talendfile.validation;
+
+import org.eclipse.emf.common.util.EList;
+
+/**
+ * A sample validator interface for {@link org.talend.designer.core.model.utils.emf.talendfile.SubjobType}.
+ * This doesn't really do anything, and it's not a real EMF artifact.
+ * It was generated by the org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's code generator can be extended.
+ * This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
+ */
+public interface SubjobTypeValidator {
+    boolean validate();
+
+    boolean validateElementParameter(EList value);
+}

main/plugins/org.talend.model/text/description.txt

@@ -1,279 +1,193 @@
-Model component
+Model TalendFile
 
 This model description is not a real EMF artifact. It was generated by the
 org.eclipse.emf.examples.generator.validator plug-in to illustrate how EMF's
 code generator can be extended.
 This can be disabled with -vmargs -Dorg.eclipse.emf.examples.generator.validator=false.
 
-Package component <platform:/resource/org.talend.model/model/Component.xsd>
+Package talendfile <platform:/resource/org.talend.model/model/TalendFile.xsd>
 
-  Class ADVANCEDPARAMETERSType
-    Reference pARAMETER : PARAMETERType<<0..*>>
+  Class ColumnType
+    Attribute comment : String
+    Attribute defaultValue : String
+    Attribute key : Boolean
+    Attribute length : Int
+    Attribute name : String
+    Attribute nullable : Boolean
+    Attribute originalDbColumnName : String
+    Attribute pattern : String
+    Attribute precision : Int
+    Attribute sourceType : String
+    Attribute type : String
+    Attribute relatedEntity : String
+    Attribute relationshipType : String
+    Attribute originalLength : Int
+    Reference additionalField : AdditionalFieldMap<<0..*>>
+    Attribute usefulColumn : EBoolean
 
-  Class ARGType
+  Class ConnectionType
+    Reference elementParameter : ElementParameterType<<0..*>>
+    Attribute connectorName : String
+    Attribute label : String
+    Attribute lineStyle : Int
+    Attribute mergeOrder : Int
+    Attribute metaname : String
+    Attribute offsetLabelX : Int
+    Attribute offsetLabelY : Int
+    Attribute outputId : Int
+    Attribute source : String
+    Attribute target : String
+
+  Class ContextParameterType
+    Attribute comment : String
+    Attribute name : String
+    Attribute prompt : String
+    Attribute promptNeeded : Boolean
+    Attribute repositoryContextId : String
+    Attribute type : String
     Attribute value : String
 
-  Class CODEGENERATIONType
-    Reference tEMPLATES : TEMPLATESType<<0..*>>
-    Reference iMPORTS : IMPORTSType
-
-  Class COLUMNType
-    Attribute cOMMENT : String
-    Attribute cUSTOM : Boolean
-    Attribute dBTYPE : String
-    Attribute dEFAULT : String
-    Attribute kEY : Boolean
-    Attribute lENGTH : Int
-    Attribute nAME : String
-    Attribute nULLABLE : Boolean
-    Attribute pATTERN : String
-    Attribute pRECISION : Int
-    Attribute rEADONLY : Boolean
-    Attribute rELATEDENTITY : String
-    Attribute rELATIONSHIPTYPE : String
-    Attribute tYPE : String
-
-  Class COMPONENTType
-    Reference hEADER : HEADERType<<1..1>>
-    Reference fAMILIES : FAMILIESType<<1..1>>
-    Reference dOCUMENTATION : DOCUMENTATIONType<<1..1>>
-    Reference cONNECTORS : CONNECTORSType<<1..1>>
-    Reference sQLTEMPLATES : SQLTEMPLATESType
-    Reference pARAMETERS : PARAMETERSType<<1..1>>
-    Reference aDVANCEDPARAMETERS : ADVANCEDPARAMETERSType
-    Reference cODEGENERATION : CODEGENERATIONType<<1..1>>
-    Reference rETURNS : RETURNSType<<1..1>>
-    Reference pLUGINDEPENDENCIES : PLUGINDEPENDENCIESType
-
-  Class CONNECTORSType
-    Reference cONNECTOR : CONNECTORType<<1..*>>
-
-  Class CONNECTORType
-    Attribute bASESCHEMA : String
-    Attribute bUILTIN : Boolean
-    Attribute cOLOR : String
-    Attribute cOMPONENT : String
-    Attribute cTYPE : String<<1..1>>
-    Attribute iNPUTLINKSELECTION : Boolean
-    Attribute lINESTYLE : Int
-    Attribute mAXINPUT : Int
-    Attribute mAXOUTPUT : Int
-    Attribute mERGEALLOWDIFFERENTSCHEMA : Boolean
-    Attribute mININPUT : Int
-    Attribute mINOUTPUT : Int
-    Attribute mULTISCHEMA : Boolean
-    Attribute nAME : String
-    Attribute nOTSHOWIF : String
-    Attribute sHOWIF : String
-
-  Class DEFAULTType
-    Attribute value : String
-    Attribute iF : String
-    Attribute nOTIF : String
-
-  Class DOCUMENTATIONType
-    Attribute uRL : String<<1..1>>
+  Class ContextType
+    Reference contextParameter : ContextParameterType<<0..*>>
+    Attribute confirmationNeeded : Boolean
+    Attribute name : String
 
   Class DocumentRoot
     Attribute mixed : EFeatureMapEntry<<0..*>>
     Reference xMLNSPrefixMap : EStringToStringMapEntry<<0..*>>
     Reference xSISchemaLocation : EStringToStringMapEntry<<0..*>>
-    Reference /aDVANCEDPARAMETERS : ADVANCEDPARAMETERSType<<0..?>>
-    Reference /cODEGENERATION : CODEGENERATIONType<<0..?>>
-    Reference /cOMPONENT : COMPONENTType<<0..?>>
-    Reference /cONNECTORS : CONNECTORSType<<0..?>>
-    Reference /dOCUMENTATION : DOCUMENTATIONType<<0..?>>
-    Reference /fAMILIES : FAMILIESType<<0..?>>
-    Reference /hEADER : HEADERType<<0..?>>
-    Reference /iTEMS : ITEMSType<<0..?>>
-    Reference /pARAMETER : PARAMETERType<<0..?>>
-    Reference /pARAMETERS : PARAMETERSType<<0..?>>
-    Reference /pLUGINDEPENDENCIES : PLUGINDEPENDENCIESType<<0..?>>
-    Reference /rETURNS : RETURNSType<<0..?>>
-    Reference /sQLTEMPLATES : SQLTEMPLATESType<<0..?>>
+    Reference /connection : ConnectionType<<0..?>>
+    Reference /context : ContextType<<0..?>>
+    Reference /elementParameter : ElementParameterType<<0..?>>
+    Reference /node : NodeType<<0..?>>
+    Reference /note : NoteType<<0..?>>
+    Reference /parameters : ParametersType<<0..?>>
+    Reference /process : ProcessType<<0..?>>
+    Reference /required : RequiredType<<0..?>>
+    Reference /subjob : SubjobType<<0..?>>
 
-  Class FAMILIESType
-    Attribute fAMILY : String<<1..*>>
+  Class ElementParameterType
+    Reference elementValue : ElementValueType<<0..*>>
+    Attribute field : String
+    Attribute name : String
+    Attribute value : String
+    Attribute contextMode : Boolean
+    Attribute show : Boolean
 
-  Class FORMATType
-    Attribute cONNECTION : String
-    Attribute hINT : String
-    Attribute lABEL : String
+  Class ElementValueType
+    Attribute elementRef : String
+    Attribute value : String
+    Attribute type : String
 
-  Class HEADERType
-    Attribute sIGNATURE : String<<1..1>>
-    Reference fORMAT : FORMATType
-    Attribute aUTHOR : String<<1..1>>
-    Attribute cOMBINE : String
-    Attribute cOMPATIBILITY : String<<1..1>>
-    Attribute dATAAUTOPROPAGATE : Boolean
-    Attribute eXTENSION : String
-    Attribute hASCONDITIONALOUTPUTS : Boolean
-    Attribute hASHCOMPONENT : Boolean
-    Attribute iSMULTIPLYINGOUTPUTS : Boolean
-    Attribute lOG4J_ENABLED : Boolean
-    Attribute mAINCODECALLED : Boolean
-    Attribute nUMBERPARALLELIZE : String
-    Attribute pARALLELIZE : Boolean
-    Attribute pARTITIONING : EString
-    Attribute pLATEFORM : String<<1..1>>
-    Attribute rEDUCE : EBoolean
-    Attribute rELEASEDATE : String<<1..1>>
-    Attribute sCHEMAAUTOPROPAGATE : Boolean
-    Attribute sERIAL : String<<1..1>>
-    Attribute sHORTNAME : String
-    Attribute sINGLETON : Boolean
-    Attribute sTARTABLE : Boolean<<1..1>>
-    Attribute sTATUS : String<<1..1>>
-    Attribute sUBJOBCOLOR : String
-    Attribute sUBJOBTITLECOLOR : String
-    Attribute sUPPORTS_DB_TYPE : Boolean
-    Attribute tECHNICAL : Boolean
-    Attribute tSTATCATCHERSTATS : Boolean
-    Attribute tYPE : String
-    Attribute vERSION : Decimal<<1..1>>
-    Attribute vISIBLE : Boolean
-    Attribute eQUIVALENT : String
-    Attribute iNPUTTYPE : String
-    Attribute oUTPUTTYPE : String
-    Attribute sPARKACTION : Boolean
+  Class JobType
+    Attribute context : String
+    Attribute name : String
 
-  Class IMPORTSType
-    Reference iMPORT : IMPORTType<<0..*>>
-    Reference iMPORTS : IMPORTSType<<0..*>>
-    Attribute rEQUIREDIF : EString
+  Class LogsType
+    Reference logToFile : LogToFileType<<1..1>>
+    Reference logToDatabase : LogToDatabaseType<<1..1>>
+    Reference logToStdOut : LogToStdOutType<<1..1>>
 
-  Class IMPORTType
-    Reference iNSTALL : INSTALLType<<0..*>>
-    Attribute uRL : String<<0..*>>
-    Attribute bundleID : String
-    Attribute mESSAGE : String
-    Attribute mODULE : String
-    Attribute mODULEGROUP : String
-    Attribute mRREQUIRED : Boolean
-    Attribute nAME : String
-    Attribute rEQUIRED : Boolean
-    Attribute rEQUIREDIF : EString
-    Attribute sHOW : Boolean
-    Attribute urlPath : String
-    Attribute MVN : String
+  Class LogToDatabaseType
+    Attribute database : String
+    Attribute level : Int
+    Attribute selected : Boolean
 
-  Class INSTALLType
-    Attribute cOMMAND : String<<1..1>>
-    Attribute oS : String
+  Class LogToFileType
+    Attribute filename : String
+    Attribute level : Int
+    Attribute selected : Boolean
 
-  Class ITEMSType
-    Reference iTEM : ITEMType<<0..*>>
-    Attribute bASEDONINPUTSCHEMAS : Boolean
-    Attribute bASEDONSCHEMA : Boolean
-    Attribute bASEDONSUBJOBSTARTS : Boolean
-    Attribute cOLUMNSBASEDONSCHEMA : Boolean
-    Attribute dEFAULT : String
+  Class LogToStdOutType
+    Attribute level : Int
+    Attribute selected : Boolean
 
-  Class ITEMType
-    Reference iTEMS : ITEMSType
-    Attribute cONTEXT : String
-    Attribute dISPLAYNAMEASVALUE : Boolean
-    Attribute fIELD : String
-    Attribute fILTER : String
-    Attribute nAME : String<<1..1>>
-    Attribute nOCONTEXTASSIST : Boolean
-    Attribute nOTREADONLYIF : String
-    Attribute nOTSHOWIF : String
-    Attribute rAW : Boolean
-    Attribute rEADONLY : Boolean
-    Attribute rEADONLYIF : String
-    Attribute rEPOSITORYITEM : String
-    Attribute sHOWIF : String
-    Attribute vALUE : String
+  Class MetadataType
+    Reference column : ColumnType<<0..*>>
+    Attribute comment : String
+    Attribute connector : String
+    Attribute label : String
+    Attribute name : String
+    Attribute source : String
+    Attribute type : EString
 
-  Class JAVACOMMANDType
-    Reference aRG : ARGType<<0..*>>
-    Attribute cLASS : String
-    Attribute fUNCTION : String
-    Attribute jAR : String
+  Class NodeType
+    Reference elementParameter : ElementParameterType<<1..*>>
+    Reference metadata : MetadataType<<0..*>>
+    Attribute binaryData : Base64Binary
+    Attribute stringData : String
+    Attribute componentName : String
+    Attribute componentVersion : String
+    Attribute offsetLabelX : Int
+    Attribute offsetLabelY : Int
+    Attribute posX : Int
+    Attribute posY : Int
+    Attribute sizeX : Int
+    Attribute sizeY : Int
+    Attribute screenshot : Base64Binary
+    Reference nodeData : AbstractExternalData
+    Reference nodeContainer : NodeContainerType
 
-  Class LINKTOType
-    Attribute cTYPE : String
-    Attribute nAME : String
+  Class NoteType
+    Attribute opaque : Boolean
+    Attribute posX : Int
+    Attribute posY : Int
+    Attribute sizeHeight : Int
+    Attribute sizeWidth : Int
+    Attribute text : String
+    Reference elementParameter : ElementParameterType<<0..*>>
 
-  Class PARAMETERSType
-    Reference pARAMETER : PARAMETERType<<0..*>>
+  Class ParametersType
+    Reference elementParameter : ElementParameterType<<1..*>>
+    Reference routinesParameter : RoutinesParameterType<<0..*>>
 
-  Class PARAMETERType
-    Reference dEFAULT : DEFAULTType<<0..*>>
-    Reference iTEMS : ITEMSType
-    Reference tABLE : TABLEType<<0..*>>
-    Reference jAVACOMMAND : JAVACOMMANDType
-    Attribute bACKGROUND : String
-    Attribute cOLOR : String
-    Attribute cONTEXT : String
-    Attribute cONTEXTMODE : Boolean
-    Attribute dYNAMICSETTINGS : Boolean
-    Attribute fIELD : String<<1..1>>
-    Attribute fILTER : String
-    Attribute gROUP : String
-    Attribute lOG4J_ENABLED : Boolean
-    Attribute mAXLENGTH : Int
-    Attribute nAME : String<<1..1>>
-    Attribute nBLINES : Int
-    Attribute nOCONTEXTASSIST : Boolean
-    Attribute nOTREADONLYIF : String
-    Attribute nOTSHOWIF : String
-    Attribute nUMROW : Int<<1..1>>
-    Attribute rAW : Boolean
-    Attribute rEADONLY : Boolean
-    Attribute rEADONLYIF : String
-    Attribute rEPOSITORYVALUE : String
-    Attribute rEQUIRED : Boolean
-    Attribute rEQUIREDIF : String
-    Attribute sHOW : Boolean
-    Attribute sHOWIF : String
+  Class ProcessType
+    Attribute description : String
+    Reference required : RequiredType
+    Reference context : ContextType<<0..*>>
+    Reference parameters : ParametersType
+    Reference node : NodeType<<0..*>>
+    Reference connection : ConnectionType<<0..*>>
+    Reference note : NoteType<<0..*>>
+    Reference logs : LogsType
+    Attribute author : String
+    Attribute comment : String
+    Attribute defaultContext : String
+    Attribute name : String
+    Attribute purpose : String
+    Attribute repositoryContextId : String
+    Attribute status : String
+    Attribute version : String
+    Reference subjob : SubjobType<<0..*>>
+    Attribute screenshot : Base64Binary
+    Reference screenshots : ScreenshotsMap<<0..*>>
+    Reference routinesDependencies : ItemInforType<<0..*>>
+    Attribute jobType : String
+    Attribute framework : String
 
-  Class PLUGINDEPENDENCIESType
-    Reference pLUGINDEPENDENCY : PLUGINDEPENDENCYType<<0..*>>
+  Class RequiredType
+    Reference job : JobType<<0..*>>
 
-  Class PLUGINDEPENDENCYType
-    Attribute iD : String<<1..1>>
+  Class SubjobType
+    Reference elementParameter : ElementParameterType<<1..*>>
 
-  Class RETURNSType
-    Reference rETURN : RETURNType<<0..*>>
+  Class ItemInforType
+    Attribute idOrName : EString
+    Attribute system : EBoolean
 
-  Class RETURNType
-    Attribute aVAILABILITY : String<<1..1>>
-    Attribute nAME : String<<1..1>>
-    Attribute sHOWIF : String
-    Attribute tYPE : String<<1..1>>
+  Class AbstractExternalData
 
-  Class SQLTEMPLATESType
-    Reference sQLTEMPLATE : SQLTEMPLATEType<<0..*>>
-    Attribute dB : String
+  Class RoutinesParameterType
+    Attribute id : EString
+    Attribute name : EString
 
-  Class SQLTEMPLATEType
-    Attribute cONTENT : String
-    Attribute nAME : String
+  Class NodeContainerType
+    Reference elementParameter : ElementParameterType<<1..*>>
 
-  Class TABLEType
-    Reference cOLUMN : COLUMNType<<0..*>>
-    Attribute iF : String
-    Attribute nOTIF : String
-    Attribute rEADONLYCOLUMNPOSITION : String
-    Attribute rEADONLY : Boolean
+  Class ScreenshotsMap
+    Attribute key : EString
+    Attribute value : Base64Binary
 
-  Class TEMPLATEPARAMType
-    Attribute sOURCE : String
-    Attribute tARGET : String
-    Attribute vALUE : String
-
-  Class TEMPLATESType
-    Reference tEMPLATE : TEMPLATEType<<0..*>>
-    Reference tEMPLATEPARAM : TEMPLATEPARAMType<<0..*>>
-    Attribute cONNECTOR : String
-    Attribute iNPUT : String
-    Attribute lOOKUP : Boolean
-    Attribute oUTPUT : String
-
-  Class TEMPLATEType
-    Reference lINKTO : LINKTOType<<0..*>>
-    Attribute cOMPONENT : String
-    Attribute nAME : String
+  Class AdditionalFieldMap
+    Attribute key : EString
+    Attribute value : EString

main/plugins/org.talend.rcp/src/main/java/org/talend/rcp/intro/ApplicationWorkbenchAdvisor.java

@@ -15,11 +15,9 @@ package org.talend.rcp.intro;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.log4j.Logger;
 import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.SubMonitor;
-import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.jface.dialogs.TrayDialog;
 import org.eclipse.ui.IPerspectiveDescriptor;
 import org.eclipse.ui.IWorkbenchPreferenceConstants;
@@ -29,11 +27,8 @@ import org.eclipse.ui.application.IWorkbenchWindowConfigurer;
 import org.eclipse.ui.application.WorkbenchWindowAdvisor;
 import org.eclipse.ui.internal.ide.application.IDEWorkbenchAdvisor;
 import org.talend.commons.CommonsPlugin;
-import org.talend.commons.exception.LoginException;
-import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.system.EclipseCommandLine;
 import org.talend.core.GlobalServiceRegister;
-import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.repository.utils.LoginTaskRegistryReader;
 import org.talend.core.ui.branding.IBrandingConfiguration;
 import org.talend.core.ui.branding.IBrandingService;
@@ -42,7 +37,6 @@ import org.talend.designer.runprocess.RunProcessPlugin;
 import org.talend.login.ILoginTask;
 import org.talend.rcp.TalendSplashHandler;
 import org.talend.registration.register.RegisterManagement;
-import org.talend.repository.RepositoryWorkUnit;
 
 /**
  * DOC ccarbone class global comment. Detailled comment <br/>
@@ -136,28 +130,7 @@ public class ApplicationWorkbenchAdvisor extends IDEWorkbenchAdvisor {
         }
 
         // PerspectiveReviewUtil.checkPerspectiveDisplayItems();
-
         CommonsPlugin.setWorkbenchCreated(true);
-        Job myJob = new Job("Remote project update and commit on startup") {
-
-            @Override
-            protected IStatus run(IProgressMonitor monitor) {
-                RepositoryWorkUnit rwu = new RepositoryWorkUnit<Object>("") {
-
-                    @Override
-                    protected void run() throws LoginException, PersistenceException {
-                        // nothing, just commit what has not been commited during the logon time.
-                    }
-                };
-                rwu.setAvoidUnloadResources(true);
-                rwu.setUnloadResourcesAfterRun(true);
-                rwu.setFilesModifiedOutsideOfRWU(true);
-                rwu.setForceTransaction(true);
-                ProxyRepositoryFactory.getInstance().executeRepositoryWorkUnit(rwu);
-                return org.eclipse.core.runtime.Status.OK_STATUS;
-            }
-        };
-        myJob.schedule();
     }
 
 }

main/plugins/org.talend.rcp/src/main/java/org/talend/rcp/intro/ApplicationWorkbenchWindowAdvisor.java

@@ -26,6 +26,9 @@ import java.util.Set;
 import org.eclipse.core.commands.IHandler;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.e4.core.contexts.ContextInjectionFactory;
 import org.eclipse.e4.core.contexts.IEclipseContext;
@@ -67,6 +70,7 @@ import org.eclipse.ui.preferences.ScopedPreferenceStore;
 import org.osgi.service.prefs.BackingStoreException;
 import org.talend.commons.exception.CommonExceptionHandler;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.ui.utils.CheatSheetPerspectiveAdapter;
 import org.talend.commons.utils.VersionUtils;
@@ -105,6 +109,7 @@ import org.talend.rcp.i18n.Messages;
 import org.talend.rcp.intro.starting.StartingEditorInput;
 import org.talend.rcp.intro.starting.StartingHelper;
 import org.talend.rcp.util.ApplicationDeletionUtil;
+import org.talend.repository.RepositoryWorkUnit;
 import org.talend.repository.ui.login.connections.ConnectionUserPerReader;
 import org.talend.repository.ui.views.IRepositoryView;
 
@@ -229,6 +234,33 @@ public class ApplicationWorkbenchWindowAdvisor extends WorkbenchWindowAdvisor {
      */
     @Override
     public void postWindowOpen() {
+        // TDQ-11355 avoid "java.nio.channels.ClosedChannelException" .If the current perspective is DQ, delay this
+        // commit at here,it will be committed with DQ side(see DQRespositoryView Constructor).
+        IWorkbenchPage activePage = PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage();
+        if (activePage != null && !(activePage.getPerspective().getId().equals(ProductUtils.PERSPECTIVE_DQ_ID))) {
+
+            Job myJob = new Job("Remote project update and commit on startup") {
+
+                @Override
+                protected IStatus run(IProgressMonitor monitor) {
+                    RepositoryWorkUnit rwu = new RepositoryWorkUnit<Object>("") {
+
+                        @Override
+                        protected void run() throws LoginException, PersistenceException {
+                            // nothing, just commit what has not been commited during the logon time.
+                        }
+                    };
+                    rwu.setAvoidUnloadResources(true);
+                    rwu.setUnloadResourcesAfterRun(true);
+                    rwu.setFilesModifiedOutsideOfRWU(true);
+                    rwu.setForceTransaction(true);
+                    ProxyRepositoryFactory.getInstance().executeRepositoryWorkUnit(rwu);
+                    return org.eclipse.core.runtime.Status.OK_STATUS;
+                }
+            };
+            myJob.schedule();
+        }
+
         try {
             JavaHomeUtil.initializeJavaHome();
         } catch (CoreException e1) {
@@ -286,8 +318,6 @@ public class ApplicationWorkbenchWindowAdvisor extends WorkbenchWindowAdvisor {
             }
             final File stateFile = path.append("workbench.xml").toFile(); //$NON-NLS-1$
             if (stateFile.exists()) {
-                IWorkbenchWindow workBenchWindow = PlatformUI.getWorkbench().getActiveWorkbenchWindow();
-                IWorkbenchPage activePage = workBenchWindow.getActivePage();
                 FileInputStream input;
                 try {
                     input = new FileInputStream(stateFile);
@@ -374,7 +404,6 @@ public class ApplicationWorkbenchWindowAdvisor extends WorkbenchWindowAdvisor {
         IEclipseContext activeContext = ((IEclipseContext) workbench.getService(IEclipseContext.class)).getActiveLeaf();
 
         ContextInjectionFactory.inject(perspProvider, activeContext);
-        IWorkbenchPage activePage = getWindowConfigurer().getWindow().getWorkbench().getActiveWorkbenchWindow().getActivePage();
         // MOD zshen TDQ-10745 when welcome page is open and current Perspective is DQ will not done
         // restoreAlwaysVisiblePerspectives action because of this method will do switch Perspectives action
         // And switch Perspectives action will cause cheat sheet view maximum display

main/plugins/org.talend.repository.items.importexport.ui/src/main/java/org/talend/repository/items/importexport/ui/wizard/imports/providers/ImportItemsViewerLabelProvider.java

@@ -41,6 +41,10 @@ public class ImportItemsViewerLabelProvider extends LabelProvider {
      */
     @Override
     public Image getImage(Object element) {
+        return getImageBasedOn(element);
+    }
+
+    public static Image getImageBasedOn(Object element) {
         if (element instanceof ImportNode) {
             if (element instanceof ProjectImportNode) {
                 return ImageProvider.getImage(ECoreImage.PROJECT_ICON);
@@ -50,7 +54,7 @@ public class ImportItemsViewerLabelProvider extends LabelProvider {
                 if (itemType == ERepositoryObjectType.PROCESS_MR || itemType == ERepositoryObjectType.PROCESS_STORM) {
                     return ImageProvider.getImage(CoreImageProvider.getIcon(item));
                 }
-                return getImage(itemType);
+                return getImageBasedOn(itemType);
             } else if (element instanceof TypeImportNode) {
                 ERepositoryObjectType repObjectType = ((TypeImportNode) element).getType();
                 // also see RepoViewCommonNavigator.java
@@ -59,14 +63,14 @@ public class ImportItemsViewerLabelProvider extends LabelProvider {
                 } else if (repObjectType == ERepositoryObjectType.PROCESS_MR) {
                     return ImageProvider.getImage(ECoreImage.PROCESS_BATCH_GENERIC_CATEGORY_OPEN_ICON);
                 } else {
-                    return getImage(repObjectType);
+                    return getImageBasedOn(repObjectType);
                 }
             } else if (element instanceof StandardJobImportNode) {
                 ERepositoryObjectType repObjectType = ((StandardJobImportNode) element).getType();
                 if (repObjectType == ERepositoryObjectType.PROCESS) {
                     return ImageProvider.getImage(ECoreImage.PROCESS_STANDARD_GENERIC_CATEGORY_OPEN_ICON);
                 }
-                return getImage(repObjectType);
+                return getImageBasedOn(repObjectType);
             } else if (element instanceof FolderImportNode) {
                 return CoreImageProvider.getImage(ERepositoryObjectType.FOLDER);
             }

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/ImportExportHandlersManager.java

@@ -122,6 +122,11 @@ public class ImportExportHandlersManager {
 
     public List<ImportItem> populateImportingItems(ResourcesManager resManager, boolean overwrite,
             IProgressMonitor progressMonitor, boolean enableProductChecking) throws Exception {
+        return populateImportingItems(resManager, overwrite, progressMonitor, enableProductChecking, true);
+    }
+
+    public List<ImportItem> populateImportingItems(ResourcesManager resManager, boolean overwrite,
+            IProgressMonitor progressMonitor, boolean enableProductChecking, boolean needCheck) throws Exception {
         IProgressMonitor monitor = progressMonitor;
         if (monitor == null) {
             monitor = new NullProgressMonitor();
@@ -169,8 +174,12 @@ public class ImportExportHandlersManager {
                 if (!importHandlerHelper.validResourcePath(path)) { // valid "*.properties" will do it later.
                     IImportItemsHandler importHandler = findValidImportHandler(resManager, path, enableProductChecking);
                     if (importHandler != null) {
-                        ImportItem importItem = importHandler.createImportItem(progressMonitor, resManager, path, overwrite,
-                                items);
+                        ImportItem importItem = null;
+                        if (needCheck) {
+                            importItem = importHandler.generateImportItem(progressMonitor, resManager, path, overwrite, items);
+                        } else {
+                            importItem = importHandler.createImportItem(progressMonitor, resManager, path, overwrite, items);
+                        }
                         // if have existed, won't add again.
                         if (importItem != null && !items.contains(importItem)) {
                             items.add(importItem);
@@ -189,7 +198,7 @@ public class ImportExportHandlersManager {
                 }
                 // process the "*.properties"
                 ImportItem importItem = importHandlerHelper.computeImportItem(monitor, resManager, path, overwrite);
-                if (importItem != null) {
+                if (importItem != null && needCheck) {
                     IImportItemsHandler importHandler = findValidImportHandler(importItem, enableProductChecking);
                     if (importHandler != null) {
                         if (importHandler instanceof ImportBasicHandler) {

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/ImportHandlerHelper.java

@@ -54,8 +54,7 @@ public class ImportHandlerHelper {
         Resource resource = HandlerUtil.loadResource(resManager, importItem);
 
         if (resource != null) {
-            importItem.setProperty((Property) EcoreUtil.getObjectByType(resource.getContents(),
-                    PropertiesPackage.eINSTANCE.getProperty()));
+            importItem.setProperty(generateProperty(resource));
         } else {
             ResourceNotFoundException ex = new ResourceNotFoundException(Messages.getString(
                     "ImportBasicHandler_LoadEMFResourceError", //$NON-NLS-1$
@@ -71,4 +70,7 @@ public class ImportHandlerHelper {
         return ImportCacheHelper.getInstance().getXmiResourceManager().isPropertyFile(resourcePath.lastSegment());
     }
 
+    public Property generateProperty(Resource resource) {
+        return (Property) EcoreUtil.getObjectByType(resource.getContents(), PropertiesPackage.eINSTANCE.getProperty());
+    }
 }

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/imports/IImportItemsHandler.java

@@ -69,6 +69,21 @@ public interface IImportItemsHandler extends IExecutableExtension {
     ImportItem createImportItem(IProgressMonitor monitor, ResourcesManager resManager, IPath resourcePath, boolean overwrite,
             List<ImportItem> existedImportItems) throws Exception;
 
+    /**
+     * 
+     * Almost seem with createImportItem, but this method won't do check
+     * 
+     * @param monitor
+     * @param resManager
+     * @param resourcePath
+     * @param overwrite
+     * @param existedImportItems
+     * @return
+     * @throws Exception
+     */
+    ImportItem generateImportItem(IProgressMonitor monitor, ResourcesManager resManager, IPath resourcePath, boolean overwrite,
+            List<ImportItem> existedImportItems) throws Exception;
+
     /**
      * Find out current item's related items. in order to implictly import the related items.
      * 

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/imports/ImportBasicHandler.java

@@ -234,6 +234,21 @@ public class ImportBasicHandler extends AbstractImportExecutableHandler {
 
     }
 
+    @Override
+    public ImportItem generateImportItem(IProgressMonitor monitor, ResourcesManager resManager, IPath resourcePath,
+            boolean overwrite, List<ImportItem> existedImportItems) throws Exception {
+        ImportItem importItem = computeImportItem(resManager, resourcePath);
+        if (monitor.isCanceled()) {
+            return null;
+        }
+        if (importItem != null && importItem.getProperty() != null) {
+            // set the import handler
+            importItem.setImportHandler(this);
+            return importItem;
+        }
+        return null;
+    }
+
     public ImportItem computeImportItem(ResourcesManager resManager, IPath path) {
         ImportItem importItem = new ImportItem(path);
         computeProperty(resManager, importItem);
@@ -1221,4 +1236,5 @@ public class ImportBasicHandler extends AbstractImportExecutableHandler {
     protected URI getReferenceFileURI(URI propertyResourceURI, ReferenceFileItem refItem) {
         return propertyResourceURI.trimFileExtension().appendFileExtension(refItem.getExtension());
     }
+
 }