chore(TPS-4452): Add release note

This reverts commit cf52e1e004.

Co-authored-by: Dmytro Chmyga <dmytro.chmyga@synapse.com>

PATCH_RELEASE_NOTE.md

@@ -0,0 +1,73 @@
+---
+version: 7.3.1
+module: https://talend.poolparty.biz/coretaxonomy/42
+product:
+- https://talend.poolparty.biz/coretaxonomy/23
+---
+
+# TPS-4452
+
+| Info             | Value |
+| ---------------- | ---------------- |
+| Patch Name       | Patch\_20201005\_TPS-4452\_v1-7.3.1 |
+| Release Date     | 2020-10-05 |
+| Target Version   | 20200219\_1130-V7.3.1 |
+| Product affected | Talend Studio |
+
+## Introduction
+
+This is a self-contained patch.
+
+**NOTE**: For information on how to obtain this patch, reach out to your Support contact at Talend.
+
+## Fixed issues
+
+This patch contains the following fixes:
+
+- TPS-4452 [7.3.1] Job fails with Stream closed exception when executed on Job server (TDI-44962)
+
+## Prerequisites
+
+Consider the following requirements for your system:
+
+- Talend Studio 7.3.1 with 7.3.1-R2020-09 patch must be installed.
+
+
+## Installation
+
+### Installing the patch using Software update
+
+1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
+
+2) Switch to Software update page, where the new patch will be listed. The patch can be downloaded from here into the nexus repository.
+
+3) On Studio Side: Logon Studio with remote mode, on the logon page the Update button is displayed: click this button to install the patch.
+
+### Installing the patch using Talend Studio
+
+1) Create a folder named "patches" under your studio installer directory and copy the patch .zip file to this folder.
+
+2) Restart your studio: a window pops up, then click OK to install the patch, or restart the commandline and the patch will be installed automatically.
+
+### Installing the patch using Commandline
+
+Execute the following commands:
+
+1. Talend-Studio-win-x86_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
+2. initRemote {tac_url} -ul {TAC login username} -up {TAC login password}
+3. checkAndUpdate -tu {TAC login username} -tup {TAC login password}
+
+## Uninstallation
+Backup the Affected files list below. Uninstall the patch by restore the backup files.
+
+## Affected files for this patch
+
+The following files are installed by this patch:
+
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/resources/java/routines/system/ResumeUtil.java
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/routines/system/ResumeUtil$JobLogItem.class
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/routines/system/ResumeUtil$LogPriority.class
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/routines/system/ResumeUtil$ResumeCommonInfo.class
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/routines/system/ResumeUtil$ResumeEventType.class
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/routines/system/ResumeUtil$SimpleCsvWriter.class
+- {Talend\_Studio\_path}/plugins/org.talend.librariesmanager\_7.3.1.20200924\_0610-patch/routines/system/ResumeUtil.class

main/plugins/org.talend.commons.runtime/META-INF/MANIFEST.MF

@@ -16,7 +16,8 @@ Require-Bundle: org.apache.log4j;visibility:=reexport,
  org.talend.utils,
  org.eclipse.core.net,
  org.eclipse.m2e.core,
- org.eclipse.m2e.maven.runtime
+ org.eclipse.m2e.maven.runtime,
+ org.eclipse.core.resources
 Export-Package: org.talend.commons,
  org.talend.commons.exception,
  org.talend.commons.i18n,

main/plugins/org.talend.commons.runtime/src/org/talend/commons/CommonsPlugin.java

@@ -111,7 +111,8 @@ public class CommonsPlugin implements BundleActivator {
     }
 
     public static boolean isDebugMode() {
-        return ArrayUtils.contains(Platform.getApplicationArgs(), TalendDebugHandler.TALEND_DEBUG);
+        return Boolean.getBoolean("talendDebug") //$NON-NLS-1$
+                || ArrayUtils.contains(Platform.getApplicationArgs(), TalendDebugHandler.TALEND_DEBUG);
     }
 
     public static boolean isJUnitTest() {

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/system/EclipseCommandLine.java

@@ -49,11 +49,17 @@ public class EclipseCommandLine {
      */
     static public final String TALEND_PROJECT_TYPE_COMMAND = "-talendProjectType"; //$NON-NLS-1$
 
+    static public final String TALEND_LICENCE_PATH = "talend.licence.path"; //$NON-NLS-1$
+
+    static public final String ARG_TALEND_LICENCE_PATH = "-" + TALEND_LICENCE_PATH; //$NON-NLS-1$
+
     /**
      * for relaunch of the plugins when relaunching the Studio
      */
     static public final String TALEND_RELOAD_COMMAND = "-talendReload"; //$NON-NLS-1$
 
+    static public final String LOGIN_ONLINE_UPDATE = "--loginOnlineUpdate";
+
     static public final String ARG_TALEND_BUNDLES_CLEANED = "-talend.studio.bundles.cleaned"; //$NON-NLS-1$
 
     static public final String PROP_TALEND_BUNDLES_DO_CLEAN = "-talend.studio.bundles.doclean"; //$NON-NLS-1$

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PerformanceLogManager.java

@@ -0,0 +1,127 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.util.Enumeration;
+import java.util.Properties;
+
+import org.apache.log4j.Hierarchy;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.log4j.RollingFileAppender;
+import org.apache.log4j.spi.LoggerFactory;
+import org.apache.log4j.spi.RootLogger;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Platform;
+
+public class PerformanceLogManager {
+
+	private Hierarchy hierarchy;
+	
+	public PerformanceLogManager() {
+	    Properties properties = new Properties();
+	    properties.put("log4j.rootCategory", ", A1");
+	    properties.put("log4j.appender.A1", RollingFileAppender.class.getName());
+	    IPath performanceLogPath = Platform.getLogFileLocation().removeLastSegments(1).append("performance.log");
+	    properties.put("log4j.appender.A1.File", performanceLogPath.toOSString());
+	    properties.put("log4j.appender.A1.MaxBackupIndex", "10");// same as .log's max backup log file count
+	    properties.put("log4j.appender.A1.MaxFileSize", "1000000");//1000*1000 byte, same as .log's max file size
+	    properties.put("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
+	    properties.put("log4j.appender.A1.layout.ConversionPattern", "%d %-5p %c %x - %m%n");
+	    
+		this.hierarchy = new Hierarchy(new RootLogger(Level.INFO));
+		new PropertyConfigurator().doConfigure(properties,hierarchy);	
+	}
+	
+	/**
+	 * Checks if this PluginLogManager is disabled for this level.
+	 * @param level level value
+	 * @return boolean true if it is disabled
+	 */
+	public boolean isDisabled(int level) {
+		return this.hierarchy.isDisabled(level);
+	}
+	
+	/**
+	 * Enable logging for logging requests with level l or higher.
+	 * By default all levels are enabled.
+	 * @param level level object
+	 */
+	public void setThreshold(Level level) {
+		this.hierarchy.setThreshold(level);
+	}
+	
+	/**
+	 * The string version of setThreshold(Level level)
+	 * @param level level string
+	 */
+	public void setThreshold(String level) {
+		this.hierarchy.setThreshold(level);
+	}
+
+	/**
+	 * Get the repository-wide threshold.
+	 * @return Level
+	 */
+	public Level getThreshold() {
+		return this.hierarchy.getThreshold();
+	}
+
+	/**
+	 * Returns a new logger instance named as the first parameter
+	 * using the default factory. If a logger of that name already exists,
+	 * then it will be returned. Otherwise, a new logger will be instantiated 
+	 * and then linked with its existing ancestors as well as children.
+	 * @param name logger name
+	 * @return Logger
+	 */
+	public Logger getLogger(String name) {
+		return this.hierarchy.getLogger(name);
+	}
+	
+	/**
+	 * The same as getLogger(String name) but using a factory instance instead of
+	 * a default factory.
+	 * @param name logger name
+	 * @param factory factory instance 
+	 * @return Logger
+	 */
+	public Logger getLogger(String name, LoggerFactory factory) {
+		return this.hierarchy.getLogger(name,factory);
+	}
+
+	public Logger getRootLogger() {
+		return this.hierarchy.getRootLogger();
+	}
+
+	public Logger exists(String name) {
+		return this.hierarchy.exists(name);
+	}
+	
+	public void shutdown() {
+	    this.hierarchy.shutdown();
+	}
+	
+	/**
+	 * Returns all the loggers in this manager.
+	 * @return Enumeration logger enumeration
+	 */
+	public Enumeration getCurrentLoggers() {
+		return this.hierarchy.getCurrentLoggers();
+	}
+
+	public void resetConfiguration() {
+		this.hierarchy.resetConfiguration();
+	}
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PerformanceStatisticUtil.java

@@ -0,0 +1,368 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.text.DecimalFormat;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.CommonExceptionHandler;
+
+/**
+ * DOC sbliu class global comment. Detailled comment
+ */
+public class PerformanceStatisticUtil {
+
+    private static final DecimalFormat DF = new DecimalFormat("###.##");
+
+    private static final int MEGABYTE = 1024 * 1024;// MB = 1024*1024 byte
+
+    private static final int KILOBYTE = 1024;// kb=1024 byte
+
+    private static final int numOfBlocks = 256;
+
+    private static final int blockSizeKb = 512;
+
+    private static final String dataFile = "testio.data";
+
+    private static String recordingFileName = "performance_record";
+
+    private static File recordingFile = null;
+
+    private static enum BlockSequence {
+        SEQUENTIAL,
+        RANDOM;
+    }
+
+    public static enum StatisticKeys {
+
+        IO_COUNT("I/O.count"), // io count
+        IO_W_MB_SEC("I/O.write"), // write speed MB
+        IO_R_MB_SEC("I/O.read"), // read speed MB
+        IO_W_AVERAGE_MB_SEC("I/O.write.average"), // average speed of write MB
+        IO_R_AVERAGE_MB_SEC("I/O.read.average"), // average speed of read
+
+        STARTUP_AVERAGE("startup.average"),
+        STARTUP_MAX("startup.max"),
+        STARTUP_COUNT("startup.count");
+
+        private String key;
+
+        StatisticKeys(String _key) {
+            key = _key;
+        }
+
+        public String get() {
+            return key;
+        }
+    }
+
+    public static void recordStartupEpapsedTime(double elapsedTimeInSeconds) {
+        File file = getRecordingFile();
+
+        Properties props = read(file, true);
+        String propCount = props.getProperty(StatisticKeys.STARTUP_COUNT.get(), "0");
+        String propMax = props.getProperty(StatisticKeys.STARTUP_MAX.get(), "0");
+        String propAverage = props.getProperty(StatisticKeys.STARTUP_AVERAGE.get(), "0");
+
+        int iPropCount = Integer.parseInt(propCount);
+        double iPropMax = Double.parseDouble(propMax);
+        double iPropAverage = Double.parseDouble(propAverage);
+
+        iPropMax = iPropMax > elapsedTimeInSeconds ? iPropMax : elapsedTimeInSeconds;
+        iPropAverage = (iPropAverage * iPropCount + elapsedTimeInSeconds) / (iPropCount + 1);
+        iPropCount++;
+
+        props.setProperty(StatisticKeys.STARTUP_COUNT.get(), "" + iPropCount);
+        props.setProperty(StatisticKeys.STARTUP_MAX.get(), "" + iPropMax);
+        props.setProperty(StatisticKeys.STARTUP_AVERAGE.get(), "" + iPropAverage);
+
+        store(file, props);
+    }
+
+    public static File getRecordingFile() {
+        if (recordingFile != null) {
+            return recordingFile;
+        }
+
+        String configurationLocation = Platform.getConfigurationLocation().getURL().getPath();
+        File file = new File(configurationLocation + "/" + recordingFileName);
+        return file;
+    }
+
+    public static void setRecordingFile(File _recordingFile) {
+        recordingFile = _recordingFile;
+    }
+
+    public static synchronized Properties read(File recordFile, boolean createIfNotExist) {
+        Properties props = new Properties();
+        if (recordFile != null && exist(recordFile, createIfNotExist)) {
+            FileInputStream inStream = null;
+            try {
+                inStream = new FileInputStream(recordFile);
+                props.load(inStream);
+            } catch (Exception e) {
+                CommonExceptionHandler.log(e.getMessage());
+            } finally {
+                if (inStream != null) {
+                    try {
+                        inStream.close();
+                    } catch (IOException e) {//
+                    }
+                }
+            }
+        }
+
+        return props;
+    }
+
+    public static synchronized void store(File recordFile, Properties props) {
+        if (props == null) {
+            return;
+        }
+
+        if (recordFile != null && exist(recordFile, true)) {
+            FileOutputStream outputStream = null;
+            try {
+                outputStream = new FileOutputStream(recordFile);
+                props.store(outputStream, "");
+            } catch (IOException e) {
+                CommonExceptionHandler.log(e.getMessage());
+            } finally {
+                if (outputStream != null) {
+                    try {
+                        outputStream.close();
+                    } catch (IOException e) {
+                        //
+                    }
+                }
+            }
+        }
+    }
+
+    private static boolean exist(File recordFile, boolean createIfNotExist) {
+        boolean exists = recordFile.exists();
+        if (!exists && createIfNotExist) {
+            try {
+                exists = recordFile.createNewFile();
+                if (!exists) {
+                    throw new FileNotFoundException(recordFile.getName());
+                }
+            } catch (Exception e) {
+                CommonExceptionHandler.log(e.getMessage());
+                return false;
+            }
+        }
+
+        return exists;
+    }
+
+    private static Lock lock = new ReentrantLock();
+    private static Condition condition = lock.newCondition();
+    private static boolean measureIOFinished = true;
+    
+    public static void waitUntilFinish() throws InterruptedException {
+        lock.lock();
+
+        try {
+            if(!measureIOFinished) {
+                condition.await(20, TimeUnit.SECONDS);
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+    
+    public static void measureIO() {
+        new Thread()  {
+            public void run() {
+                measureIOFinished = false;
+                try {
+                    _measureIO();
+                } finally {
+                    measureIOFinished = true;
+                }
+            }
+        }.start();
+    }
+
+    private static void _measureIO() {
+        File file = getRecordingFile();
+        Properties props = read(file, true);
+
+        IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
+        File workspace = root.getLocation().makeAbsolute().toFile();
+        File locationDir = new File(workspace, "temp"); // here is workspace/temp dir
+        File testFile = detectTestDataFile(locationDir);
+
+        if (testFile != null) {
+            measureWrite(props, testFile);
+            measureRead(props, testFile);
+
+            store(file, props);
+        }
+    }
+    
+    private static void measureWrite(Properties props, File testFile) {
+        int blockSize = blockSizeKb * KILOBYTE;
+
+        long startTime = System.nanoTime();
+        long totalBytesWrittenInMark = writeIO(numOfBlocks, BlockSequence.RANDOM, blockSize, testFile);
+        totalBytesWrittenInMark = totalBytesWrittenInMark + writeIO(numOfBlocks, BlockSequence.SEQUENTIAL, blockSize, testFile);
+        long endTime = System.nanoTime();
+
+        long elapsedTimeNs = endTime - startTime;
+        double sec = (double) elapsedTimeNs / (double) 1000000000;
+        double mbWritten = (double) totalBytesWrittenInMark / (double) MEGABYTE;
+        double bwMbSec = mbWritten / sec;
+
+        String ioCount = props.getProperty(StatisticKeys.IO_COUNT.get(), "0");
+        String ioWAverageMbSec = props.getProperty(StatisticKeys.IO_W_AVERAGE_MB_SEC.get(), "0");
+        String ioWMbSec = props.getProperty(StatisticKeys.IO_W_MB_SEC.get(), "0");
+
+        int digital_ioCount = Integer.parseInt(ioCount);
+        double digital_ioWAverageMbSec = Double.parseDouble(ioWAverageMbSec);
+        double digital_ioWMbSec = Double.parseDouble(ioWMbSec);
+
+        digital_ioWAverageMbSec = (digital_ioWAverageMbSec * digital_ioCount + bwMbSec) / (digital_ioCount + 1);
+        digital_ioWMbSec = bwMbSec;
+
+        props.setProperty(StatisticKeys.IO_W_AVERAGE_MB_SEC.get(), "" + DF.format(digital_ioWAverageMbSec));
+        props.setProperty(StatisticKeys.IO_W_MB_SEC.get(), "" + DF.format(digital_ioWMbSec));
+    }
+
+    private static long writeIO(int numOfBlocks, BlockSequence blockSequence, int blockSize, File testFile) {
+        byte[] blockArr = new byte[blockSize];
+        for (int b = 0; b < blockArr.length; b++) {
+            if (b % 2 == 0) {
+                blockArr[b] = (byte) 0xFF;
+            }
+        }
+        String mode = "rwd";// "rwd"
+
+        long totalBytesWrittenInMark = 0;
+        try {
+            try (RandomAccessFile rAccFile = new RandomAccessFile(testFile, mode)) {
+                for (int b = 0; b < numOfBlocks; b++) {
+                    if (blockSequence == BlockSequence.RANDOM) {
+                        int rLoc = randInt(0, numOfBlocks - 1);
+                        rAccFile.seek(rLoc * blockSize);
+                    } else {
+                        rAccFile.seek(b * blockSize);
+                    }
+                    rAccFile.write(blockArr, 0, blockSize);
+                    totalBytesWrittenInMark += blockSize;
+                }
+            }
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+
+        return totalBytesWrittenInMark;
+    }
+
+    private static File detectTestDataFile(File location) {
+        if (!location.exists()) {
+            location.mkdirs();
+        }
+
+        File testFile = null;
+        try {
+            testFile = new File(location.getAbsolutePath() + File.separator + dataFile);
+            testFile.deleteOnExit();
+            testFile.createNewFile();
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+
+        return testFile;
+    }
+
+    public static void measureRead(Properties props, File testFile) {
+        int blockSize = blockSizeKb * KILOBYTE;
+
+        long startTime = System.nanoTime();
+        long totalBytesReadInMark = readIO(numOfBlocks, BlockSequence.RANDOM, blockSize, testFile);
+        totalBytesReadInMark = totalBytesReadInMark + readIO(numOfBlocks, BlockSequence.SEQUENTIAL, blockSize, testFile);
+        long endTime = System.nanoTime();
+        long elapsedTimeNs = endTime - startTime;
+        double sec = (double) elapsedTimeNs / (double) 1000000000;
+        double mbRead = (double) totalBytesReadInMark / (double) MEGABYTE;
+        double bwMbSec = mbRead / sec;
+
+        String ioCount = props.getProperty(StatisticKeys.IO_COUNT.get(), "0");
+        String ioRAverageMbSec = props.getProperty(StatisticKeys.IO_R_AVERAGE_MB_SEC.get(), "0");
+        String ioRMbSec = props.getProperty(StatisticKeys.IO_R_MB_SEC.get(), "0");
+
+        int digital_ioCount = Integer.parseInt(ioCount);
+        double digital_ioRAverageMbSec = Double.parseDouble(ioRAverageMbSec);
+        double digital_ioRMbSec = Double.parseDouble(ioRMbSec);
+        digital_ioRAverageMbSec = (digital_ioRAverageMbSec * digital_ioCount + bwMbSec) / (digital_ioCount + 1);
+        digital_ioRMbSec = bwMbSec;
+        digital_ioCount++;
+
+        props.setProperty(StatisticKeys.IO_R_AVERAGE_MB_SEC.get(), "" + DF.format(digital_ioRAverageMbSec));
+        props.setProperty(StatisticKeys.IO_R_MB_SEC.get(), "" + DF.format(digital_ioRMbSec));
+        props.setProperty(StatisticKeys.IO_COUNT.get(), "" + digital_ioCount);
+    }
+
+    private static long readIO(int numOfBlocks, BlockSequence blockSequence, int blockSize, File testFile) {
+        long totalBytesReadInMark = 0;
+
+        byte[] blockArr = new byte[blockSize];
+        for (int b = 0; b < blockArr.length; b++) {
+            if (b % 2 == 0) {
+                blockArr[b] = (byte) 0xFF;
+            }
+        }
+        try {
+            try (RandomAccessFile rAccFile = new RandomAccessFile(testFile, "r")) {
+                for (int b = 0; b < numOfBlocks; b++) {
+                    if (blockSequence == BlockSequence.RANDOM) {
+                        int rLoc = randInt(0, numOfBlocks - 1);
+                        rAccFile.seek(rLoc * blockSize);
+                    } else {
+                        rAccFile.seek(b * blockSize);
+                    }
+                    rAccFile.readFully(blockArr, 0, blockSize);
+                    totalBytesReadInMark += blockSize;
+                }
+            }
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+        return totalBytesReadInMark;
+    }
+
+    private static int randInt(int min, int max) {
+        // nextInt is normally exclusive of the top value,
+        // so add 1 to make it inclusive
+        int randomNum = new Random().nextInt((max - min) + 1) + min;
+
+        return randomNum;
+    }
+
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/TimeMeasurePerformance.java

@@ -0,0 +1,116 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+
+/**
+ * DOC sbliu  class global comment. Detailled comment
+ */
+public class TimeMeasurePerformance extends TimeMeasure{
+    static private Logger logger;
+    
+    private static HashMap<String, TimeStack> timers;
+    
+    private static long startTime = -1L;
+
+    private static int indent = 0;
+    
+    public static void begin(String idTimer) {
+        startTime = System.nanoTime();
+        
+        init();
+        if (timers.containsKey(idTimer)) {
+            log(indent(indent) + "Warning (start): timer " + idTimer + " already exists"); //$NON-NLS-1$  //$NON-NLS-2$
+        } else {
+            indent++;
+            TimeStack times = new TimeStack();
+            timers.put(idTimer, times);
+            log(indent(indent) + "Start '" + idTimer + "' ..."); //$NON-NLS-1$  //$NON-NLS-2$
+        }
+    }
+    
+    private static void init() {
+        if (timers == null) {
+            timers = new HashMap<String, TimeStack>();
+        }
+        
+        if(logger == null) {
+            configureLogger();
+        }
+    }
+
+    private static void log (String message) {
+        logger.info(message);
+    }
+    
+    public static long end(String idTimer) {
+        init();
+        if (!timers.containsKey(idTimer)) {
+            log(indent(indent) + "Warning (end): timer " + idTimer + " doesn't exist"); //$NON-NLS-1$  //$NON-NLS-2$
+            return -1;
+        } else {
+            TimeStack timeStack = timers.get(idTimer);
+            timers.remove(idTimer);
+            long elapsedTimeSinceLastRequest = timeStack.getLastStepElapsedTime();
+            log(indent(indent) + "End '" + idTimer + "', elapsed time since last request: " //$NON-NLS-1$  //$NON-NLS-2$
+                    + elapsedTimeSinceLastRequest + " ms "); //$NON-NLS-1$
+            
+            long totalElapsedTime = timeStack.getTotalElapsedTime();
+            
+            log(indent(indent) + "End '" + idTimer + "', total elapsed time: " + totalElapsedTime + " ms "); //$NON-NLS-1$  //$NON-NLS-2$  //$NON-NLS-3$
+            
+            indent--;
+            return totalElapsedTime;
+        }
+    }
+    
+    public static long step(String idTimer, String stepName) {
+        init();
+        if (!timers.containsKey(idTimer)) {
+            log(indent(indent) + "Warning (end): timer " + idTimer + " does'nt exist"); //$NON-NLS-1$  //$NON-NLS-2$
+            return -1;
+        } else {
+            TimeStack timeStack = timers.get(idTimer);
+            timeStack.addStep();
+            /*
+             * trace the timeline of every step,problem is that the code below " Calendar ca = Calendar.getInstance();
+             * Date now = ca.getTime();" will cost almost 13ms~15ms
+             */
+            long time = timeStack.getLastStepElapsedTime();
+            String timerStepName = idTimer + "', step name '" + stepName; //$NON-NLS-1$
+            
+            log(indent(indent)
+                    + "-> '" + timerStepName + "', elapsed time since previous step: " + time + " ms"); //$NON-NLS-1$  //$NON-NLS-2$  //$NON-NLS-3$
+            
+            return time;
+        }
+    }
+    
+    private static void configureLogger() {
+        try {
+            PerformanceLogManager logManager = new PerformanceLogManager();
+            logger = logManager.getLogger(TimeMeasurePerformance.class.getName());
+        } catch (Exception e) {
+            throw new RuntimeException("Error while initializing log properties.", e);
+        }
+    }
+    
+    public static void afterStartup() {
+        double elapsedTimeInSeconds = (double)(System.nanoTime() - startTime)/1000000000;
+        PerformanceStatisticUtil.recordStartupEpapsedTime(elapsedTimeInSeconds);
+        PerformanceStatisticUtil.measureIO();
+    }
+}

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/proposal/CommonTextCellEditorWithProposal.java

@@ -212,4 +212,8 @@ public class CommonTextCellEditorWithProposal {
         return this.contentProposalAdapter;
     }
 
+    public int getPreviousActivatedIndex() {
+        return previousActivatedIndex;
+    }
+
 }

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/proposal/ExtendedTextCellEditorWithProposal.java

@@ -97,4 +97,7 @@ public class ExtendedTextCellEditorWithProposal extends ExtendedTextCellEditor i
         super.fireCancelEditor();
     }
 
+    public CommonTextCellEditorWithProposal getCommonTextEditor() {
+        return commonTextEditor;
+    }
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/handlers/JobAndNodesParametersRelationshipHandler.java

@@ -117,6 +117,7 @@ public class JobAndNodesParametersRelationshipHandler implements IItemRelationsh
                 }
             }
         }
+        relationsMap.values().removeIf(value -> value.isEmpty());
         return relationsMap;
 
     }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/handlers/JobRoutinesItemRelationshipHandler.java

@@ -14,6 +14,7 @@ package org.talend.core.repository.handlers;
 
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 import org.talend.core.model.properties.Item;
@@ -43,8 +44,13 @@ public class JobRoutinesItemRelationshipHandler extends AbstractJobItemRelations
         Set<Relation> relationSet = new HashSet<Relation>();
 
         if (processType.getParameters() != null && processType.getParameters().getRoutinesParameter() != null) {
+            Map<String, String> currentSystemRoutinesMap = RelationshipItemBuilder.getInstance().getCurrentSystemRoutinesMap();
             for (Object o : processType.getParameters().getRoutinesParameter()) {
                 RoutinesParameterType itemInfor = (RoutinesParameterType) o;
+                if (currentSystemRoutinesMap.containsValue(itemInfor.getName())) {
+                    // exclude system routines relation
+                    continue;
+                }
 
                 Relation addedRelation = new Relation();
                 addedRelation.setId(itemInfor.getName());

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/logintask/SyncLibrariesLoginTask.java

@@ -67,4 +67,9 @@ public class SyncLibrariesLoginTask extends AbstractLoginTask implements IRunnab
                 });
     }
 
+    @Override
+    public boolean isRequiredAlways() {
+        return true;
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/AbstractEMFRepositoryFactory.java

@@ -837,8 +837,8 @@ public abstract class AbstractEMFRepositoryFactory extends AbstractRepositoryFac
         Object fullFolder = getFullFolder(project, type, relativeFolder);
         serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, false, true, true);
         if (serializableAllVersion.isEmpty()) {
-            // look in all folders
-            serializableAllVersion = getSerializable(project, id, false, false);
+            // look in all folders for this item type
+            serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, true, true, true, true);
         }
         int size = serializableAllVersion.size();
 

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/IRepositoryFactory.java

@@ -270,6 +270,8 @@ public interface IRepositoryFactory {
 
     public void create(Project project, Item item, IPath path, boolean... isImportItem) throws PersistenceException;
 
+    public void save(Project project, Item item, boolean isMigrationTask) throws PersistenceException;
+    
     public void save(Project project, Item item) throws PersistenceException;
 
     public void save(Project project, Property property) throws PersistenceException;

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProjectRepositoryNode.java

@@ -1064,6 +1064,9 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         }
         if (newProject != null && newProject.getEmfProject() != null) {
             List<FolderItem> folderItems = ProjectManager.getInstance().getFolders(newProject.getEmfProject());
+            if (folderItems != null) {
+                folderItems = new ArrayList<>(folderItems);
+            }
             for (FolderItem folder : folderItems) {
                 String folderName = folder.getProperty().getLabel();
                 if (("process".equals(folderName) || "joblets".equals(folderName)) && folder.getChildren() != null

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -15,6 +15,7 @@ package org.talend.core.repository.model;
 import java.beans.PropertyChangeListener;
 import java.beans.PropertyChangeSupport;
 import java.io.File;
+import java.lang.reflect.InvocationTargetException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Date;
@@ -71,7 +72,7 @@ import org.talend.commons.ui.gmf.util.DisplayUtils;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.utils.data.container.RootContainer;
 import org.talend.commons.utils.network.TalendProxySelector;
-import org.talend.commons.utils.time.TimeMeasure;
+import org.talend.commons.utils.time.TimeMeasurePerformance;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.AbstractDQModelService;
 import org.talend.core.GlobalServiceRegister;
@@ -84,6 +85,7 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.exception.TalendInternalPersistenceException;
 import org.talend.core.hadoop.BigDataBasicUtil;
+import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.MetadataTalendType;
@@ -108,6 +110,7 @@ import org.talend.core.model.properties.SpagoBiServer;
 import org.talend.core.model.properties.Status;
 import org.talend.core.model.properties.User;
 import org.talend.core.model.properties.impl.FolderItemImpl;
+import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.Folder;
 import org.talend.core.model.repository.IRepositoryContentHandler;
@@ -123,6 +126,8 @@ import org.talend.core.repository.constants.Constant;
 import org.talend.core.repository.constants.FileConstants;
 import org.talend.core.repository.i18n.Messages;
 import org.talend.core.repository.recyclebin.RecycleBinManager;
+import org.talend.core.repository.utils.LoginTaskRegistryReader;
+import org.talend.core.repository.utils.ProjectDataJsonProvider;
 import org.talend.core.repository.utils.RepositoryPathProvider;
 import org.talend.core.repository.utils.XmiResourceManager;
 import org.talend.core.runtime.CoreRuntimePlugin;
@@ -134,6 +139,7 @@ import org.talend.core.service.IUpdateService;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.TableHelper;
 import org.talend.designer.runprocess.IRunProcessService;
+import org.talend.login.ILoginTask;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.ReferenceProjectProblemManager;
 import org.talend.repository.ReferenceProjectProvider;
@@ -175,6 +181,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
     private boolean isCancelled;
 
+    private static final LoginTaskRegistryReader LOGIN_TASK_REGISTRY_READER = new LoginTaskRegistryReader();
+
     @Override
     public synchronized void addPropertyChangeListener(PropertyChangeListener l) {
         if (l == null) {
@@ -224,6 +232,13 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         return null;
     }
 
+    private ILibrariesService getLibrariesService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibrariesService.class)) {
+            return GlobalServiceRegister.getDefault().getService(ILibrariesService.class);
+        }
+        return null;
+    }
+    
     /*
      * (non-Javadoc)
      *
@@ -1211,6 +1226,28 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         return this.repositoryFactoryFromProvider.getLastVersion(project, ProcessUtils.getPureItemId(id), folderPath, type);
     }
 
+
+    @Override
+    public IRepositoryViewObject getLastVersion(String id, ERepositoryObjectType type)
+            throws PersistenceException {
+        return getLastVersion(id , "", type);
+    }
+    
+    @Override
+    public IRepositoryViewObject getLastVersion(String id, List<ERepositoryObjectType> types) throws PersistenceException {
+        if (types != null) {
+            IRepositoryViewObject object = null;
+            for (ERepositoryObjectType type : types) {
+                object = getLastVersion(id, type);
+                if (object != null) {
+                    return object;
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
     public IRepositoryViewObject getLastVersion(String id, String folderPath, ERepositoryObjectType type)
             throws PersistenceException {
         String objId = id;
@@ -1222,7 +1259,25 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 return this.repositoryFactoryFromProvider.getLastVersion(project, objId, folderPath, type);
             }
         }
-        return this.repositoryFactoryFromProvider.getLastVersion(projectManager.getCurrentProject(), objId , folderPath, type);
+        return getLastRefVersion(projectManager.getCurrentProject(), objId , folderPath, type);
+    }
+    
+    @Override
+    public IRepositoryViewObject getLastRefVersion(Project project, String id, String folderPath, ERepositoryObjectType type) throws PersistenceException {
+        String projectLabel = ProcessUtils.getProjectLabelFromItemId(id);
+        IRepositoryViewObject lastVersion = getLastVersion(project, ProcessUtils.getPureItemId(id), folderPath, type);
+        if (lastVersion == null) {
+            for (Project p : projectManager.getReferencedProjects(project)) {
+                if (projectLabel != null && !projectLabel.equals(p.getTechnicalLabel())) {
+                    continue;
+                }
+                lastVersion = getLastRefVersion(p, id);
+                if (lastVersion != null) {
+                    break;
+                }
+            }
+        }
+        return lastVersion;
     }
 
     @Override
@@ -1544,14 +1599,15 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
     @Override
     public void save(Project project, Item item, boolean... isMigrationTask) throws PersistenceException {
-        this.repositoryFactoryFromProvider.save(project, item);
         if (isMigrationTask == null || isMigrationTask.length == 0 || !isMigrationTask[0]) {
+            this.repositoryFactoryFromProvider.save(project, item);
             boolean avoidGenerateProm = false;
             if (isMigrationTask != null && isMigrationTask.length == 2) {
                 avoidGenerateProm = isMigrationTask[1];
             }
             fireRepositoryPropertyChange(ERepositoryActionName.SAVE.getName(), avoidGenerateProm, item);
-
+        } else {
+            this.repositoryFactoryFromProvider.save(project, item, true);
         }
     }
 
@@ -2033,11 +2089,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
      */
     public void logOnProject(Project project, IProgressMonitor monitor) throws LoginException, PersistenceException {
         try {
-            TimeMeasure.display = CommonsPlugin.isDebugMode();
-            TimeMeasure.displaySteps = CommonsPlugin.isDebugMode();
-            TimeMeasure.measureActive = CommonsPlugin.isDebugMode();
-
-            TimeMeasure.begin("logOnProject"); //$NON-NLS-1$
+            TimeMeasurePerformance.begin("logOnProject"); //$NON-NLS-1$
             try {
                 /**
                  * init/check proxy selector, in case default proxy selector is not registed yet
@@ -2063,6 +2115,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 SubMonitor subMonitor = SubMonitor.convert(monitor, MAX_TASKS);
                 SubMonitor currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                 currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.logonInProgress"), 1); //$NON-NLS-1$
+
                 project.setReferenceProjectProvider(null);
                 getRepositoryContext().setProject(null);
                 initEmfProjectContent();
@@ -2080,10 +2133,21 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 this.repositoryFactoryFromProvider.beforeLogon(project);
                 ProjectManager.getInstance().getBeforeLogonRecords().clear();
                 ProjectManager.getInstance().getUpdatedRemoteHandlerRecords().clear();
+                ILibrariesService librariesService = getLibrariesService();
+                if (librariesService != null) {
+                	librariesService.setForceReloadCustomUri();
+                }
+
+                ProjectDataJsonProvider.checkAndRectifyRelationShipSetting(project.getEmfProject());
 
                 // init dynamic distirbution after `beforeLogon`, before loading libraries.
                 initDynamicDistribution(monitor);
 
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IUpdateService.class)) {
+                    IUpdateService updateService = GlobalServiceRegister.getDefault().getService(IUpdateService.class);
+                    updateService.syncComponentM2Jars(currentMonitor);
+                }
+
                 // init sdk component
                 try {
                     currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
@@ -2101,7 +2165,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                     throw new OperationCanceledException(""); //$NON-NLS-1$
                 }
                 // monitorWrap.worked(1);
-                TimeMeasure.step("logOnProject", "beforeLogon"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "beforeLogon"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 // Check project compatibility
                 checkProjectCompatibility(project);
@@ -2119,7 +2183,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                     currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                     currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.synchronizeLibraries"), 1); //$NON-NLS-1$
                     coreService.syncLibraries(currentMonitor);
-                    TimeMeasure.step("logOnProject", "Sync components libraries"); //$NON-NLS-1$
+                    TimeMeasurePerformance.step("logOnProject", "Sync components libraries"); //$NON-NLS-1$
                 }
 
                 currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
@@ -2128,7 +2192,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 executeMigrations(project, true, currentMonitor);
                 ProjectManager.getInstance().getMigrationRecords().clear();
                 // monitorWrap.worked(1);
-                TimeMeasure.step("logOnProject", "executeMigrations(beforeLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "executeMigrations(beforeLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                 currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.logonInProgress"), 1); //$NON-NLS-1$
@@ -2136,7 +2200,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 this.repositoryFactoryFromProvider.logOnProject(project);
                 ProjectManager.getInstance().getLogonRecords().clear();
                 // monitorWrap.worked(1);
-                TimeMeasure.step("logOnProject", "logOnProject"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "logOnProject"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 emptyTempFolder(project);
 
@@ -2152,11 +2216,6 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
                 fireRepositoryPropertyChange(ERepositoryActionName.PROJECT_PREFERENCES_RELOAD.getName(), null, null);
 
-                if (GlobalServiceRegister.getDefault().isServiceRegistered(IUpdateService.class)) {
-                    IUpdateService updateService = GlobalServiceRegister.getDefault().getService(IUpdateService.class);
-                    updateService.syncComponentM2Jars(currentMonitor);
-                }
-
                 IRunProcessService runProcessService = getRunProcessService();
                 if (runProcessService != null) {
                     runProcessService.initMavenJavaProject(monitor, project);
@@ -2167,7 +2226,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 ProjectManager.getInstance().getMigrationRecords().clear();
                 executeMigrations(project, false, currentMonitor);
                 ProjectManager.getInstance().getMigrationRecords().clear();
-                TimeMeasure.step("logOnProject", "executeMigrations(afterLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "executeMigrations(afterLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
                 if (monitor != null && monitor.isCanceled()) {
                     throw new OperationCanceledException(""); //$NON-NLS-1$
                 }
@@ -2183,12 +2242,12 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                     // clean workspace
                     currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.cleanWorkspace"), 1); //$NON-NLS-1$
 
-                    TimeMeasure.step("logOnProject", "clean Java project"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "clean Java project"); //$NON-NLS-1$ //$NON-NLS-2$
 
                     if (workspace instanceof Workspace) {
                         ((Workspace) workspace).getFileSystemManager().getHistoryStore().clean(currentMonitor);
                     }
-                    TimeMeasure.step("logOnProject", "clean workspace history"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "clean workspace history"); //$NON-NLS-1$ //$NON-NLS-2$
 
                     currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                     currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.synch.repo.items"), 1); //$NON-NLS-1$
@@ -2214,12 +2273,12 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 if (monitor != null && monitor.isCanceled()) {
                     throw new OperationCanceledException(""); //$NON-NLS-1$
                 }
-                TimeMeasure.step("logOnProject", "sync repository (routines/rules/beans)"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "sync repository (routines/rules/beans)"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 // log4j prefs
                 if (coreUiService != null && coreService != null) {
                     coreService.syncLog4jSettings(null);
-                    TimeMeasure.step("logOnProject", "sync log4j"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "sync log4j"); //$NON-NLS-1$ //$NON-NLS-2$
                 }
 
                 try {
@@ -2228,6 +2287,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                         // set the project mappings url
                         System.setProperty("talend.mappings.url", url.toString()); // $NON-NLS-1$
                     }
+                    // for new added mapping file, sync to project mapping folder
+                    MetadataTalendType.syncNewMappingFileToProject();
                 } catch (SystemException e) {
                     // ignore
                     ExceptionHandler.process(e);
@@ -2236,7 +2297,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 if (runProcessService != null && !isCommandLineLocalRefProject) {
                     runProcessService.initializeRootPoms(monitor);
 
-                    TimeMeasure.step("logOnProject", "install / setup root poms"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "install / setup root poms"); //$NON-NLS-1$ //$NON-NLS-2$
                 }
                 if (GlobalServiceRegister.getDefault().isServiceRegistered(ITDQRepositoryService.class)) {
                     ITDQRepositoryService tdqRepositoryService = GlobalServiceRegister.getDefault()
@@ -2245,13 +2306,15 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                         tdqRepositoryService.initProxyRepository();
                     }
                 }
+                // regenerate relationship index
+                if (project.getEmfProject().getItemsRelations().isEmpty()) {
+                    RelationshipItemBuilder.getInstance().buildAndSaveIndex();
+                }
+
                 fullLogonFinished = true;
                 this.repositoryFactoryFromProvider.afterLogon(monitor);
             } finally {
-                TimeMeasure.end("logOnProject"); //$NON-NLS-1$
-                TimeMeasure.display = false;
-                TimeMeasure.displaySteps = false;
-                TimeMeasure.measureActive = false;
+                TimeMeasurePerformance.end("logOnProject"); //$NON-NLS-1$
             }
             String str[] = new String[] { getRepositoryContext().getUser() + "", projectManager.getCurrentProject() + "" }; //$NON-NLS-1$ //$NON-NLS-2$
             log.info(Messages.getString("ProxyRepositoryFactory.log.loggedOn", str)); //$NON-NLS-1$
@@ -2652,4 +2715,13 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
     public RepositoryWorkUnit getWorkUnitInProgress() {
         return repositoryFactoryFromProvider.getWorkUnitInProgress();
     }
+
+    public void executeRequiredLoginTasks(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
+        ILoginTask[] allLoginTasks = LOGIN_TASK_REGISTRY_READER.getAllTaskListInstance();
+        for (ILoginTask task : allLoginTasks) {
+            if (task.isRequiredAlways()) {
+                task.run(monitor);
+            }
+        }
+    }
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/recyclebin/RecycleBinManager.java

@@ -14,6 +14,7 @@ package org.talend.core.repository.recyclebin;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
@@ -24,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.EList;
@@ -106,7 +108,8 @@ public class RecycleBinManager {
         List<IRepositoryViewObject> deletedObjects = new ArrayList<IRepositoryViewObject>();
         final EList<TalendItem> deletedItems = projectRecyclebins.get(project.getTechnicalLabel()).getDeletedItems();
         List<TalendItem> notDeletedItems = new ArrayList<TalendItem>();
-        for (TalendItem deletedItem : deletedItems) {
+        List<TalendItem> dup_deletedItems = new ArrayList<>(deletedItems);
+        for (TalendItem deletedItem : dup_deletedItems) {
             try {
                 final ERepositoryObjectType type = ERepositoryObjectType.getType(deletedItem.getType());
                 // ignore the generated doc in recycle bin
@@ -274,7 +277,40 @@ public class RecycleBinManager {
                 resource = createRecycleBinResource(project);
             }
             resource.getContents().clear();
-            recycleBin.setLastUpdate(new Date());
+            EList<String> deletedFolders = recycleBin.getDeletedFolders();
+            if (deletedFolders != null) {
+                List<String> folders = new LinkedList<>(deletedFolders);
+                Collections.sort(folders);
+                deletedFolders.clear();
+                deletedFolders.addAll(folders);
+            }
+            EList<TalendItem> deletedItems = recycleBin.getDeletedItems();
+            if (deletedItems != null) {
+                List<TalendItem> items = new LinkedList<>(deletedItems);
+                items.sort((l, r) -> {
+                    if (l == null && r == null) {
+                        return 0;
+                    } else if (l == null) {
+                        return -1;
+                    } else if (r == null) {
+                        return 1;
+                    }
+                    int result = StringUtils.compare(l.getType(), r.getType());
+                    if (result != 0) {
+                        return result;
+                    }
+                    result = StringUtils.compare(l.getPath(), r.getPath());
+                    if (result != 0) {
+                        return result;
+                    }
+                    return StringUtils.compare(l.getId(), r.getId());
+                });
+                deletedItems.clear();
+                deletedItems.addAll(items);
+            }
+
+            // set date to null to avoid timezone conflict
+            recycleBin.setLastUpdate(null);
             resource.getContents().add(recycleBin);
             EmfHelper.saveResource(resource);
             lastSavedRecycleBinMap.put(recycleBin, EcoreUtil.copy(recycleBin));

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ProjectDataJsonProvider.java

@@ -22,6 +22,7 @@ import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
@@ -32,6 +33,7 @@ import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.Path;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.model.properties.ImplicitContextSettings;
@@ -43,6 +45,7 @@ import org.talend.core.model.properties.Project;
 import org.talend.core.model.properties.StatAndLogsSettings;
 import org.talend.core.model.properties.Status;
 import org.talend.core.model.properties.impl.PropertiesFactoryImpl;
+import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.repository.constants.FileConstants;
 import org.talend.core.repository.recyclebin.RecycleBinManager;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
@@ -140,9 +143,10 @@ public class ProjectDataJsonProvider {
             ProjectDataJsonProvider.loadProjectSettings(project, input);
         }
         IPath relationShipPath = settingFolderPath.append(FileConstants.RELATIONSHIP_FILE_NAME);
-        input = inputStreamProvider.getStream(relationShipPath);
-        if (input != null) {
-            ProjectDataJsonProvider.loadRelationShips(project, input);
+        try (InputStream is = inputStreamProvider.getStream(relationShipPath)) {
+            if (is != null) {
+                ProjectDataJsonProvider.loadRelationShips(project, is);
+            }
         }
         IPath migrationTaskPath = settingFolderPath.append(FileConstants.MIGRATION_TASK_FILE_NAME);
         input = inputStreamProvider.getStream(migrationTaskPath);
@@ -184,10 +188,11 @@ public class ProjectDataJsonProvider {
     private static void loadRelationShips(Project project, IPath projectFolderPath) throws PersistenceException {
         File file = getLoadingConfigurationFile(projectFolderPath, FileConstants.RELATIONSHIP_FILE_NAME);
         if (file != null && file.exists()) {
-            try {
-                loadRelationShips(project, new FileInputStream(file));
-            } catch (FileNotFoundException e) {
-                throw new PersistenceException(e);
+            try (FileInputStream fis = new FileInputStream(file)) {
+                loadRelationShips(project, fis);
+            } catch (Exception e) {
+                // ignore
+                ExceptionHandler.process(e);
             }
         }
     }
@@ -206,11 +211,72 @@ public class ProjectDataJsonProvider {
                     project.getItemsRelations().add(json.toEmfObject());
                 }
             }
+        } catch (Exception e) {
+            // ignore
+            ExceptionHandler.process(e);
+        }
+    }
+
+    public static void checkAndRectifyRelationShipSetting(Project project) throws PersistenceException {
+        File file = getSavingConfigurationFile(project.getTechnicalLabel(), FileConstants.RELATIONSHIP_FILE_NAME);
+        if (file == null || !file.exists()) {
+            return;
+        }
+
+        List<ItemRelationsJson> itemRelationsJsonsList = null;
+        TypeReference<List<ItemRelationsJson>> typeReference = new TypeReference<List<ItemRelationsJson>>() {
+        };
+        FileInputStream input = null;
+        try {
+            input = new FileInputStream(file);
+            itemRelationsJsonsList = new ObjectMapper().readValue(new FileInputStream(file), typeReference);
         } catch (Exception e) {
             throw new PersistenceException(e);
         } finally {
             closeInputStream(input);
         }
+
+        if (itemRelationsJsonsList == null || itemRelationsJsonsList.isEmpty()) {
+            return;
+        }
+
+        Set<String> idVersionSet = new HashSet<String>();
+        List<ItemRelationsJson> relationJsonList = new ArrayList<ItemRelationsJson>();
+        Map<String, String> currentSystemRoutinesMap = RelationshipItemBuilder.getInstance().getCurrentSystemRoutinesMap();
+        boolean needModify = false;
+        for (ItemRelationsJson relationJson : itemRelationsJsonsList) {
+            ItemRelationJson baseItem = relationJson.getBaseItem();
+            String idversion = baseItem.getId() + ";" + baseItem.getVersion();
+            if (idVersionSet.contains(idversion)) {
+                // in case duplicate
+                needModify = true;
+                continue;
+            }
+            // remove system routines relation
+            int originalSize = relationJson.getRelatedItems().size();
+            relationJson.getRelatedItems()
+                    .removeIf(relatedItem -> RelationshipItemBuilder.ROUTINE_RELATION.equals(relatedItem.getType())
+                            && currentSystemRoutinesMap.containsValue(relatedItem.getId()));
+            if (relationJson.getRelatedItems().size() != originalSize) {
+                needModify = true;
+            }
+            if (!relationJson.getRelatedItems().isEmpty()) {
+                relationJsonList.add(relationJson);
+            }
+            idVersionSet.add(idversion);
+        }
+
+        if (needModify) {
+            // re-load to project
+            if (relationJsonList != null && !relationJsonList.isEmpty()) {
+                project.getItemsRelations().clear();
+                for (ItemRelationsJson json : relationJsonList) {
+                    project.getItemsRelations().add(json.toEmfObject());
+                }
+            }
+            // re-save relationship setting json file
+            saveRelationShips(project);
+        }
     }
 
     private static void loadMigrationTaskSetting(Project project, IPath projectFolderPath) throws PersistenceException {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/XmiResourceManager.java

@@ -26,6 +26,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.log4j.Priority;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
@@ -37,6 +38,7 @@ import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.common.util.URI;
 import org.eclipse.emf.ecore.EObject;
 import org.eclipse.emf.ecore.resource.Resource;
@@ -69,6 +71,7 @@ import org.talend.core.model.properties.ValidationRulesConnectionItem;
 import org.talend.core.model.properties.helper.ByteArrayResource;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.repository.constants.FileConstants;
+import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.repository.utils.ResourceFilenameHelper.FileName;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
@@ -647,6 +650,23 @@ public class XmiResourceManager {
     }
 
     public void saveResource(Resource resource) throws PersistenceException {
+        try {
+            if (resource != null) {
+                Object objectByType = EcoreUtil.getObjectByType(resource.getContents(), PropertiesPackage.eINSTANCE.getProject());
+                if (objectByType != null) {
+                    Project project = (Project) objectByType;
+                    EList migrationTasks = project.getMigrationTask();
+                    if (migrationTasks != null && 1 < migrationTasks.size()) {
+                        org.talend.commons.exception.ExceptionHandler.process(new Exception("Bad saving logic for Project"),
+                                Priority.WARN);
+                        ProxyRepositoryFactory.getInstance().saveProject(new org.talend.core.model.general.Project(project));
+                        return;
+                    }
+                }
+            }
+        } catch (Throwable e) {
+            org.talend.commons.exception.ExceptionHandler.process(e);
+        }
         EmfHelper.saveResource(resource);
     }
 

main/plugins/org.talend.core.runtime/mappings/mapping_Databricks_Delta_Lake.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0"?>
+<mapping>
+	<dbms product="DATABRICKS_DELTA_LAKE" id="databricks_delta_lake_id" label="Mapping Delta Lake" default="true">
+		<dbTypes>
+			<dbType type="SMALLINT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="FLOAT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DOUBLE" ignoreLen="true" ignorePre="true"/>
+			<dbType type="BIGINT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="INT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="TINYINT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="STRING" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DECIMAL" ignoreLen="false" ignorePre="false"/>
+			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true"/>
+			<dbType type="STRUCT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="MAP" ignoreLen="true" ignorePre="true"/>
+			<dbType type="ARRAY" ignoreLen="true" ignorePre="true"/>
+			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DATE" ignoreLen="true" ignorePre="true"/>
+		</dbTypes>
+		
+		<language name="java">
+			<talendToDbTypes><!-- Adviced mappings  -->
+				<talendType type="id_List"/>
+				<talendType type="id_Boolean">	
+					<dbType type="BOOLEAN" default="true"/>				
+				</talendType>
+				<talendType type="id_Byte">
+					<dbType type="TINYINT" default="true"/>
+					<dbType type="BIGINT"/>
+					<dbType type="INT"/>
+					<dbType type="SMALLINT"/>				
+				</talendType>
+				<talendType type="id_byte[]"/>
+				<talendType type="id_Character">
+					<dbType type="STRING" default="true"/>
+				</talendType>
+				<talendType type="id_Date">
+				    <dbType type="TIMESTAMP" default="true"/>
+				    <dbType type="DATE"/>
+				</talendType>
+				<talendType type="id_BigDecimal">
+				    <dbType type="DECIMAL" default="true"/>
+					<dbType type="BIGINT"/>
+					<dbType type="FLOAT"/>
+					<dbType type="DOUBLE"/>
+				</talendType>
+				<talendType type="id_Double">
+					<dbType type="DOUBLE" default="true" />
+					<dbType type="FLOAT"/>
+				</talendType>
+				<talendType type="id_Float">
+					<dbType type="FLOAT" default="true" />
+					<dbType type="DOUBLE"/>
+				</talendType>
+				<talendType type="id_Integer">
+					<dbType type="INT" default="true" />
+					<dbType type="BIGINT" />
+				</talendType>
+				<talendType type="id_Long">
+					<dbType type="BIGINT" default="true" />		
+				</talendType>
+				<talendType type="id_Object">
+					<dbType type="STRUCT" default="true" />
+					<dbType type="MAP" />
+					<dbType type="ARRAY" />	
+				</talendType>
+				<talendType type="id_Short">
+					<dbType type="SMALLINT" default="true" />
+					<dbType type="INT" />
+				</talendType>
+				<talendType type="id_String">
+					<dbType type="STRING" default="true"/>
+				</talendType>
+			</talendToDbTypes>
+			<dbToTalendTypes><!-- Adviced mappings  -->
+				<dbType type="STRING">
+					<talendType type="id_String" default="true"/>
+				</dbType>
+				<dbType type="BOOLEAN">
+					<talendType type="id_Boolean" default="true"/>
+				</dbType>
+				<dbType type="DOUBLE">
+					<talendType type="id_Double" default="true"/>
+					<talendType type="id_BigDecimal"/>
+					<talendType type="id_Float"/>
+				</dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true"/>
+                </dbType>
+				<dbType type="BIGINT">
+					<talendType type="id_Long" default="true"/>
+					<talendType type="id_Integer"/>
+				</dbType>
+				<dbType type="INT">
+					<talendType type="id_Integer" default="true"/>
+					<talendType type="id_Short"/>
+				</dbType>
+				<dbType type="SMALLINT">
+					<talendType type="id_Short" default="true"/>
+					<talendType type="id_Byte"/>
+				</dbType>
+				<dbType type="TINYINT">
+					<talendType type="id_Byte" default="true"/>
+				</dbType>
+				<dbType type="FLOAT">
+					<talendType type="id_Float" default="true"/>
+					<talendType type="id_Double"/>
+					<talendType type="id_BigDecimal"/>
+				</dbType>
+				<dbType type="STRUCT">
+					<talendType type="id_Object" default="true"/>
+				</dbType>
+				<dbType type="MAP">
+					<talendType type="id_Object" default="true"/>
+				</dbType>
+				<dbType type="ARRAY">
+					<talendType type="id_Object" default="true"/>
+				</dbType>
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true"/>
+                </dbType>
+                <dbType type="DATE">
+                    <talendType type="id_Date" default="true"/>
+                </dbType>
+			</dbToTalendTypes>
+		</language>
+	</dbms>
+</mapping>

main/plugins/org.talend.core.runtime/mappings/mapping_Mysql.xml

@@ -40,7 +40,7 @@
 			<dbType type="SET" ignoreLen="true" ignorePre="true" />
 			<dbType type="TEXT" ignoreLen="true" ignorePre="true" />
 			<dbType type="TIME" ignoreLen="true" ignorePre="true" />
-			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true" />
+			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="false" />
 			<dbType type="TINYBLOB" ignoreLen="true" ignorePre="true" />
 			<dbType type="TINYINT" ignorePre="true" />
 			<dbType type="TINYINT UNSIGNED" ignoreLen="true" ignorePre="true" />

main/plugins/org.talend.core.runtime/mappings/mapping_Sybase.xml

@@ -9,7 +9,7 @@
 			<dbType type="CHAR" defaultLength="50" ignorePre="true"/>
 			<dbType type="CHARACTER" ignoreLen="true" ignorePre="true" />
 			<dbType type="CHARACTER VARYING" ignoreLen="true" ignorePre="true" />
-			<dbType type="DATE" ignoreLen="true" ignorePre="true" />
+			<dbType type="DATE" ignoreLen="true" ignorePre="true" defaultPattern="dd-MM-yyyy" />
 			<dbType type="DATETIME" ignoreLen="true" ignorePre="true" />
 			<dbType type="DECIMAL" defaultLength="20" defaultPrecision="10" />
 			<dbType type="DOUBLE PRECISION" ignoreLen="true" ignorePre="true" />

main/plugins/org.talend.core.runtime/mappings/mapping_validate.xsd

@@ -100,6 +100,11 @@
 					<xs:restriction base="xs:short"/>
 				</xs:simpleType>
 			</xs:attribute>
+			<xs:attribute name="defaultPattern">
+    <xs:simpleType>
+     <xs:restriction base="xs:string"/>
+    </xs:simpleType>
+   </xs:attribute>
 			<xs:attribute name="default" type="xs:boolean"/>
 		</xs:complexType>
 	</xs:element>

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ITDQRepositoryService.java

@@ -214,5 +214,5 @@ public interface ITDQRepositoryService extends IService {
     /**
      * @param chooseContext the context name which want to swtich
      */
-    void popupSwitchContextFailedMessage(String chooseContext);
+    boolean popupSwitchContextFailedMessage(String chooseContext);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EImpalaDriver.java

@@ -0,0 +1,84 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.database;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.talend.core.runtime.hd.hive.HiveMetadataHelper;
+
+/**
+ * DOC hzhao  class global comment. Detailled comment
+ */
+public enum EImpalaDriver {
+
+    HIVE2("HIVE2", "HIVE2", "org.apache.hive.jdbc.HiveDriver"),
+    IMPALA40("IMPALA40", "IMPALA40", "com.cloudera.impala.jdbc4.Driver"),
+    IMPALA41("IMPALA41", "IMPALA41", "com.cloudera.impala.jdbc41.Driver");
+
+    EImpalaDriver(String displayName, String name, String driver) {
+        this.displayName = displayName;
+        this.name = name;
+        this.driver = driver;
+    }
+
+    private String displayName;
+
+    private String name;
+
+    private String driver;
+
+    public String getDisplayName() {
+        return displayName;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getDriver() {
+        return driver;
+    }
+
+    public static boolean isSupport(String distribution, String version, boolean byDisplay, String supportMethodName) {
+        return HiveMetadataHelper.doSupportMethod(distribution, version, byDisplay, supportMethodName);
+    }
+
+    public static String[] getImpalaDriverDisplay(String distribution, String version, boolean byDisplay) {
+        List<String> list = new ArrayList<>(0);
+        for (EImpalaDriver driver : EImpalaDriver.values()) {
+            if (isSupport(distribution, version, byDisplay, "doSupportImpalaConnector")) {
+                list.add(driver.getDisplayName());
+            }
+        }
+        return list.toArray(new String[0]);
+    }
+
+    public static EImpalaDriver getByDisplay(String display) {
+        for (EImpalaDriver driver : EImpalaDriver.values()) {
+            if (driver.getDisplayName().equals(display)) {
+                return driver;
+            }
+        }
+        return null;
+    }
+
+    public static EImpalaDriver getByName(String name) {
+        for (EImpalaDriver driver : EImpalaDriver.values()) {
+            if (driver.getName().equals(name)) {
+                return driver;
+            }
+        }
+        return null;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/ConnParameterKeys.java

@@ -267,6 +267,8 @@ public class ConnParameterKeys {
 
     public static final String IMPALA_AUTHENTICATION_PRINCIPLA = "IMPALA_AUTHENTICATION_PRINCIPLA";//$NON-NLS-1$
 
+    public static final String IMPALA_DRIVER = "IMPALA_DRIVER";
+
     /**
      * Google Dataproc keys.
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -166,7 +166,7 @@ public enum EDatabaseVersion4Drivers {
     REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$ //$NON-NLS-2$
             "redshift-jdbc42-no-awssdk-1.2.37.1061.jar")), //$NON-NLS-1$
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO, "redshift sso", "REDSHIFT_SSO", //$NON-NLS-1$ //$NON-NLS-2$
-            new String[] { "redshift-jdbc42-no-awssdk-1.2.37.1061.jar", "aws-java-sdk-1.11.729.jar", "jackson-core-2.10.1.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+            new String[] { "redshift-jdbc42-no-awssdk-1.2.37.1061.jar", "aws-java-sdk-1.11.848.jar", "jackson-core-2.10.1.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                     "jackson-databind-2.10.1.jar", "jackson-annotations-2.10.1.jar", "httpcore-4.4.11.jar", "httpclient-4.5.9.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
                     "joda-time-2.8.1.jar", "commons-logging-1.2.jar", "commons-codec-1.11.jar" })), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/JobContextManager.java

@@ -341,7 +341,8 @@ public class JobContextManager implements IContextManager {
 
                 String repositoryContextId = contextParamType.getRepositoryContextId();
                 String source = IContextParameter.BUILT_IN;
-                if (repositoryContextId != null && !"".equals(repositoryContextId)) { //$NON-NLS-1$
+                if (repositoryContextId != null && !"".equals(repositoryContextId) //$NON-NLS-1$
+                        && !IContextParameter.BUILT_IN.equals(repositoryContextId)) {
                     Item item = ContextUtils.getContextItemById(contextItemList, repositoryContextId);
                     if (item == null) {
                         item = ContextUtils.getRepositoryContextItemById(repositoryContextId);
@@ -585,15 +586,16 @@ public class JobContextManager implements IContextManager {
                                 contextParamType.setRepositoryContextId(contextId);
                             }
                         }
-                    } else if (useInternalId) {
+                    } else {
+                        contextParamType.setRepositoryContextId(contextParam.getSource());
+                    }
+                    if (useInternalId) {
                         String internalId = contextParam.getInternalId();
                         if (StringUtils.isEmpty(internalId)) {
                             internalId = EcoreUtil.generateUUID();
-                            contextParamType.setInternalId(internalId);
                             contextParam.setInternalId(internalId);
-                        } else {
-                            contextParamType.setInternalId(internalId);
                         }
+                        contextParamType.setInternalId(internalId);
                     }
                 }
                 contextTypeParamList.clear(); // remove old

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/link/AbstractItemContextLinkService.java

@@ -18,15 +18,18 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
+import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.context.ContextUtils;
 import org.talend.core.model.properties.ContextItem;
 import org.talend.core.model.properties.Item;
+import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.cwm.helper.ResourceHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
+import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IRepositoryService;
 
 
@@ -39,7 +42,8 @@ public abstract class AbstractItemContextLinkService implements IItemContextLink
         return saveContextLink(contextTypeList, item, backupContextLink, remoteContextLink);
     }
     
-    private synchronized boolean saveContextLink(List<ContextType> contextTypeList, Item item, ItemContextLink backupContextLink, ItemContextLink remoteContextLink)
+    public synchronized boolean saveContextLink(List<ContextType> contextTypeList, Item item,
+            ItemContextLink backupContextLink, ItemContextLink remoteContextLink)
             throws PersistenceException {
         boolean hasLinkFile = false;
         String itemId = item.getProperty().getId();
@@ -105,16 +109,26 @@ public abstract class AbstractItemContextLinkService implements IItemContextLink
         }
 
         Item contextItem = tempCache.get(repositoryContextId);
+        IRepositoryService repositoryService = GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
         if (contextItem == null) {
             contextItem = ContextUtils.getRepositoryContextItemById(repositoryContextId);
             tempCache.put(repositoryContextId, contextItem);
+            if (contextItem != null && !(contextItem instanceof ContextItem)
+                    && ProjectManager.getInstance().isInCurrentMainProject(contextItem)
+                    && checkRepoItemContextParamInternalId(contextItem)) {
+                try {
+                    repositoryService.getProxyRepositoryFactory().save(contextItem, true); // This should only using for migration phase                                                         // case the internal id is null
+                } catch (Exception ex) {
+                    ExceptionHandler.process(ex);
+                }
+            }
         }
         if (contextItem != null) {
             ContextType contextType = ContextUtils.getContextTypeByName(contextItem, contextName);
             ContextParameterType repoContextParameterType = ContextUtils.getContextParameterTypeByName(contextType, paramName);
             if (repoContextParameterType != null && repoContextParameterType.eResource() == null) {
                 // processItem save before than contextItem, caused eResource null
-                IRepositoryService repositoryService = GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
+
                 try {
                     repositoryService.getProxyRepositoryFactory().save(contextItem, false);
                 } catch (PersistenceException e) {
@@ -169,6 +183,30 @@ public abstract class AbstractItemContextLinkService implements IItemContextLink
         }
         return null;
     }
+    
+    protected boolean checkRepoItemContextParamInternalId(Item item) {
+        boolean isModified = false;
+        EList<?> contextTypeList = ContextUtils.getAllContextType(item);
+        if (contextTypeList != null) {
+            for (Object typeObj : contextTypeList) {
+                if (typeObj instanceof ContextType) {
+                    ContextType type = (ContextType) typeObj;
+                    for (Object obj : type.getContextParameter()) {
+                        if (obj instanceof ContextParameterType) {
+                            ContextParameterType contextParam = (ContextParameterType) obj;
+                            if (ContextUtils.isBuildInParameter(contextParam)
+                                    && StringUtils.isEmpty(contextParam.getInternalId())) {
+                                contextParam
+                                        .setInternalId(CoreRuntimePlugin.getInstance().getProxyRepositoryFactory().getNextId());
+                                isModified = true;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return isModified;
+    }
 }
 
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ILibrariesService.java

@@ -119,6 +119,8 @@ public interface ILibrariesService extends IService {
     List<ModuleNeeded> getModuleNeeded(String id, boolean isGroup);
 
     public void deployProjectLibrary(File source) throws IOException;
+    
+    public void setForceReloadCustomUri();
 
     /**
      * Listener used to fire that libraries status has been changed (new lib or new check install).

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/DbDefaultDatePattern.java

@@ -0,0 +1,26 @@
+package org.talend.core.model.metadata;
+
+public class DbDefaultDatePattern {
+    private String defaultPattern;
+    private String dbTypeName;
+
+    public DbDefaultDatePattern() {
+
+    }
+
+    public String getDefaultPattern() {
+        return defaultPattern;
+    }
+
+    public void setDefaultPattern(String defaultPattern) {
+        this.defaultPattern = defaultPattern;
+    }
+
+    public String getDbTypeName() {
+        return dbTypeName;
+    }
+
+    public void setDbTypeName(String dbTypeName) {
+        this.dbTypeName = dbTypeName;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/Dbms.java

@@ -41,6 +41,8 @@ public class Dbms {
 	private Set<MappingType> talendToDbTypes;
 
 	private List<DbDefaultLengthAndPrecision> defaultLengthPrecision;
+	
+	private List<DbDefaultDatePattern> defaultPattern;
 
 	private List<DbIgnoreLengthAndPrecision> ignoreLengthPrecision;
 
@@ -62,6 +64,14 @@ public class Dbms {
 			List<DbDefaultLengthAndPrecision> defaultLengthPrecision) {
 		this.defaultLengthPrecision = defaultLengthPrecision;
 	}
+	public List<DbDefaultDatePattern> getDefaultPattern() {
+        return defaultPattern;
+    }
+
+    public void setDefaultPattern(
+            List<DbDefaultDatePattern> defaultLengthPrecision) {
+        this.defaultPattern = defaultLengthPrecision;
+    }
 
 	/**
 	 * DOC amaumont Dbms constructor comment.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MappingFileLoader.java

@@ -125,10 +125,12 @@ public class MappingFileLoader {
                 // search and load preBeforelen nodes
                 ArrayList<String> dbTypes = new ArrayList<String>();
                 ArrayList<DbDefaultLengthAndPrecision> dbDefault = new ArrayList<DbDefaultLengthAndPrecision>();
+                ArrayList<DbDefaultDatePattern> dbDefaultPattern = new ArrayList<DbDefaultDatePattern>();
                 ArrayList<DbIgnoreLengthAndPrecision> dbIgnore = new ArrayList<DbIgnoreLengthAndPrecision>();
                 ArrayList<DbPreBeforeLength> dbPbeforeLList = new ArrayList<DbPreBeforeLength>();
                 dbms.setDbmsTypes(dbTypes);
                 dbms.setDefaultLengthPrecision(dbDefault);
+                dbms.setDefaultPattern(dbDefaultPattern);
                 dbms.setIgnoreLengthPrecision(dbIgnore);
                 dbms.setPrebeforelength(dbPbeforeLList);
                 List<Node> typeNodes = getChildElementNodes(dbTypesNode);
@@ -149,19 +151,25 @@ public class MappingFileLoader {
                     dbTypes.add(typeValue);
                     hAllDbTypes.add(typeValue);
                     DbDefaultLengthAndPrecision dbDefaultLP = new DbDefaultLengthAndPrecision();
+                    DbDefaultDatePattern dbDefaultPt = new DbDefaultDatePattern();
                     DbIgnoreLengthAndPrecision dbIgnoreLP = new DbIgnoreLengthAndPrecision();
                     DbPreBeforeLength dbPBeforeL = new DbPreBeforeLength();
                     // default length and precision
                     Node defaultLengthItem = typeNodeAtttributes.getNamedItem("defaultLength"); //$NON-NLS-1$
                     Node defaultPrecision = typeNodeAtttributes.getNamedItem("defaultPrecision"); //$NON-NLS-1$
+                    Node defaultPattern = typeNodeAtttributes.getNamedItem("defaultPattern"); //$NON-NLS-1$
 
                     if (defaultLengthItem != null)
                         dbDefaultLP.setDefaultLength(Integer.parseInt(defaultLengthItem.getNodeValue()));
                     if (defaultPrecision != null)
                         dbDefaultLP.setDefaultPrecision(Integer.parseInt(defaultPrecision.getNodeValue()));
+                    if (defaultPattern != null)
+                        dbDefaultPt.setDefaultPattern(defaultPattern.getNodeValue());
 
                     dbDefaultLP.setDbTypeName(typeValue);
+                    dbDefaultPt.setDbTypeName(typeValue);
                     dbDefault.add(dbDefaultLP);
+                    dbDefaultPattern.add(dbDefaultPt);
 
                     // ignore Length and Precision
                     Node ignoreLength = typeNodeAtttributes.getNamedItem("ignoreLen"); //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MappingTypeRetriever.java

@@ -191,6 +191,24 @@ public class MappingTypeRetriever {
         }// end for
         return false;
     }
+    
+    public String getDefaultPattern(String dbmsId, String dbType) {
+        Dbms dbms = MetadataTalendType.getDbms(dbmsId);
+        List<DbDefaultDatePattern> defaultPattern =  dbms.getDefaultPattern();
+        String pattern = new String(""); //$NON-NLS-1$
+        for (int i = 0; i < defaultPattern.size(); i++) {
+            DbDefaultDatePattern dbPattern = (DbDefaultDatePattern) defaultPattern.get(i);
+            if (dbPattern.getDbTypeName().equalsIgnoreCase(dbType)) {
+                pattern = dbPattern.getDefaultPattern();
+                if (pattern == null) {
+                    return "";
+                }  else {
+                    return pattern;
+                }
+            }// end if
+        }// end for
+        return pattern;
+    }
 
     private boolean isExtensionPrecisionIgnored(String dbmsId, String dbType) {
         Map<String, Map<String, List<DBTypeUtil>>> javaTypeMappingFromExtension = JavaTypesManager

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTalendType.java

@@ -13,6 +13,7 @@
 package org.talend.core.model.metadata;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -31,8 +32,11 @@ import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.ProjectScope;
+import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.Path;
@@ -488,6 +492,42 @@ public final class MetadataTalendType {
         }
     }
 
+    public static void syncNewMappingFileToProject() throws SystemException {
+        try {
+            File sysMappingFiles = new File(MetadataTalendType.getSystemForderURLOfMappingsFile().getPath());
+            IFolder projectMappingFolder = ResourceUtils.getProject(ProjectManager.getInstance().getCurrentProject())
+                    .getFolder(MetadataTalendType.PROJECT_MAPPING_FOLDER);
+            File projectMappingFiles = projectMappingFolder.getFullPath().toFile();
+            if (sysMappingFiles.list().length == new File(projectMappingFolder.getLocationURI()).list().length) {
+                return;
+            }
+
+            for (File sysMapping : sysMappingFiles.listFiles()) {
+                IFile projectMapping = projectMappingFolder.getFile(sysMapping.getName());
+                if (!projectMapping.exists()) {
+                    FileInputStream fis = null;
+                    try {
+                        fis = new FileInputStream(sysMapping);
+                        projectMapping.create(fis, true, null);
+                    } catch (CoreException coreExc) {
+                        throw new SystemException(coreExc);
+                    } finally {
+                        if (fis != null) {
+                            try {
+                                fis.close();
+                            } catch (IOException ioExc) {
+                                throw new SystemException(ioExc);
+                            }
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            throw new SystemException(e);
+        }
+    }
+
     /**
      *
      * Load db types and mapping with the current activated language (Java, Perl, ...).

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/ComponentToRepositoryProperty.java

@@ -18,6 +18,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.jface.dialogs.MessageDialog;
 import org.eclipse.swt.widgets.Display;
@@ -26,6 +27,7 @@ import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.database.EDatabase4DriverClassName;
 import org.talend.core.database.EDatabaseTypeName;
+import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.database.conn.DatabaseConnStrUtil;
 import org.talend.core.database.conn.version.EDatabaseVersion4Drivers;
 import org.talend.core.model.components.EComponentType;
@@ -70,6 +72,7 @@ import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.model.utils.IDragAndDropServiceHandler;
 import org.talend.core.runtime.i18n.Messages;
 import org.talend.core.runtime.services.IGenericDBService;
+import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.cwm.helper.ConnectionHelper;
 import org.talend.cwm.helper.PackageHelper;
@@ -365,6 +368,21 @@ public class ComponentToRepositoryProperty {
                 if (para.getRepositoryValue().endsWith(EDatabaseTypeName.GENERAL_JDBC.getProduct())) {
                     connection.setDatabaseType(EDatabaseTypeName.GENERAL_JDBC.getProduct());
                     connection.setProductId(EDatabaseTypeName.GENERAL_JDBC.getProduct());
+                    if (!node.getComponent().getDisplayName().equals(node.getComponent().getName())) {
+                        // additional JDBC e.g. Delta Lake
+                        if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericWizardService.class)) {
+                            IGenericWizardService service = GlobalServiceRegister.getDefault()
+                                    .getService(IGenericWizardService.class);
+                            if (service != null) {
+                                String database = service.getDatabseNameByNode(node);
+                                if (StringUtils.isNotBlank(database)) {
+                                    connection.setProductId(database);
+                                }
+                            }
+
+                        }
+                    }
+
                 }
                 // vertica output component have no TYPE ElementParameter .
                 if (para.getRepositoryValue().endsWith(EDatabaseTypeName.VERTICA.getProduct())) {
@@ -667,6 +685,18 @@ public class ComponentToRepositoryProperty {
                 connection.setServerName(value);
             }
         }
+        if ("IMPALA_DRIVER".equals(param.getRepositoryValue())) {
+            String value = getParameterValue(connection, node, param);
+            if (value != null) {
+                connection.getParameters().put(ConnParameterKeys.IMPALA_DRIVER, value);
+            }
+        }
+        if ("IMPALA_ADDITIONAL_JDBC".equals(param.getRepositoryValue())) {
+            String value = getParameterValue(connection, node, param);
+            if (value != null) {
+                connection.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_ADDITIONAL_JDBC_SETTINGS, value);
+            }
+        }
 
         if (connection.getDatabaseType().equals(EDatabaseTypeName.ORACLEFORSID.getDisplayName())) {
             setDatabaseValueForOracleSid(connection, node, param);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -1755,8 +1755,18 @@ public class RepositoryToComponentProperty {
         if (StringUtils.equals("MAPPING", value)) {//$NON-NLS-1$
             return connection.getDbmsId();
         }
+        if ("IMPALA_ADDITIONAL_JDBC".equals(value)) { //$NON-NLS-1$
+            String additionJdbc = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_ADDITIONAL_JDBC_SETTINGS);
+            if (isContextMode(connection, additionJdbc)) {
+                return additionJdbc;
+            } else {
+                return TalendQuoteUtils.addQuotes(additionJdbc);
+            }
+        }
+        if ("IMPALA_DRIVER".equals(value)) {
+            return connection.getParameters().get(ConnParameterKeys.IMPALA_DRIVER);
+        }
         return null;
-
     }
 
     private static String getAppropriateValue(Connection connection, String rawValue) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/types/ContextParameterJavaTypeManager.java

@@ -240,6 +240,11 @@ public class ContextParameterJavaTypeManager {
         if (javaType == null) {
             return null;
         }
+        
+        if(javaType.equals(JavaTypesManager.PASSWORD)) {
+        	return "Password";
+        }
+        
         Class primitiveClass = javaType.getPrimitiveClass();
         Class nullableClass = javaType.getNullableClass();
         if (nullable) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/AbsJobletReplaceNodeHandler.java

@@ -0,0 +1,25 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public abstract class AbsJobletReplaceNodeHandler extends AbsReplaceNodeHandler implements IJobletReplaceNodeHandler {
+
+    public AbsJobletReplaceNodeHandler(String prefix) {
+        super();
+        this.setPrefix(prefix);
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/AbsReplaceNodeHandler.java

@@ -0,0 +1,31 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+/**
+ * DOC cmeng class global comment. Detailled comment
+ */
+public abstract class AbsReplaceNodeHandler implements IReplaceNodeHandler {
+
+    private String prefix;
+
+    @Override
+    public String getPrefix() {
+        return prefix;
+    }
+
+    public void setPrefix(String prefix) {
+        this.prefix = prefix;
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/AbstractNode.java

@@ -122,6 +122,8 @@ public abstract class AbstractNode implements INode {
     // for MR, tag this component is the ref(lookup) start node
     private boolean isRefNode = false;
 
+    private IReplaceNodeHandler replaceNodeHandler;
+
     public String getComponentName() {
         return componentName;
     }
@@ -1322,6 +1324,15 @@ public abstract class AbstractNode implements INode {
         return componentProperties;
     }
 
+    @Override
+    public IReplaceNodeHandler getReplaceNodeHandler() {
+        return replaceNodeHandler;
+    }
+
+    public void setReplaceNodeHandler(IReplaceNodeHandler replaceNodeHandler) {
+        this.replaceNodeHandler = replaceNodeHandler;
+    }
+
     public INode getRealGraphicalNode() {
         return realGraphicalNode;
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IJobletReplaceNodeHandler.java

@@ -0,0 +1,20 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public interface IJobletReplaceNodeHandler extends IReplaceNodeHandler {
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/INode.java

@@ -312,6 +312,10 @@ public interface INode extends IElement {
 
     public boolean isSubtreeStart();
 
+    default public IReplaceNodeHandler getReplaceNodeHandler() {
+        return null;
+    }
+
     public void setComponentProperties(ComponentProperties props);
 
     public ComponentProperties getComponentProperties();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IReplaceNodeHandler.java

@@ -0,0 +1,22 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public interface IReplaceNodeHandler {
+
+    String getPrefix();
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/JobletReplaceNodeHandler.java

@@ -0,0 +1,25 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.process;
+
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public class JobletReplaceNodeHandler extends AbsJobletReplaceNodeHandler {
+
+    public JobletReplaceNodeHandler(String prefix) {
+        super(prefix);
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/relationship/RelationshipItemBuilder.java

@@ -43,6 +43,7 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.PropertiesFactory;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.model.routines.RoutinesUtil;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.i18n.Messages;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
@@ -50,6 +51,7 @@ import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IProxyRepositoryFactory;
+import org.talend.repository.model.IRepositoryNode;
 
 /**
  * This class store all relationships between jobs/joblets and other items from the repository. Be sure to update the
@@ -130,6 +132,8 @@ public class RelationshipItemBuilder {
 
     private Map<Relation, Set<Relation>> referencesItemsRelations;
 
+    private Map<String, String> systemRoutinesMap;
+
     private boolean loaded = false;
 
     private boolean loading = false;
@@ -219,6 +223,28 @@ public class RelationshipItemBuilder {
         this.proxyRepositoryFactory = proxyRepositoryFactory;
     }
 
+    /**
+     * 
+     * Current System Routines Map, key: routine id, value: routine name
+     * 
+     * @return
+     */
+    public Map<String, String> getCurrentSystemRoutinesMap() {
+        if (systemRoutinesMap != null && !systemRoutinesMap.isEmpty()) {
+            return systemRoutinesMap;
+        }
+
+        if (systemRoutinesMap == null) {
+            systemRoutinesMap = new HashMap<String, String>();
+        }
+        List<IRepositoryViewObject> currentSystemRoutines = RoutinesUtil.getCurrentSystemRoutines();
+        for (IRepositoryViewObject object : currentSystemRoutines) {
+            systemRoutinesMap.put(object.getProperty().getId(), object.getProperty().getLabel());
+        }
+
+        return systemRoutinesMap;
+    }
+
     /**
      * Look for every linked items who use the selected id, no matter the version. Usefull when want to delete an item
      * since it will delete every versions.
@@ -468,7 +494,7 @@ public class RelationshipItemBuilder {
     }
 
     private Set<Relation> getItemsRelatedTo(Map<Relation, Set<Relation>> itemsRelations, String itemId, String version,
-            String relationType) {
+                String relationType) {
 
         Relation itemToTest = new Relation();
         itemId = ProcessUtils.getPureItemId(itemId);
@@ -904,7 +930,10 @@ public class RelationshipItemBuilder {
         }
     }
 
-    private void addRelationShip(Item baseItem, String relatedId, String relatedVersion, String type) {
+    public void addRelationShip(Item baseItem, String relatedId, String relatedVersion, String type) {
+        if (!loaded) {
+            loadRelations();
+        }
         Relation relation = new Relation();
         relation.setId(baseItem.getProperty().getId());
         relation.setType(getTypeFromItem(baseItem));
@@ -920,8 +949,32 @@ public class RelationshipItemBuilder {
             itemRelations.put(relation, new HashSet<Relation>());
         }
         itemRelations.get(relation).add(addedRelation);
+        
+        autoSaveRelations();
     }
 
+	public Set<Relation> getBeanRelations(Collection<Item> items) {
+		Set<Relation> relationships = new HashSet<Relation>();
+		for (Item item : items) {
+	        Relation relation = new Relation();
+	        relation.setId(item.getProperty().getId());
+	        relation.setType(getTypeFromItem(item));
+	        relation.setVersion(item.getProperty().getVersion());
+	        
+	        Map<Relation, Set<Relation>> itemRelations = getRelatedRelations(item);
+	        
+	        Set<Relation> repositoryNode = (Set<Relation>) itemRelations.get(relation);
+	        
+	        for (Relation rel : repositoryNode) {
+	        	if (rel.getType().equals("Beans")) {
+	        		relationships.add(rel);
+	        	}
+	        }
+		}
+		
+		return relationships;
+	}
+
     private Map<Relation, Set<Relation>> getRelatedRelations(Item baseItem) {
         Map<Relation, Set<Relation>> itemRelations = currentProjectItemsRelations;
         if (!ProjectManager.getInstance().isInMainProject(getAimProject(), baseItem)) {
@@ -937,18 +990,26 @@ public class RelationshipItemBuilder {
 
     private void buildIndex(Map<Relation, Set<Relation>> itemRelations, Project project, IProgressMonitor monitor) {
         modified = true;
-
         if (!project.getEmfProject().getItemsRelations().isEmpty()) {
             loadRelations(itemRelations, project);
             if (loaded) { // check if already loaded successfully
                 return;
             }
         }
+        generateIndex(project, getTypes(), true, monitor);
+        if (modified) {
+            autoSaveRelations();
+        }
+        monitor.done();
+        loaded = true;
+    }
 
+    private void generateIndex(Project project, List<ERepositoryObjectType> supportedTypes, boolean fromMigration,
+            IProgressMonitor monitor) {
         IProxyRepositoryFactory factory = getProxyRepositoryFactory();
         List<IRepositoryViewObject> list = new ArrayList<IRepositoryViewObject>();
         try {
-            for (ERepositoryObjectType curTyp : getTypes()) {
+            for (ERepositoryObjectType curTyp : supportedTypes) {
                 if (curTyp != null) {
                     list.addAll(factory.getAll(project, curTyp, true, true));
                 }
@@ -962,21 +1023,44 @@ public class RelationshipItemBuilder {
             for (IRepositoryViewObject object : list) {
                 Item item = object.getProperty().getItem();
                 monitor.subTask(Messages.getString("RelationshipItemBuilder.forItem") + item.getProperty().getLabel()); //$NON-NLS-1$
-                addOrUpdateItem(item, true);
+                findRelationItems(item, fromMigration);
                 monitor.worked(1);
                 if (monitor.isCanceled()) {
                     return;
                 }
             }
-            autoSaveRelations();
-            monitor.done();
-            loaded = true;
-
         } catch (PersistenceException e) {
             ExceptionHandler.process(e);
         }
     }
 
+    public void buildAndSaveIndex() {
+        log.info("relationship.index generating");
+        currentProjectItemsRelations = new ConcurrentHashMap<Relation, Set<Relation>>();
+        referencesItemsRelations = new ConcurrentHashMap<Relation, Set<Relation>>();
+        generateIndex(this.aimProject, allSupportedTypes(), false, new NullProgressMonitor());
+        try {
+            // sync to project
+            synchronizeItemRelationToProject(this.aimProject);
+            // persist index
+            getProxyRepositoryFactory().saveProject(this.aimProject);
+            log.info("relationship.index generated");
+        } catch (PersistenceException e) {
+            log.error("relationship.index generating error", e);
+            ExceptionHandler.process(e);
+        }
+    }
+    
+    private List<ERepositoryObjectType> allSupportedTypes() {
+        List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfProcess());
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfProcess2());
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfTestContainer());
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfCodes());
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfJoblet());
+        return toReturn;
+    }
+
     private List<ERepositoryObjectType> getTypes() {
         List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
         toReturn.add(ERepositoryObjectType.PROCESS);
@@ -1098,14 +1182,24 @@ public class RelationshipItemBuilder {
     }
 
     public void addOrUpdateItem(Item item, boolean fromMigration) {
-        if (!supportRelation(item)) {
-            return;
-        }
         if (!loaded) {
             loadRelations();
         }
+        boolean relationsModified = findRelationItems(item, fromMigration);
+        if (relationsModified && !modified) {
+            modified = true;
+        }
+        if (!fromMigration && modified) {
+            autoSaveRelations();
+        }
+    }
+
+    private boolean findRelationItems(Item item, boolean fromMigration) {
+        if (!supportRelation(item)) {
+            return false;
+        }
         if (item == null) {
-            return;
+            return false;
         }
 
         boolean relationsModified = true;
@@ -1149,12 +1243,7 @@ public class RelationshipItemBuilder {
                 currentProjectItemsRelations.get(relation).addAll(oldProjectRelations);
             }
         }
-        if (relationsModified && !modified) {
-            modified = true;
-        }
-        if (!fromMigration && modified) {
-            autoSaveRelations();
-        }
+        return relationsModified;
     }
 
     public Set<Relation> getItemRelations(Item item) {
@@ -1433,4 +1522,5 @@ public class RelationshipItemBuilder {
             return 0;
         }
     }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/NodeUtil.java

@@ -46,7 +46,11 @@ import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.INodeConnector;
 import org.talend.core.model.process.IProcess;
+import org.talend.core.model.process.IProcess2;
+import org.talend.core.model.process.ProcessUtils;
+import org.talend.core.model.properties.Property;
 import org.talend.core.runtime.IAdditionalInfo;
+import org.talend.core.runtime.projectsetting.RuntimeLineageManager;
 import org.talend.designer.core.ICamelDesignerCoreService;
 
 /**
@@ -866,6 +870,170 @@ public class NodeUtil {
         }
         throw new IllegalArgumentException();
     }
+    
+    
+    public static String getRuntimeParameterValue(INode node, IElementParameter ep) {
+        if (EParameterFieldType.TABLE.equals(ep.getFieldType())) {
+            Map<String, IElementParameter> types = new HashMap<String, IElementParameter>();
+            Object[] itemsValue = ep.getListItemsValue();
+            if (itemsValue != null) {
+                for (Object o : itemsValue) {
+                    IElementParameter cep = (IElementParameter) o;
+                    if (cep.isShow(node.getElementParameters())) {
+                        types.put(cep.getName(), cep);
+                    }
+                }
+            }
+            List<Map<String, String>> lines = (List<Map<String, String>>) ElementParameterParser.getObjectValue(node,
+                    "__" + ep.getName() + "__");
+            StringBuilder value = new StringBuilder();
+            // implement List & Map toString(), different is the value of Map
+            Iterator<Map<String, String>> linesIter = lines.iterator();
+            if (!linesIter.hasNext()) {
+                return "\"[]\"";
+            }
+            value.append("new StringBuilder().append(\"[");
+            for (;;) {
+                Map<String, String> columns = linesIter.next();
+                Iterator<Entry<String, String>> columnsIter = columns.entrySet().iterator();
+
+                value.append("{");
+                Entry<String, String> column = null;
+                boolean printedColumnExist = false;
+                while (columnsIter.hasNext()) {
+                    column = columnsIter.next();
+                    if (types.get(column.getKey()) == null) {
+                        continue;
+                    }
+                    printedColumnExist = true;
+
+                    value.append(column.getKey());
+                    value.append("=\").append(");
+                    value.append(getRuntimeParameterValue(column.getValue(), types.get(column.getKey()), true));
+                    value.append(").append(\"");
+
+                    if (columnsIter.hasNext()) {
+                        value.append(", ");
+                    }
+                }
+                if (printedColumnExist && column != null && (types.get(column.getKey()) == null)) {
+                    value.setLength(value.length() - 2);
+                }
+                value.append("}");
+
+                if (!linesIter.hasNext()) {
+                    return value.append("]\").toString()").toString();
+                }
+                value.append(",").append(" ");
+            }
+        } else {
+            String value = ElementParameterParser.getValue(node, "__" + ep.getName() + "__");
+            if (EParameterFieldType.TABLE_BY_ROW.equals(ep.getFieldType())) {
+                value = ep.getValue().toString();
+            }
+            return getRuntimeParameterValue(value, ep, false);
+        }
+    }
+    
+    private static String getRuntimeParameterValue(String value, IElementParameter ep, boolean itemFromTable) {
+        if (value == null) {
+            value = "";
+        }
+        
+        value = value.trim();
+        
+        boolean isMemo = false;
+        
+        List<EParameterFieldType> needRemoveCRLFList = Arrays.asList(EParameterFieldType.MEMO, EParameterFieldType.MEMO_JAVA,
+                EParameterFieldType.MEMO_SQL, EParameterFieldType.MEMO_IMPORT, EParameterFieldType.MEMO_MESSAGE);
+        if (needRemoveCRLFList.contains(ep.getFieldType())) {
+            isMemo = true;
+            value = value.replaceAll("[\r\n]", " ");
+        }
+        
+        List<EParameterFieldType> needQuoteList = Arrays.asList(EParameterFieldType.CLOSED_LIST,
+                EParameterFieldType.COMPONENT_LIST, EParameterFieldType.COLUMN_LIST, EParameterFieldType.PREV_COLUMN_LIST,
+                EParameterFieldType.CONNECTION_LIST, EParameterFieldType.LOOKUP_COLUMN_LIST,
+                EParameterFieldType.CONTEXT_PARAM_NAME_LIST, EParameterFieldType.PROCESS_TYPE, EParameterFieldType.COLOR,
+                EParameterFieldType.TABLE_BY_ROW, EParameterFieldType.HADOOP_JARS_DIALOG, EParameterFieldType.UNIFIED_COMPONENTS);
+        List<EParameterFieldType> needQuoteListForItem = itemFromTable ? Arrays.asList(EParameterFieldType.SCHEMA_TYPE,
+                EParameterFieldType.SAP_SCHEMA_TYPE, EParameterFieldType.MODULE_LIST) : new ArrayList<EParameterFieldType>();
+        // TODO: add RAW attribute when SCHEMA_COLUMN generated by BASED_ON_SCHEMA
+        List<String> needQuoteListByName = Arrays.asList("SCHEMA_COLUMN");// SCHEMA_COLUMN for BASED_ON_SCHEMA="true"
+
+        if (needQuoteList.contains(ep.getFieldType()) || needQuoteListForItem.contains(ep.getFieldType())
+                || needQuoteListByName.contains(ep.getName()) || ep.isRaw()) {
+            value = value.replaceAll("\\\\", "\\\\\\\\");
+            value = value.replaceAll("\\\"", "\\\\\\\"");
+            return "\"" + value + "\"";
+        }
+
+        if (itemFromTable) {
+            if ("*".equals(value)) {
+                return "\"" + value + "\"";
+            }
+            if (value.endsWith(";")) {
+                value = value.substring(0, value.length() - 1);
+            }
+        }
+        
+        if("".equals(value) || "\"\"".equals(value)) {
+            return "\"\"";
+        } else if("null".equals(value)) {
+            return "(Object)null";
+        }
+        
+        // copied it from Log4jFileUtil.javajet but need more comment for this script
+        if ("\"\\n\"".equals(value) || "\"\\r\"".equals(value) || "\"\\r\\n\"".equals(value)) {
+            // for the value is "\n" "\r" "\r\n"
+            return value.replaceAll("\\\\", "\\\\\\\\");
+        } else if ("\"\"\"".equals(value)) {
+            return "\"" + "\\" + "\"" + "\"";
+        } else if ("\"\"\\r\\n\"\"".equals(value)) {
+            return "\"\\\\r\\\\n\"";
+        } else if ("\"\"\\r\"\"".equals(value)) {
+            return "\"\\\\r\"";
+        } else if ("\"\"\\n\"\"".equals(value)) {
+            return "\"\\\\n\"";
+        }
+        // ftom 20141008 - patch to fix javajet compilation errors due to hard-coded studio TableEditor mechanism
+        // linked to BUILDIN properties checks, this item is a boolean set to TRUE or FALSE
+        // fix is just transforming into true or false to make logging OK
+        else if ("BUILDIN".equals(ep.getName())) {
+            return value.toLowerCase();
+        }
+        
+        //suppose all memo fields are processed well already, no need to go though this with dangerous
+        if (!isMemo && !org.talend.core.model.utils.ContextParameterUtils.isDynamic(value)) {
+            if(value.length() > 1 && value.startsWith("\"") && value.endsWith("\"")) {
+                if(itemFromTable && "ARGS".equals(ep.getName())) {
+                    value = value.substring(1, value.length());
+                    value = value.substring(0, value.length() - 1);
+                    return "\"" + checkStringQuotationMarks(value) + "\"";
+                } else {
+                    //do nothing
+                    return value;
+                }
+            } else {
+            	return "\"" + checkStringQuotationMarks(value) + "\"";
+            }
+        }
+        
+        //TODO remove it
+        if (value.endsWith("*")) {
+            return value.substring(0, value.length() - 1) + "\"*\"";
+        }
+        
+        return value;
+    }
+    
+    private static String checkStringQuotationMarks(String str) {
+        String result = str;
+        if (result.contains("\"")) {
+            result = result.replace("\"", "\\\"");
+        }
+        return result;
+    }
 
     public static String getNormalizeParameterValue(INode node, IElementParameter ep) {
         if (EParameterFieldType.TABLE.equals(ep.getFieldType())) {
@@ -1189,7 +1357,7 @@ public class NodeUtil {
         } else if (ComponentCategory.CATEGORY_4_CAMEL.getName().equals(node.getComponent().getType())) {
             INodeConnector tmp = null;
             if (GlobalServiceRegister.getDefault().isServiceRegistered(ICamelDesignerCoreService.class)) {
-                ICamelDesignerCoreService camelService = (ICamelDesignerCoreService) GlobalServiceRegister.getDefault()
+                ICamelDesignerCoreService camelService = GlobalServiceRegister.getDefault()
                         .getService(ICamelDesignerCoreService.class);
                 tmp = node.getConnectorFromType(camelService.getTargetConnectionType(node));
             } else {
@@ -1232,4 +1400,37 @@ public class NodeUtil {
 
         return true;
     }
+
+    public static boolean isJobUsingRuntimeLineage(IProcess process) {
+        // Just support DI jobs now
+        boolean isSupport = isStandardJob(process) && !ProcessUtils.isTestContainer(process) && !isGuessSchemaJob(process);
+        if (!isSupport) {
+            return false;
+        }
+        RuntimeLineageManager runtimeLineageManager = new RuntimeLineageManager();
+        if (runtimeLineageManager.isUseRuntimeLineageAll()) {
+            return true;
+        }
+        if (runtimeLineageManager.getSelectedJobIds().isEmpty()) {
+            runtimeLineageManager.load();
+        }
+        return runtimeLineageManager.isRuntimeLineageSetting(process.getId());
+    }
+
+    public static boolean isStandardJob(IProcess process) {
+        if (process != null && process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            return property != null && property.getItem() != null
+                    && ComponentCategory.CATEGORY_4_DI.getName().equals(process.getComponentsType());
+        }
+        return false;
+    }
+
+    public static boolean isGuessSchemaJob(IProcess process) {
+        if (process != null && process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            return property != null && "ID".equals(property.getId()) && "Mock_job_for_Guess_schema".equals(property.getLabel()); //$NON-NLS-1$ //$NON-NLS-2$
+        }
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/evaluator/AbstractPropertyValueEvaluator.java

@@ -15,6 +15,7 @@ package org.talend.core.runtime.evaluator;
 import java.util.List;
 
 import org.apache.commons.lang3.StringEscapeUtils;
+import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.runtime.util.GenericTypeUtils;
 import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.daikon.properties.property.Property;
@@ -28,6 +29,10 @@ import org.talend.daikon.properties.property.StringProperty;
 public abstract class AbstractPropertyValueEvaluator implements PropertyValueEvaluator {
 
     public Object getTypedValue(Property property, Object rawValue) {
+        return getTypedValue(property, null, rawValue);
+    }
+
+    public Object getTypedValue(Property property, Object storedValue, Object rawValue) {
         if (GenericTypeUtils.isSchemaType(property)) {
             return rawValue;
         }
@@ -105,6 +110,12 @@ public abstract class AbstractPropertyValueEvaluator implements PropertyValueEva
                 String stringStoredValue = TalendQuoteUtils.removeQuotes(stringValue);
                 for (Object possibleValue : possibleValues) {
                     if (possibleValue.toString().equals(stringStoredValue)) {
+                        // Update since enum type set as context for tcompv0 .
+                        String currentStoredValue = String.valueOf(storedValue);
+                        if (storedValue != null && ContextParameterUtils.isContainContextParam(currentStoredValue)) {
+                            property.setTaggedValue("IS_CONTEXT_MODE", false);
+                            property.setValue(possibleValue);
+                        }
                         return possibleValue;
                     }
                 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -44,6 +44,8 @@ public interface MavenConstants {
 
     static final String USE_PROFILE_MODULE = "USE_PROFILE_MODULE";
 
+    static final String EXCLUDE_DELETED_ITEMS = "EXCLUDE_DELETED_ITEMS";
+
     /*
      * for lib
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessArgumentConstant.java

@@ -64,6 +64,8 @@ public interface TalendProcessArgumentConstant {
 
     static final String ARG_AVOID_BRANCH_NAME = "AVOID_BRANCH_NAME";
 
+    static final String ARG_CLEAR_PASSWORD_CONTEXT_PARAMETERS = "CLEAR_PASSWORD_CONTEXT_PARAMETERS";
+
     static final String CMD_ARG_STATS_PORT_PARAM = "stat_port";
 
     static final String CMD_ARG_TRACE_PORT_PARAM = "trace_port";

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/projectsetting/RuntimeLineageManager.java

@@ -0,0 +1,156 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.runtime.projectsetting;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IFolder;
+import org.eclipse.core.resources.IProject;
+import org.talend.commons.exception.PersistenceException;
+import org.talend.commons.ui.runtime.exception.ExceptionHandler;
+import org.talend.commons.utils.workbench.resources.ResourceUtils;
+import org.talend.repository.ProjectManager;
+import org.talend.repository.model.RepositoryConstants;
+import org.talend.repository.model.RepositoryNode;
+import org.talend.utils.json.JSONObject;
+
+import us.monoid.json.JSONArray;
+
+/**
+ * created by hcyi on Jul 27, 2020
+ * Detailled comment
+ *
+ */
+public class RuntimeLineageManager {
+
+    public static final String RUNTIMELINEAGE_RESOURCES = "org.talend.runtimelineage"; //$NON-NLS-1$
+
+    public static final String RUNTIMELINEAGE_ALL = "runtimelineage.all"; //$NON-NLS-1$
+
+    public static final String RUNTIMELINEAGE_SELECTED = "runtimelineage.selected"; //$NON-NLS-1$
+
+    public static final String JOB_ID = "id"; //$NON-NLS-1$
+
+    public static final String RUNTIMELINEAGE_OUTPUT_PATH = "-Druntime.lineage.outputpath="; //$NON-NLS-1$
+
+    public static final String OUTPUT_PATH = "output.path"; //$NON-NLS-1$
+
+    private List<String> selectedJobIds = new ArrayList<String>();
+
+    private ProjectPreferenceManager prefManager = null;
+
+    private boolean useRuntimeLineageAll = false;
+
+    private String outputPath = null;
+
+    public RuntimeLineageManager() {
+        if (prefManager == null) {
+            prefManager = new ProjectPreferenceManager(RUNTIMELINEAGE_RESOURCES, true);
+        }
+        useRuntimeLineageAll = prefManager.getBoolean(RUNTIMELINEAGE_ALL);
+        outputPath = prefManager.getValue(OUTPUT_PATH);
+    }
+
+    public void load() {
+        try {
+            String jobsJsonStr = prefManager.getValue(RUNTIMELINEAGE_SELECTED);
+            if (StringUtils.isNotBlank(jobsJsonStr)) {
+                JSONArray jobsJsonArray = new JSONArray(jobsJsonStr);
+                for (int i = 0; i < jobsJsonArray.length(); i++) {
+                    Object jobJsonObj = jobsJsonArray.get(i);
+                    JSONObject jobJson = new JSONObject(String.valueOf(jobJsonObj));
+                    Iterator sortedKeys = jobJson.sortedKeys();
+                    String jobId = null;
+                    while (sortedKeys.hasNext()) {
+                        String key = (String) sortedKeys.next();
+                        if (JOB_ID.equals(key)) {
+                            jobId = jobJson.getString(key);
+                        }
+                    }
+                    if (jobId != null) {
+                        selectedJobIds.add(jobId);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    public void save(List<RepositoryNode> checkedObjects, boolean all) {
+        try {
+            JSONArray jobsJson = new JSONArray();
+            if (!all) {
+                for (RepositoryNode node : checkedObjects) {
+                    JSONObject jobJson = new JSONObject();
+                    if (!jobsJson.toString().contains(node.getId())) {
+                        jobJson.put(JOB_ID, node.getId());
+                        jobsJson.put(jobJson);
+                    }
+                }
+            }
+            prefManager.setValue(RUNTIMELINEAGE_ALL, all);
+            prefManager.setValue(RUNTIMELINEAGE_SELECTED, jobsJson.toString());
+            prefManager.setValue(OUTPUT_PATH, outputPath);
+            prefManager.save();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    public boolean isRuntimeLineageSetting(String id) {
+        return selectedJobIds.contains(id);
+    }
+
+    public boolean isRuntimeLineagePrefsExist() {
+        try {
+            IProject project = ResourceUtils.getProject(ProjectManager.getInstance().getCurrentProject());
+            IFolder prefSettingFolder = ResourceUtils.getFolder(project, RepositoryConstants.SETTING_DIRECTORY, false);
+            IFile presRuntimeLineageFile = prefSettingFolder.getFile(RUNTIMELINEAGE_RESOURCES + ".prefs"); //$NON-NLS-1$
+            if (presRuntimeLineageFile.exists()) {
+                return true;
+            }
+        } catch (PersistenceException e) {
+            e.printStackTrace();
+        }
+        return false;
+    }
+
+    public boolean isUseRuntimeLineageAll() {
+        return this.useRuntimeLineageAll;
+    }
+
+    public ProjectPreferenceManager getPrefManager() {
+        return this.prefManager;
+    }
+
+    public List<String> getSelectedJobIds() {
+        return this.selectedJobIds;
+    }
+
+    public void setSelectedJobIds(List<String> selectedJobIds) {
+        this.selectedJobIds = selectedJobIds;
+    }
+
+    public String getOutputPath() {
+        return this.outputPath;
+    }
+
+    public void setOutputPath(String outputPath) {
+        this.outputPath = outputPath;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericService.java

@@ -18,6 +18,7 @@ import java.util.Map;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
 import org.talend.core.model.components.IComponent;
+import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 
@@ -46,6 +47,8 @@ public interface IGenericService extends IService {
 
     public boolean isTcompv0(IComponent component);
 
+    public void validateGenericConnection(Connection conn) throws Exception;
+
     public static IGenericService getService() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericService.class)) {
             return GlobalServiceRegister.getDefault().getService(IGenericService.class);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericWizardService.java

@@ -22,9 +22,11 @@ import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.IService;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.connection.Connection;
+import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.EComponentCategory;
 import org.talend.core.model.process.Element;
+import org.talend.core.model.process.IElement;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.repository.ERepositoryObjectType;
@@ -146,4 +148,14 @@ public interface IGenericWizardService extends IService {
      * @return the default action which will be invoked when double click the node.
      */
     public ITreeContextualAction getDefaultAction(RepositoryNode node);
+
+    public void initAdditionalJDBCRepositoryObjType();
+
+    public boolean getIfAdditionalJDBCDBType(String dbType);
+
+    public void initAdditonalJDBCConnectionValue(DatabaseConnection connection, Composite dynamicForm, String dbType,
+            String propertyId);
+
+    public String getDatabseNameByNode(IElement node);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -246,6 +246,8 @@ public interface IRunProcessService extends IService {
 
     public boolean isCIMode();
 
+    public boolean isExcludeDeletedItems(Property property);
+
     public static IRunProcessService get() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             return GlobalServiceRegister.getDefault().getService(IRunProcessService.class);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/login/AbstractLoginTask.java

@@ -42,4 +42,9 @@ public abstract class AbstractLoginTask implements ILoginTask {
         return gc.getTime();
     }
 
+    @Override
+    public boolean isRequiredAlways() {
+        return false;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/login/ILoginTask.java

@@ -27,4 +27,10 @@ public interface ILoginTask {
     public boolean isCommandlineTask();
 
     public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException;
+
+    /**
+     * Which indicates the task will be executed for each logon of a project, by default return false(execute only once
+     * at the time of logon studio).
+     */
+    boolean isRequiredAlways();
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/IProxyRepositoryFactory.java

@@ -268,12 +268,20 @@ public interface IProxyRepositoryFactory {
 
     public abstract IRepositoryViewObject getLastVersion(Project project, String id, String relativeFolder,
             ERepositoryObjectType type) throws PersistenceException;
+    
+    public abstract IRepositoryViewObject getLastVersion(String id, String folderPath, ERepositoryObjectType type)  throws PersistenceException;
+    
+    public abstract IRepositoryViewObject getLastVersion(String id, ERepositoryObjectType type)  throws PersistenceException;
+    
+    public abstract IRepositoryViewObject getLastVersion(String id, List<ERepositoryObjectType> types)  throws PersistenceException;
 
     public abstract IRepositoryViewObject getLastVersion(Project project, String id) throws PersistenceException;
 
     public abstract IRepositoryViewObject getLastVersion(String id) throws PersistenceException;
 
     public IRepositoryViewObject getLastRefVersion(Project project, String id) throws PersistenceException;
+    
+    public IRepositoryViewObject getLastRefVersion(Project project, String id, String folderPath, ERepositoryObjectType type) throws PersistenceException;
 
     public abstract IRepositoryViewObject getSpecificVersion(Project project, String id, String version, boolean avoidSaveProject)
             throws PersistenceException;

main/plugins/org.talend.core.ui/.classpath

@@ -1,5 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
+	<classpathentry exported="true" kind="lib" path="lib/jna-platform.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/jna.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/oshi-core.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
 	<classpathentry kind="src" path="src/main/java"/>

main/plugins/org.talend.core.ui/META-INF/MANIFEST.MF

@@ -31,7 +31,10 @@ Require-Bundle: org.apache.commons.lang,
  org.talend.themes.core,
  ca.odell.glazedlists,
  org.talend.core,
- org.apache.commons.io
+ org.apache.commons.io,
+ org.apache.httpcomponents.httpcore,
+ org.apache.httpcomponents.httpclient,
+ org.slf4j.api
 Import-Package: org.eclipse.jdt.internal.ui.workingsets
 Export-Package: org.talend.core.ui,
  org.talend.core.ui.actions,
@@ -79,3 +82,7 @@ Bundle-Vendor: .Talend SA.
 Bundle-ActivationPolicy: lazy
 Bundle-Activator: org.talend.core.ui.CoreUIPlugin
 Bundle-Localization: plugin
+Bundle-ClassPath: lib/jna-platform.jar,
+ lib/jna.jar,
+ lib/oshi-core.jar,
+ .

main/plugins/org.talend.core.ui/build.properties

@@ -5,4 +5,7 @@ bin.includes = .,\
                icons/,\
                plugin.properties,\
                schema/,\
-               META-INF/
+               META-INF/,\
+               lib/jna-platform.jar,\
+               lib/jna.jar,\
+               lib/oshi-core.jar

main/plugins/org.talend.core.ui/plugin.xml

@@ -24,6 +24,12 @@
             id="org.talend.core.runtime.defaultProvider"
             name="default">
       </provider>
+      <provider
+            collector="org.talend.core.ui.token.PerformanceTokenCollector"
+            description="collect such as hardware info, I/O info, startup time"
+            id="org.talend.core.ui.token.PerformanceProvider"
+            name="performance">
+      </provider>
    </extension>
    
       <extension

main/plugins/org.talend.core.ui/pom.xml

@@ -9,4 +9,42 @@
   </parent>
   <artifactId>org.talend.core.ui</artifactId>
   <packaging>eclipse-plugin</packaging>
+  
+  <dependencies>
+  	<dependency>
+	  	<groupId>com.github.oshi</groupId>
+		<artifactId>oshi-core</artifactId>
+		<version>5.2.5</version>
+		<exclusions>
+			<exclusion>
+				<groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+			</exclusion>
+		</exclusions>
+	 </dependency>
+  </dependencies>
+  
+  <build>
+     	<plugins>
+	      <plugin>
+	        <groupId>org.apache.maven.plugins</groupId>
+	        <artifactId>maven-dependency-plugin</artifactId>
+	        <version>3.0.2</version>
+	        <executions>
+	          <execution>
+	            <id>copy-dependencies</id>
+	            <phase>generate-resources</phase>
+	            <goals>
+	              <goal>copy-dependencies</goal>
+	            </goals>
+	            <configuration>
+	              <includeScope>runtime</includeScope>
+	              <outputDirectory>${project.basedir}/lib</outputDirectory>
+	              <stripVersion>true</stripVersion>
+	            </configuration>
+	          </execution>
+	        </executions>
+	      </plugin>
+      </plugins>
+     </build>
 </project>

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/component/settings/ComponentsSettingsHelper.java

@@ -75,9 +75,9 @@ public class ComponentsSettingsHelper {
                 hiddenComponents.put(component.getPaletteType(), new HashMap<String, Set<String>>());
             }
             if (!component.isVisibleInComponentDefinition()) {
-                hiddenComponents.get(component.getPaletteType()).put(component.getName(), new HashSet<String>());
+                hiddenComponents.get(component.getPaletteType()).put(component.getDisplayName(), new HashSet<String>());
                 for (String family : component.getOriginalFamilyName().split(ComponentsFactoryProvider.FAMILY_SEPARATOR_REGEX)) {
-                    hiddenComponents.get(component.getPaletteType()).get(component.getName()).add(family);
+                    hiddenComponents.get(component.getPaletteType()).get(component.getDisplayName()).add(family);
                 }
             }
         }
@@ -168,8 +168,8 @@ public class ComponentsSettingsHelper {
         }
 
         if (hiddenComponents.containsKey(component.getPaletteType())) {
-            if (hiddenComponents.get(component.getPaletteType()).containsKey(component.getName())) {
-                if (hiddenComponents.get(component.getPaletteType()).get(component.getName()).contains(family)) {
+            if (hiddenComponents.get(component.getPaletteType()).containsKey(component.getDisplayName())) {
+                if (hiddenComponents.get(component.getPaletteType()).get(component.getDisplayName()).contains(family)) {
                     // not visible if in the hidden component list;
                     return false;
                 }
@@ -225,7 +225,7 @@ public class ComponentsSettingsHelper {
         EList list = emfProject.getComponentsSettings();
         if (!list.isEmpty()) {
             list.clear();
-            IRepositoryService service = (IRepositoryService) GlobalServiceRegister.getDefault().getService(
+            IRepositoryService service = GlobalServiceRegister.getDefault().getService(
                     IRepositoryService.class);
 
             IProxyRepositoryFactory prf = service.getProxyRepositoryFactory();

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/nattableTree/ContextAutoResizeTextPainter.java

@@ -29,6 +29,8 @@ public class ContextAutoResizeTextPainter extends TextPainter {
 
     private boolean changeBackgroundColor = false;
 
+    private boolean containsRowName = false;
+
     public ContextAutoResizeTextPainter(boolean wrapText, boolean paintBg, boolean calculate) {
         super(wrapText, paintBg, calculate);
     }
@@ -43,6 +45,8 @@ public class ContextAutoResizeTextPainter extends TextPainter {
         super.setupGCFromConfig(gc, cellStyle);
         if (cellStyle.getAttributeValue(CellStyleAttributes.FOREGROUND_COLOR).equals(GUIHelper.COLOR_RED)) {
             gc.setForeground(GUIHelper.COLOR_BLACK);
+        } else if (containsRowName) {
+            gc.setForeground(GUIHelper.COLOR_RED);
         } else if (changeBackgroundColor) {
             gc.setForeground(GUIHelper.COLOR_WIDGET_DARK_SHADOW);
         }
@@ -52,6 +56,9 @@ public class ContextAutoResizeTextPainter extends TextPainter {
         changeBackgroundColor = isChange;
     }
 
+    public void setContainsRowName(boolean containsRowName) {
+        this.containsRowName = containsRowName;
+    }
 
 	@Override
 	protected void setNewMinLength(ILayerCell cell, int contentWidth) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/nattableTree/ContextNatTableBackGroudPainter.java

@@ -12,6 +12,11 @@
 // ============================================================================
 package org.talend.core.ui.context.nattableTree;
 
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.nebula.widgets.nattable.config.IConfigRegistry;
 import org.eclipse.nebula.widgets.nattable.data.IDataProvider;
 import org.eclipse.nebula.widgets.nattable.extension.glazedlists.GlazedListsDataProvider;
@@ -23,6 +28,7 @@ import org.eclipse.swt.graphics.GC;
 import org.eclipse.swt.graphics.Rectangle;
 import org.talend.core.ui.context.ContextTreeTable.ContextTreeNode;
 import org.talend.core.ui.context.model.ContextTabChildModel;
+import org.talend.core.ui.context.model.ContextTabParentModel;
 import org.talend.core.ui.context.model.table.ContextTableTabParentModel;
 
 /**
@@ -33,6 +39,8 @@ public class ContextNatTableBackGroudPainter extends BackgroundPainter {
 
     private IDataProvider dataProvider;
 
+    private Map<String, String> rowNames = new HashMap<String, String>();
+
     public ContextNatTableBackGroudPainter(ICellPainter painter, IDataProvider dataProvider) {
         super(painter);
         this.dataProvider = dataProvider;
@@ -53,9 +61,11 @@ public class ContextNatTableBackGroudPainter extends BackgroundPainter {
             ContextTabChildModel rowChildModel = (ContextTabChildModel) rowNode.getTreeData();
             if (rowChildModel != null) {
                 ((ContextAutoResizeTextPainter) getWrappedPainter()).setChangeBackgroundColor(true);
+                checkContainsRowName(rowNode, rowChildModel);
             }
         }
         super.paintCell(cell, gc, bounds, configRegistry);
+        ((ContextAutoResizeTextPainter) getWrappedPainter()).setContainsRowName(false);
     }
 
     @Override
@@ -63,4 +73,33 @@ public class ContextNatTableBackGroudPainter extends BackgroundPainter {
         return super.getBackgroundColour(cell, configRegistry);
     }
 
+    private void checkContainsRowName(ContextTreeNode rowNode, ContextTabChildModel rowChildModel) {
+        // Check any duplicate / similar variable from different context to show in red .
+        String rowName = rowNode.getName();
+        String parentSourceName = null;
+        ContextTabParentModel rowTabParentModel = rowChildModel.getParent();
+        if (rowTabParentModel != null && rowTabParentModel instanceof ContextTableTabParentModel) {
+            ContextTableTabParentModel rowTableTabParentModel = (ContextTableTabParentModel) rowTabParentModel;
+            parentSourceName = rowTableTabParentModel.getSourceName();
+        }
+        if (StringUtils.isNotBlank(rowName) && StringUtils.isNotBlank(parentSourceName)) {
+            rowName = rowName.toUpperCase();
+            parentSourceName = parentSourceName.toUpperCase();
+            if (rowNames.containsValue(rowName)) {
+                Iterator<String> iterator = rowNames.keySet().iterator();
+                while (iterator.hasNext()) {
+                    String key = iterator.next();
+                    String value = rowNames.get(key);
+                    if (rowName.equalsIgnoreCase(value)) {
+                        if (!parentSourceName.equalsIgnoreCase(key)) {
+                            ((ContextAutoResizeTextPainter) getWrappedPainter()).setContainsRowName(true);
+                        }
+                        break;
+                    }
+                }
+            } else {
+                rowNames.put(parentSourceName, rowName);
+            }
+        }
+    }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/DefaultTokenCollector.java

@@ -81,7 +81,7 @@ public class DefaultTokenCollector extends AbstractTokenCollector {
         jsonObject.put("os.arch", System.getProperty("os.arch"));
         jsonObject.put("os.version", System.getProperty("os.version"));
         tokenStudioObject.put(OS.getKey(), jsonObject);
-
+        
         final IPreferenceStore preferenceStore = CoreUIPlugin.getDefault().getPreferenceStore();
         long syncNb = preferenceStore.getLong(COLLECTOR_SYNC_NB);
         tokenStudioObject.put(SYNC_NB.getKey(), syncNb);
@@ -91,6 +91,7 @@ public class DefaultTokenCollector extends AbstractTokenCollector {
         } else {
             tokenStudioObject.put(STOP_COLLECTOR.getKey(), "0"); //$NON-NLS-1$
         }
+        
         return tokenStudioObject;
     }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/PerformanceTokenCollector.java

@@ -0,0 +1,83 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.ui.token;
+
+import java.util.Properties;
+
+import org.talend.commons.exception.CommonExceptionHandler;
+import org.talend.commons.utils.time.PerformanceStatisticUtil;
+import org.talend.commons.utils.time.PerformanceStatisticUtil.StatisticKeys;
+
+import oshi.SystemInfo;
+import oshi.hardware.Baseboard;
+import oshi.hardware.CentralProcessor;
+import oshi.hardware.CentralProcessor.ProcessorIdentifier;
+import oshi.hardware.ComputerSystem;
+import oshi.hardware.GlobalMemory;
+import oshi.hardware.HardwareAbstractionLayer;
+import us.monoid.json.JSONObject;
+
+/**
+ * DOC sbliu  class global comment. Detailled comment
+ */
+public class PerformanceTokenCollector extends AbstractTokenCollector {
+
+    /* (non-Javadoc)
+     * @see org.talend.core.ui.token.AbstractTokenCollector#collect()
+     */
+    @Override
+    public JSONObject collect() throws Exception {
+        checkAndWait();
+        
+        JSONObject tokenStudioObject = new JSONObject();
+        //
+        JSONObject jsonObjectHDInfo = new JSONObject();
+        
+        SystemInfo si = new SystemInfo();
+        HardwareAbstractionLayer hal = si.getHardware();
+        CentralProcessor processor = hal.getProcessor();
+        ProcessorIdentifier processorIdentifier = processor.getProcessorIdentifier();
+        ComputerSystem cs = hal.getComputerSystem();//computer system
+        Baseboard baseboard = cs.getBaseboard();//motherboard
+        GlobalMemory memory = hal.getMemory();
+        
+        jsonObjectHDInfo.put("computer vendor", cs.getManufacturer());
+        jsonObjectHDInfo.put("board vendor", baseboard.getManufacturer());
+        jsonObjectHDInfo.put("board version", baseboard.getVersion());
+        jsonObjectHDInfo.put("processor", processorIdentifier.getName());
+        jsonObjectHDInfo.put("physical memory", Math.ceil((memory.getTotal() /(1024d*1024*1024))) + "GB");
+        tokenStudioObject.put("hardware", jsonObjectHDInfo);
+        
+        //
+        JSONObject jsonObjectIOInfo = new JSONObject();
+        Properties props = PerformanceStatisticUtil.read(PerformanceStatisticUtil.getRecordingFile(),false);
+        jsonObjectIOInfo.put(StatisticKeys.STARTUP_AVERAGE.get(), props.getProperty(StatisticKeys.STARTUP_AVERAGE.get()));
+        jsonObjectIOInfo.put(StatisticKeys.STARTUP_MAX.get(), props.getProperty(StatisticKeys.STARTUP_MAX.get()));
+        jsonObjectIOInfo.put(StatisticKeys.IO_R_MB_SEC.get(), props.getProperty(StatisticKeys.IO_R_MB_SEC.get()));
+        jsonObjectIOInfo.put(StatisticKeys.IO_R_AVERAGE_MB_SEC.get(), props.getProperty(StatisticKeys.IO_R_AVERAGE_MB_SEC.get()));
+        jsonObjectIOInfo.put(StatisticKeys.IO_W_MB_SEC.get(), props.getProperty(StatisticKeys.IO_W_MB_SEC.get()));
+        jsonObjectIOInfo.put(StatisticKeys.IO_W_AVERAGE_MB_SEC.get(), props.getProperty(StatisticKeys.IO_W_AVERAGE_MB_SEC.get()));
+        tokenStudioObject.put("performance", jsonObjectIOInfo);
+        
+        return tokenStudioObject;
+    }
+
+    private void checkAndWait() {
+        try {
+            PerformanceStatisticUtil.waitUntilFinish();
+        } catch (InterruptedException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+    }
+    
+}

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/TokenCollectorFactory.java

@@ -14,15 +14,39 @@ package org.talend.core.ui.token;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.lang.reflect.Field;
-import java.net.Authenticator;
+import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
+import java.net.Proxy;
+import java.net.Proxy.Type;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.zip.GZIPOutputStream;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.StatusLine;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.entity.mime.content.ByteArrayBody;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
 import org.apache.log4j.Logger;
 import org.apache.log4j.Priority;
 import org.eclipse.core.runtime.CoreException;
@@ -36,17 +60,13 @@ import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.ui.preferences.ScopedPreferenceStore;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.network.NetworkUtil;
+import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.ui.CoreUIPlugin;
 import org.talend.core.ui.branding.IBrandingService;
 
 import us.monoid.json.JSONObject;
-import us.monoid.web.AbstractContent;
-import us.monoid.web.FormData;
-import us.monoid.web.Resty;
-import us.monoid.web.TextResource;
-import us.monoid.web.mime.MultipartContent;
 
 /**
  * ggu class global comment. Detailled comment
@@ -214,26 +234,18 @@ public final class TokenCollectorFactory {
                 @Override
                 protected IStatus run(IProgressMonitor monitor) {
                     if (NetworkUtil.isNetworkValid()) {
-                        Authenticator defaultAuth = NetworkUtil.getDefaultAuthenticator();
                         try {
                             JSONObject tokenInfors = collectTokenInfors();
-                            Resty r = new Resty();
-                            // set back the rath for Resty.
-                            Field rathField = Resty.class.getDeclaredField("rath"); //$NON-NLS-1$
-                            rathField.setAccessible(true);
-                            Authenticator auth = (Authenticator) rathField.get(null);
-                            Authenticator.setDefault(auth);
 
                             ByteArrayOutputStream baos = new ByteArrayOutputStream();
                             GZIPOutputStream gzos = new GZIPOutputStream(baos);
                             gzos.write(tokenInfors.toString().getBytes());
                             gzos.close();
-                            AbstractContent ac = Resty.content(baos.toByteArray());
+                            byte[] data = baos.toByteArray();
                             baos.close();
-                            MultipartContent mpc = Resty.form(new FormData("data", ac)); //$NON-NLS-1$
 
-                            TextResource result = r.text("https://www.talend.com/TalendRegisterWS/tokenstudio_v2.php", mpc); //$NON-NLS-1$
-                            String resultStr = new JSONObject(result.toString()).getString("result"); //$NON-NLS-1$
+                            String responseString = sendData(data);
+                            String resultStr = new JSONObject(responseString).getString("result"); //$NON-NLS-1$
                             boolean okReturned = (resultStr != null && resultStr.endsWith("OK")); //$NON-NLS-1$
                             if (okReturned) {
                                 // set new days
@@ -253,7 +265,6 @@ public final class TokenCollectorFactory {
                         } catch (Exception e) {
                             ExceptionHandler.process(e);
                         } finally {
-                            Authenticator.setDefault(defaultAuth);
                         }
                     }
                     return org.eclipse.core.runtime.Status.OK_STATUS;
@@ -274,4 +285,84 @@ public final class TokenCollectorFactory {
             }
         }
     }
+
+    private void addProxy(String url, HttpClientBuilder clientBuilder) throws URISyntaxException {
+        TalendProxySelector proxySelector = TalendProxySelector.getInstance();
+        final List<Proxy> proxyList = proxySelector.getDefaultProxySelector().select(new URI(url));
+        Proxy usedProxy = null;
+        if (proxyList != null && !proxyList.isEmpty()) {
+            usedProxy = proxyList.get(0);
+        }
+
+        if (usedProxy != null) {
+            if (!Type.DIRECT.equals(usedProxy.type())) {
+                final Proxy finalProxy = usedProxy;
+                InetSocketAddress address = (InetSocketAddress) finalProxy.address();
+                String proxyServer = address.getHostString();
+                int proxyPort = address.getPort();
+                PasswordAuthentication proxyAuthentication = proxySelector.getHttpPasswordAuthentication();
+                if (proxyAuthentication != null) {
+                    String proxyUser = proxyAuthentication.getUserName();
+                    if (StringUtils.isNotBlank(proxyUser)) {
+                        String proxyPassword = "";
+                        char[] passwordChars = proxyAuthentication.getPassword();
+                        if (passwordChars != null) {
+                            proxyPassword = new String(passwordChars);
+                        }
+                        BasicCredentialsProvider credProvider = new BasicCredentialsProvider();
+                        credProvider.setCredentials(new AuthScope(proxyServer, proxyPort),
+                                new UsernamePasswordCredentials(proxyUser, proxyPassword));
+                        clientBuilder.setDefaultCredentialsProvider(credProvider);
+                    }
+                }
+                HttpHost proxyHost = new HttpHost(proxyServer, proxyPort);
+                clientBuilder.setProxy(proxyHost);
+            }
+        }
+    }
+
+    private String sendData(byte[] data) throws Exception {
+        CloseableHttpClient client = null;
+        CloseableHttpResponse response = null;
+        try {
+            final String url = "https://www.talend.com/TalendRegisterWS/tokenstudio_v2.php";
+
+            HttpClientBuilder clientBuilder = HttpClients.custom();
+            clientBuilder.disableCookieManagement();
+            addProxy(url, clientBuilder);
+            client = clientBuilder.build();
+
+            HttpPost httpPost = new HttpPost(url);
+            httpPost.setConfig(RequestConfig.DEFAULT);
+
+            MultipartEntityBuilder dataBuilder = MultipartEntityBuilder.create();
+            dataBuilder.addPart("data", new ByteArrayBody(data, null));
+            HttpEntity reqEntity = dataBuilder.build();
+            httpPost.setEntity(reqEntity);
+
+            response = client.execute(httpPost, HttpClientContext.create());
+            StatusLine statusLine = response.getStatusLine();
+            String responseStr = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+            if (HttpURLConnection.HTTP_OK != statusLine.getStatusCode()) {
+                throw new Exception(statusLine.toString() + ", server message: [" + responseStr + "]");
+            }
+            return responseStr;
+        } finally {
+            if (response != null) {
+                try {
+                    response.close();
+                } catch (Throwable e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+            if (client != null) {
+                try {
+                    client.close();
+                } catch (Throwable e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        }
+    }
+
 }

main/plugins/org.talend.core/src/main/java/org/talend/core/services/ICoreTisService.java

@@ -23,9 +23,11 @@ import org.eclipse.emf.common.util.URI;
 import org.osgi.service.prefs.BackingStoreException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.exception.SystemException;
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.process.INode;
+import org.talend.core.model.properties.Property;
 import org.talend.core.runtime.maven.MavenArtifact;
 
 /**
@@ -78,4 +80,18 @@ public interface ICoreTisService extends IService {
 
     Set<String> getComponentBlackList();
     
+    public void afterImport (Property property) throws PersistenceException;  
+
+    boolean hasNewPatchInPatchesFolder();
+
+    boolean isDefaultLicenseAndProjectType();
+
+    void refreshPatchesFolderCache();
+
+    static ICoreTisService get() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ICoreTisService.class)) {
+            return GlobalServiceRegister.getDefault().getService(ICoreTisService.class);
+        }
+        return null;
+    }
 }

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -78,6 +78,7 @@ import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.components.IComponentsFactory;
 import org.talend.core.model.components.IComponentsService;
+import org.talend.core.model.context.ContextUtils;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.IMetadataColumn;
@@ -1006,6 +1007,7 @@ public class ProcessorUtilities {
                     if (context.getName().equals(currentContext.getName())) {
                         // override parameter value before generate current context
                         IContext checkedContext = checkNeedOverrideContextParameterValue(currentContext, jobInfo);
+                        checkedContext = checkCleanSecureContextParameterValue(checkedContext, jobInfo);
                         processor.setContext(checkedContext); // generate current context.
                     } else {
                         processor.setContext(context);
@@ -1078,6 +1080,49 @@ public class ProcessorUtilities {
         }
         return context;
     }
+	
+    private static IContext checkCleanSecureContextParameterValue(IContext currentContext, JobInfo jobInfo) {
+        
+        JobInfo job = null;
+        
+        if (jobInfo.getFatherJobInfo() == null) {
+        	job = jobInfo;
+        } else {
+        	job = getRootJob(jobInfo);
+        	if (job.getProcess() == null || "route".equalsIgnoreCase(job.getProcess().getElementName())) {
+        		// cleanup context only for child jobs which are referenced
+        		// by tRunJob component or for Joblets (see TESB-29718 for details) 
+        		return currentContext;
+        	}
+        }
+        
+        if (job.getArgumentsMap() == null
+            || job.getArgumentsMap().get(TalendProcessArgumentConstant.ARG_CLEAR_PASSWORD_CONTEXT_PARAMETERS) == null 
+                || !Boolean.parseBoolean((ProcessUtils.getOptionValue(job.getArgumentsMap(), TalendProcessArgumentConstant.ARG_CLEAR_PASSWORD_CONTEXT_PARAMETERS,
+                    (String) null)))) {
+            return currentContext;
+        }
+        
+        IContext context = currentContext.clone();
+
+        List<IContextParameter> contextParameterList = context.getContextParameterList();
+        for (IContextParameter contextParameter : contextParameterList) {
+            if (PasswordEncryptUtil.isPasswordType(contextParameter.getType()) 
+                || ContextUtils.isSecureSensitiveParam(contextParameter.getName())) {
+                    contextParameter.setValue("");
+            }
+        }
+        return context;
+    }
+
+    private static JobInfo getRootJob(JobInfo jobInfo) {
+    	
+    	if (jobInfo  != null && jobInfo.getFatherJobInfo() != null)  {
+    		return getRootJob(jobInfo.getFatherJobInfo());
+        }
+        
+        return  jobInfo;
+    }
 
     private static void generateDataSet(IProcess process, IProcessor processor) {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
@@ -1560,6 +1605,10 @@ public class ProcessorUtilities {
                     List<ProcessItem> testsItems =
                             testContainerService.getTestContainersByVersion(jobInfo.getProcessItem());
                     for (ProcessItem testItem : testsItems) {
+                        if (testItem.getProperty().getItem().getState().isDeleted()
+                                && IRunProcessService.get().isExcludeDeletedItems(testItem.getProperty())) {
+                            continue;
+                        }
                         JobInfo subJobInfo = new JobInfo(testItem, testItem.getProcess().getDefaultContext());
                         subJobInfo.setTestContainer(true);
                         subJobInfo.setFatherJobInfo(jobInfo);
@@ -2916,5 +2965,12 @@ public class ProcessorUtilities {
         }
         return false;
     }
+    
+    public static boolean isJob(Property p) {
+        if (p != null) {
+            return ERepositoryObjectType.getType(p).equals(ERepositoryObjectType.PROCESS);
+        }
+        return false;
+    }
 
 }

main/plugins/org.talend.designer.maven.tos/resources/components/pom.xml

@@ -56,8 +56,8 @@
                 <version>17</version>
                 <type>pom</type>
             </dependency>             
-            <!-- It fixes provided dependency scope to compile, because maven-dependency-plugin is setup to include
-             only compile and runtime scopes, but provided is also required. It's not possible to setup plugin to
+            <!-- It fixes provided dependency scope to compile, because maven-dependency-plugin is setup to include
+             only compile and runtime scopes, but provided is also required. It's not possible to setup plugin to
              include all: compile, runtime and provided dependencies -->
             <dependency>
                 <groupId>biz.aQute.bnd</groupId>
@@ -161,7 +161,7 @@
             <artifactId>components-api</artifactId>
             <version>${components.version}</version>
         </dependency>
-        <!-- Dependencies in provided scope should be explicitly added as dependency in this module,
+        <!-- Dependencies in provided scope should be explicitly added as dependency in this module,
             because provided scope is not transitive -->
 		<dependency>
             <groupId>org.osgi</groupId>
@@ -199,6 +199,11 @@
 			<artifactId>jackson-mapper-asl</artifactId>
 			<version>1.9.14-TALEND</version>
 		</dependency>
+		<dependency>
+			<groupId>commons-beanutils</groupId>
+			<artifactId>commons-beanutils</artifactId>
+			<version>1.9.4</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.9.0</tacokit.components.version>
+		<tacokit.components.version>1.13.0</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
     
     <properties>
-        <tcomp.version>1.1.15</tcomp.version>
+        <tcomp.version>1.1.25</tcomp.version>
         <slf4j.version>1.7.25</slf4j.version>
     </properties>
     
@@ -106,6 +106,11 @@
             <artifactId>slf4j-simple</artifactId>
             <version>${slf4j.version}</version>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.14</version>
+        </dependency>
     </dependencies>
     
     <build>

main/plugins/org.talend.designer.maven.ui/src/main/java/messages.properties

@@ -12,4 +12,6 @@ ProjectPomProjectSettingPage_FilterErrorMessage=Invalid filter: {0}
 ProjectPomProjectSettingPage.syncAllPomsButtonText=Force full re-synchronize poms
 AbstractPersistentProjectSettingPage.syncAllPoms=Do you want to update all poms? \n This operation might take long time depends on your project size.
 MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
-MavenProjectSettingPage.refModuleText=Set reference project modules in profile
+MavenProjectSettingPage.refModuleText=Set reference project modules in profile
+MavenProjectSettingPage.excludeDeletedItems=Exclude deleted items
+MavenProjectSettingPage.syncAllPomsWarning=Click the Force full re-synchronize poms button to apply the new settings.

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/initializer/MavenScriptsProjectSettingInitializer.java

@@ -32,6 +32,7 @@ import org.talend.designer.maven.template.AbstractMavenTemplateManager;
 import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.tools.extension.PomExtensionRegistry;
 import org.talend.designer.maven.ui.DesignerMavenUiPlugin;
+import org.talend.designer.maven.utils.PomUtil;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -68,6 +69,7 @@ public class MavenScriptsProjectSettingInitializer extends AbstractProjectPrefer
                 Model model = MavenPlugin.getMavenModelManager().readMavenModel(stream);
                 PomExtensionRegistry.getInstance().updatePomTemplate(model);
                 ByteArrayOutputStream out = new ByteArrayOutputStream();
+                PomUtil.sortModules(model);
                 MavenPlugin.getMaven().writeModel(model, out);
                 String content = out.toString(TalendMavenConstants.DEFAULT_ENCODING);
                 if (content != null) {

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/page/MavenProjectSettingPage.java

@@ -25,6 +25,7 @@ import org.eclipse.swt.widgets.Button;
 import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.Label;
 import org.eclipse.swt.widgets.Text;
+import org.eclipse.ui.forms.IMessage;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
@@ -52,6 +53,8 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 
     private Button useProfileModuleCheckbox;
 
+    private Button excludeDeletedItemsCheckbox;
+
 	public MavenProjectSettingPage() {
 		noDefaultAndApplyButton();
 	}
@@ -89,12 +92,31 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
         useProfileModuleCheckbox = new Button(parent, SWT.CHECK);
         useProfileModuleCheckbox.setText(Messages.getString("MavenProjectSettingPage.refModuleText")); //$NON-NLS-1$
         useProfileModuleCheckbox.setSelection(preferenceStore.getBoolean(MavenConstants.USE_PROFILE_MODULE));
+        useProfileModuleCheckbox.addSelectionListener(new SelectionAdapter() {
+
+            @Override
+            public void widgetSelected(SelectionEvent e) {
+                addSyncWarning();
+            }
+        });
+
+        excludeDeletedItemsCheckbox = new Button(parent, SWT.CHECK);
+        excludeDeletedItemsCheckbox.setText(Messages.getString("MavenProjectSettingPage.excludeDeletedItems")); //$NON-NLS-1$
+        excludeDeletedItemsCheckbox.setSelection(preferenceStore.getBoolean(MavenConstants.EXCLUDE_DELETED_ITEMS));
+        excludeDeletedItemsCheckbox.addSelectionListener(new SelectionAdapter() {
+
+            @Override
+            public void widgetSelected(SelectionEvent e) {
+                addSyncWarning();
+            }
+        });
 
         filterText.setText(filter);
 		filterText.addModifyListener(new ModifyListener() {
 
 			@Override
 			public void modifyText(ModifyEvent e) {
+                addSyncWarning();
 				if (GlobalServiceRegister.getDefault().isServiceRegistered(IFilterService.class)) {
 					IFilterService service = (IFilterService) GlobalServiceRegister.getDefault()
 							.getService(IFilterService.class);
@@ -122,6 +144,7 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 				try {
                     preferenceStore.setValue(MavenConstants.POM_FILTER, getRealVersionFilter(filter));
                     preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
+                    preferenceStore.setValue(MavenConstants.EXCLUDE_DELETED_ITEMS, excludeDeletedItemsCheckbox.getSelection());
 					new AggregatorPomsHelper().syncAllPoms();
 				} catch (Exception e) {
 					ExceptionHandler.process(e);
@@ -135,12 +158,17 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 
 	}
 
+    private void addSyncWarning() {
+        setMessage(Messages.getString("MavenProjectSettingPage.syncAllPomsWarning"), IMessage.WARNING); //$NON-NLS-1$
+    }
+
 	@Override
 	public boolean performOk() {
 		boolean ok = super.performOk();
 		if (preferenceStore != null) {
             preferenceStore.setValue(MavenConstants.POM_FILTER, getRealVersionFilter(filter));
             preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
+            preferenceStore.setValue(MavenConstants.EXCLUDE_DELETED_ITEMS, excludeDeletedItemsCheckbox.getSelection());
 		}
 		return ok;
 	}

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java

@@ -17,7 +17,9 @@ import static org.talend.designer.maven.model.TalendJavaProjectConstants.*;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -55,6 +57,7 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded;
+import org.talend.core.model.general.Project;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProjectReference;
@@ -169,9 +172,18 @@ public class AggregatorPomsHelper {
 
             @Override
             protected void run() {
-                updateCodeProject(monitor, ERepositoryObjectType.ROUTINES, forceBuild);
-                if (ProcessUtils.isRequiredBeans(null)) {
-                    updateCodeProject(monitor, ERepositoryObjectType.valueOf("BEANS"), forceBuild); //$NON-NLS-1$
+                Project currentProject = ProjectManager.getInstance().getCurrentProject();
+                for (ERepositoryObjectType codeType : ERepositoryObjectType.getAllTypesOfCodes()) {
+                    try {
+                        if (CodeM2CacheManager.needUpdateCodeProject(currentProject, codeType)) {
+                            ITalendProcessJavaProject codeProject = getCodesProject(codeType);
+                            updateCodeProjectPom(monitor, codeType, codeProject.getProjectPom());
+                            buildAndInstallCodesProject(monitor, codeType, true, forceBuild);
+                            CodeM2CacheManager.updateCodeProjectCache(currentProject, codeType);
+                        }
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
                 }
             }
         };
@@ -179,16 +191,6 @@ public class AggregatorPomsHelper {
         ProxyRepositoryFactory.getInstance().executeRepositoryWorkUnit(workUnit);
     }
 
-    private void updateCodeProject(IProgressMonitor monitor, ERepositoryObjectType codeType, boolean forceBuild) {
-        try {
-            ITalendProcessJavaProject codeProject = getCodesProject(codeType);
-            updateCodeProjectPom(monitor, codeType, codeProject.getProjectPom());
-            buildAndInstallCodesProject(monitor, codeType, true, forceBuild);
-        } catch (Exception e) {
-            ExceptionHandler.process(e);
-        }
-    }
-
     public void updateCodeProjectPom(IProgressMonitor monitor, ERepositoryObjectType type, IFile pomFile)
             throws Exception {
         if (type != null) {
@@ -209,27 +211,15 @@ public class AggregatorPomsHelper {
     }
 
     public static void updateAllCodesProjectNeededModules(IProgressMonitor monitor) {
-        updateCodesProjectNeededModulesByType(ERepositoryObjectType.ROUTINES, monitor);
-        if (ProcessUtils.isRequiredBeans(null)) {
-            updateCodesProjectNeededModulesByType(ERepositoryObjectType.valueOf("BEANS"), monitor); //$NON-NLS-1$
-        }
-    }
-
-    private static void updateCodesProjectNeededModulesByType(ERepositoryObjectType codeType,
-            IProgressMonitor monitor) {
-        Set<ModuleNeeded> neededModules = null;
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibrariesService.class)) {
-            ILibrariesService librariesService =
-                    (ILibrariesService) GlobalServiceRegister.getDefault().getService(ILibrariesService.class);
-            neededModules = librariesService.getCodesModuleNeededs(codeType);
-        }
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
-            ILibraryManagerService repositoryBundleService =
-                    (ILibraryManagerService) GlobalServiceRegister.getDefault().getService(
-                            ILibraryManagerService.class);
-            if (neededModules != null && !neededModules.isEmpty()) {
-                repositoryBundleService.installModules(neededModules, monitor);
-            }
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibrariesService.class)
+                && GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
+            Set<ModuleNeeded> neededModules = new HashSet<>();
+            ILibrariesService librariesService = GlobalServiceRegister.getDefault().getService(ILibrariesService.class);
+            ERepositoryObjectType.getAllTypesOfCodes()
+                    .forEach(c -> neededModules.addAll(librariesService.getCodesModuleNeededs(c)));
+            ILibraryManagerService repositoryBundleService = GlobalServiceRegister.getDefault()
+                    .getService(ILibraryManagerService.class);
+            repositoryBundleService.installModules(neededModules, monitor);
         }
     }
 
@@ -266,7 +256,7 @@ public class AggregatorPomsHelper {
         if (install) {
             Map<String, Object> argumentsMap = new HashMap<>();
             argumentsMap.put(TalendProcessArgumentConstant.ARG_GOAL, TalendMavenConstants.GOAL_INSTALL);
-            argumentsMap.put(TalendProcessArgumentConstant.ARG_PROGRAM_ARGUMENTS, "-Dmaven.main.skip=true"); //$NON-NLS-1$
+            argumentsMap.put(TalendProcessArgumentConstant.ARG_PROGRAM_ARGUMENTS, TalendMavenConstants.ARG_MAIN_SKIP);
             codeProject.buildModules(monitor, null, argumentsMap);
             BuildCacheManager.getInstance().updateCodeLastBuildDate(codeType);
         }
@@ -328,26 +318,10 @@ public class AggregatorPomsHelper {
     }
 
     public static void addToParentModules(IFile pomFile, Property property, boolean checkFilter) throws Exception {
-        // Check relation for ESB service job, should not be added into main pom
-        if (property != null) {
-            List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(property.getId(),
-                    property.getVersion(), RelationshipItemBuilder.JOB_RELATION);
-            for (Relation relation : relations) {
-                if (RelationshipItemBuilder.SERVICES_RELATION.equals(relation.getType())) {
-                    return;
-                }
-            }
+        if (!checkIfCanAddToParentModules(property, checkFilter)) {
+            return;
         }
 
-        if (checkFilter) {
-            if (GlobalServiceRegister.getDefault().isServiceRegistered(IFilterService.class)) {
-                IFilterService filterService = (IFilterService) GlobalServiceRegister.getDefault()
-                        .getService(IFilterService.class);
-                if (property != null && !filterService.isFilterAccepted(property.getItem(), PomIdsHelper.getPomFilter())) {
-                    return;
-                }
-            }
-        }
         IFile parentPom = getParentModulePomFile(pomFile);
         if (parentPom != null) {
             if (!parentPom.isSynchronized(IResource.DEPTH_ZERO)) {
@@ -367,6 +341,36 @@ public class AggregatorPomsHelper {
         }
     }
 
+    private static boolean checkIfCanAddToParentModules(Property property, boolean checkFilter) {
+        // Check relation for ESB service job, should not be added into main pom
+        if (property != null) {
+            List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(property.getId(),
+                    property.getVersion(), RelationshipItemBuilder.JOB_RELATION);
+            for (Relation relation : relations) {
+                if (RelationshipItemBuilder.SERVICES_RELATION.equals(relation.getType())) {
+                    return false;
+                }
+            }
+
+            // for import won't add for exclude option
+            if (property.getItem() != null && property.getItem().getState() != null && property.getItem().getState().isDeleted()
+                    && PomIdsHelper.getIfExcludeDeletedItems(property)) {
+                return false;
+            }
+        }
+
+        if (checkFilter) {
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(IFilterService.class)) {
+                IFilterService filterService = (IFilterService) GlobalServiceRegister.getDefault()
+                        .getService(IFilterService.class);
+                if (property != null && !filterService.isFilterAccepted(property.getItem(), PomIdsHelper.getPomFilter())) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
     public static void removeFromParentModules(IFile pomFile) throws Exception {
         IFile parentPom = getParentModulePomFile(pomFile);
         if (parentPom != null) {
@@ -384,6 +388,69 @@ public class AggregatorPomsHelper {
         }
     }
 
+    public static void removeAllVersionsFromParentModules(Property property) throws Exception {
+        IFile parentPom = getParentModulePomFile(
+                AggregatorPomsHelper.getItemPomFolder(property).getFile(TalendMavenConstants.POM_FILE_NAME));
+        if (parentPom == null) {
+            return;
+        }
+
+        List<String> relativePathList = new ArrayList<String>();
+        List<IRepositoryViewObject> allVersion = ProxyRepositoryFactory.getInstance().getAllVersion(property.getId());
+        for (IRepositoryViewObject object : allVersion) {
+            IFile pomFile = AggregatorPomsHelper.getItemPomFolder(object.getProperty())
+                    .getFile(TalendMavenConstants.POM_FILE_NAME);
+            String relativePath = pomFile.getParent().getLocation().makeRelativeTo(parentPom.getParent().getLocation())
+                    .toPortableString();
+            if (StringUtils.isNotBlank(relativePath)) {
+                relativePathList.add(relativePath);
+            }
+        }
+
+        Model model = MavenPlugin.getMaven().readModel(parentPom.getContents());
+        List<String> modules = model.getModules();
+        if (modules != null && modules.size() > 0) {
+            modules.removeAll(relativePathList);
+            PomUtil.savePom(null, model, parentPom);
+        }
+    }
+    
+    public static void restoreAllVersionsFromParentModules(Property property) throws Exception {
+        IFile parentPom = getParentModulePomFile(
+                AggregatorPomsHelper.getItemPomFolder(property).getFile(TalendMavenConstants.POM_FILE_NAME));
+        if (parentPom == null) {
+            return;
+        }
+        
+        List<String> relativePathList = new ArrayList<String>();
+        Model model = MavenPlugin.getMaven().readModel(parentPom.getContents());
+        List<String> modules = model.getModules();
+        if (modules == null) {
+            modules = new ArrayList<>();
+            model.setModules(modules);
+        }
+
+        List<IRepositoryViewObject> allVersion = ProxyRepositoryFactory.getInstance().getAllVersion(property.getId());
+        for (IRepositoryViewObject object : allVersion) {
+            Property itemProperty = object.getProperty();
+            if (!checkIfCanAddToParentModules(itemProperty, true)) {
+                continue;
+            }
+            IFile pomFile = AggregatorPomsHelper.getItemPomFolder(object.getProperty(), object.getVersion())
+                    .getFile(TalendMavenConstants.POM_FILE_NAME);
+
+            String relativePath = pomFile.getParent().getLocation().makeRelativeTo(parentPom.getParent().getLocation())
+                    .toPortableString();
+            if (StringUtils.isNoneBlank(relativePath) && !modules.contains(relativePath)) {
+                relativePathList.add(relativePath);
+            }
+        }
+        Collections.sort(relativePathList);
+        modules.addAll(relativePathList);
+        PomUtil.savePom(null, model, parentPom);
+
+    }
+
     private static IFile getParentModulePomFile(IFile pomFile) {
         IFile parentPom = null;
         if (pomFile == null || pomFile.getParent() == null || pomFile.getParent().getParent() == null) {
@@ -538,7 +605,10 @@ public class AggregatorPomsHelper {
         String jobFolderName = getJobProjectFolderName(property.getLabel(), version);
         ERepositoryObjectType type = ERepositoryObjectType.getItemType(property.getItem());
         IFolder jobFolder = helper.getProcessFolder(type).getFolder(itemRelativePath).getFolder(jobFolderName);
-        createFoldersIfNeeded(jobFolder);
+        List<ERepositoryObjectType> allTypesOfProcess2 = ERepositoryObjectType.getAllTypesOfProcess2();
+        if (allTypesOfProcess2.contains(type)) {
+            createFoldersIfNeeded(jobFolder);
+        }
         return jobFolder;
     }
 
@@ -810,6 +880,9 @@ public class AggregatorPomsHelper {
                 }
             }
             if (object.getProperty() != null && object.getProperty().getItem() != null) {
+                if (object.isDeleted() && PomIdsHelper.getIfExcludeDeletedItems()) {
+                    continue;
+                }
                 Item item = object.getProperty().getItem();
                 if (ProjectManager.getInstance().isInCurrentMainProject(item)) {
                     monitor.subTask("Synchronize job pom: " + item.getProperty().getLabel() //$NON-NLS-1$

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/BuildCacheManager.java

@@ -395,4 +395,24 @@ public class BuildCacheManager {
         return false;
     }
 
+    public boolean containsMultipleVersionModules() {
+        return containsMultipleVersionModules(currentJobletmodules) || containsMultipleVersionModules(currentJobmodules);
+    }
+
+    private static boolean containsMultipleVersionModules(Set<String> mods) {
+        Set<String> joblets = new HashSet<String>();
+        for (String mod : mods) {
+            int idx = mod.lastIndexOf('_');
+            if (idx == -1) {
+                continue;
+            }
+            String jobletWithoutVersion = mod.substring(0, idx);
+            if (joblets.contains(jobletWithoutVersion)) {
+                return true;
+            } else {
+                joblets.add(jobletWithoutVersion);
+            }
+        }
+        return false;
+    }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/CodeM2CacheManager.java

@@ -0,0 +1,109 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.maven.tools;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.text.ParseException;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.m2e.core.MavenPlugin;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.exception.PersistenceException;
+import org.talend.core.model.general.Project;
+import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
+import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.repository.model.ProxyRepositoryFactory;
+import org.talend.core.runtime.repository.item.ItemProductKeys;
+import org.talend.cwm.helper.ResourceHelper;
+import org.talend.designer.maven.utils.PomIdsHelper;
+
+public class CodeM2CacheManager {
+
+    private static final String KEY_SEPERATOR = "|"; //$NON-NLS-1$
+
+    public static boolean needUpdateCodeProject(Project project, ERepositoryObjectType codeType) {
+        try {
+            String projectTechName = project.getTechnicalLabel();
+            File cacheFile = getCacheFile(projectTechName, codeType);
+            if (!cacheFile.exists()) {
+                return true;
+            }
+            Properties cache = new Properties();
+            cache.load(new FileInputStream(cacheFile));
+            List<IRepositoryViewObject> allCodes = ProxyRepositoryFactory.getInstance().getAll(project, codeType, false);
+            // check A/D
+            if (allCodes.size() != cache.size()) {
+                return true;
+            }
+            // check M
+            for (IRepositoryViewObject codeItem : allCodes) {
+                Property property = codeItem.getProperty();
+                String key = getCacheKey(projectTechName, property);
+                String cachedTimestamp = cache.getProperty(key);
+                if (cachedTimestamp != null) {
+                    Date currentDate = ResourceHelper.dateFormat().parse(getCacheDate(projectTechName, property));
+                    Date cachedDate = ResourceHelper.dateFormat().parse(cachedTimestamp);
+                    if (currentDate.compareTo(cachedDate) != 0) {
+                        return true;
+                    }
+                }
+            }
+        } catch (PersistenceException | IOException | ParseException e) {
+            ExceptionHandler.process(e);
+            // if any exception, still update in case breaking build job
+            return true;
+        }
+        return false;
+    }
+
+    public static void updateCodeProjectCache(Project project, ERepositoryObjectType codeType) {
+        String projectTechName = project.getTechnicalLabel();
+        File cacheFile = getCacheFile(projectTechName, codeType);
+        try (OutputStream out = new FileOutputStream(cacheFile)) {
+            List<IRepositoryViewObject> allCodes = ProxyRepositoryFactory.getInstance().getAll(project, codeType, false);
+            Properties cache = new Properties();
+            for (IRepositoryViewObject codeItem : allCodes) {
+                Property property = codeItem.getProperty();
+                String key = getCacheKey(projectTechName, property);
+                String value = getCacheDate(projectTechName, property);
+                cache.put(key, value);
+            }
+            cache.store(out, StringUtils.EMPTY);
+        } catch (PersistenceException | IOException e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    private static File getCacheFile(String projectTechName, ERepositoryObjectType codeType) {
+        String cacheFileName = PomIdsHelper.getProjectGroupId(projectTechName) + "." + codeType.name().toLowerCase() + "-" //$NON-NLS-1$ //$NON-NLS-2$
+                + PomIdsHelper.getCodesVersion(projectTechName) + ".cache"; // $NON-NLS-1$
+        return new File(MavenPlugin.getMaven().getLocalRepositoryPath(), cacheFileName);
+    }
+
+    private static String getCacheKey(String projectTechName, Property property) {
+        return projectTechName + KEY_SEPERATOR + property.getId() + KEY_SEPERATOR + property.getVersion(); // $NON-NLS-1$
+    }
+
+    private static String getCacheDate(String projectTechName, Property property) {
+        return (String) property.getAdditionalProperties().get(ItemProductKeys.DATE.getModifiedKey());
+    }
+
+}

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -18,7 +18,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -49,6 +48,7 @@ import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.eclipse.m2e.core.embedder.IMaven;
+import org.talend.commons.exception.CommonExceptionHandler;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.VersionUtils;
@@ -657,8 +657,11 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         jobCoordinate.add(getJobCoordinate(currentJobProperty));
         
         // children jobs without test cases
-        Set<JobInfo> childrenJobInfo = !hasLoopDependency() ? processor.getBuildChildrenJobs().stream().filter(j -> !j.isTestContainer()).collect(Collectors.toSet()) : Collections.emptySet();
-        childrenJobInfo.forEach(j -> jobCoordinate.add(getJobCoordinate(j.getProcessItem().getProperty())));
+        Set<JobInfo> childrenJobInfo = processor.getBuildChildrenJobs().stream().filter(j -> !j.isTestContainer())
+                .collect(Collectors.toSet());
+        if (!hasLoopDependency()) {
+            childrenJobInfo.forEach(j -> jobCoordinate.add(getJobCoordinate(j.getProcessItem().getProperty())));
+        }
 
         // talend libraries and codes
         String projectGroupId = PomIdsHelper.getProjectGroupId(ProjectManager.getInstance().getProject(currentJobProperty).getTechnicalLabel());
@@ -894,6 +897,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         }
         IMaven maven = MavenPlugin.getMaven();
         ArtifactRepository repository = maven.getLocalRepository();
+        boolean isDIJob = ERepositoryObjectType.getItemType(getJobProcessor().getProperty().getItem()) == ERepositoryObjectType.PROCESS;
         for (Dependency dependency : duplicateDependencies) {
             if (((SortableDependency) dependency).isAssemblyOptional()) {
                 continue;
@@ -902,6 +906,10 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                     dependency.getVersion(), dependency.getType(), dependency.getClassifier());
             Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
             sourceLocation = path.toString();
+            if (isDIJob && !new File(sourceLocation).exists()) {
+                CommonExceptionHandler.warn("Job dependency [" + sourceLocation + "] does not exist!");
+                continue;
+            }
             String destName = path.getFileName().toString();
             Node fileNode = document.createElement("file");
             filesNode.appendChild(fileNode);

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomIdsHelper.java

@@ -266,6 +266,18 @@ public class PomIdsHelper {
         return manager.getBoolean(MavenConstants.USE_PROFILE_MODULE);
     }
 
+    public static boolean getIfExcludeDeletedItems() {
+        String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(MavenConstants.EXCLUDE_DELETED_ITEMS);
+    }
+
+    public static boolean getIfExcludeDeletedItems(Property property) {
+        String projectTechName = ProjectManager.getInstance().getProject(property).getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(MavenConstants.EXCLUDE_DELETED_ITEMS);
+    }
+
     private static String getGroupId(String projectTechName, String baseName, Property property) {
         if (projectTechName == null) {
             projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
@@ -330,6 +342,12 @@ public class PomIdsHelper {
             if (preferenceStore.getString(MavenConstants.POM_FILTER) == null) {
                 preferenceStore.setValue(MavenConstants.POM_FILTER, "");
             }
+
+            if (!preferenceManager.exist()
+                    && StringUtils.isBlank(preferenceStore.getString(MavenConstants.EXCLUDE_DELETED_ITEMS))) {
+                // for new project, set EXCLUDE_DELETED_ITEMS=true as default
+                preferenceStore.setValue(MavenConstants.EXCLUDE_DELETED_ITEMS, true);
+            }
             preferenceManager.save();
             preferenceManagers.put(projectTechName, preferenceManager);
             return preferenceManager;

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomUtil.java

@@ -30,6 +30,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -140,6 +141,7 @@ public class PomUtil {
          * copied the codes from createMavenModel of MavenModelManager
          */
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        sortModules(model);
         MavenPlugin.getMaven().writeModel(model, buf);
 
         ByteArrayInputStream source = new ByteArrayInputStream(buf.toByteArray());
@@ -161,6 +163,7 @@ public class PomUtil {
         properties.putAll(model.getProperties());
         model.setProperties(properties);
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        sortModules(model);
         MavenPlugin.getMaven().writeModel(model, buf);
 
         ByteArrayInputStream source = new ByteArrayInputStream(buf.toByteArray());
@@ -182,6 +185,17 @@ public class PomUtil {
         }
     }
 
+    public static void sortModules(Model model) {
+        if (model != null) {
+            List<String> modules = model.getModules();
+            if (modules != null && !modules.isEmpty()) {
+                List<String> sortedModules = new LinkedList<>(modules);
+                Collections.sort(sortedModules);
+                model.setModules(sortedModules);
+            }
+        }
+    }
+
     private static void safeClose(Closeable stream) {
         try {
             if (stream != null) {
@@ -669,7 +683,9 @@ public class PomUtil {
         File pomFile = new File(baseFolder, TalendMavenConstants.POM_FILE_NAME);
 
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
-        MavenPlugin.getMaven().writeModel(createModel(artifact), buf);
+        Model model = createModel(artifact);
+        sortModules(model);
+        MavenPlugin.getMaven().writeModel(model, buf);
 
         DocumentBuilderFactory documentBuilderFactory = XmlUtils.getSecureDocumentBuilderFactory();
         documentBuilderFactory.setNamespaceAware(false);

main/plugins/org.talend.libraries.dom4j-jaxen/.classpath

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
 	<classpathentry exported="true" kind="lib" path="lib/jaxen-1.1.6.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/dom4j-2.1.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/dom4j-2.1.3.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

main/plugins/org.talend.libraries.dom4j-jaxen/META-INF/MANIFEST.MF

@@ -5,7 +5,7 @@ Bundle-SymbolicName: org.talend.libraries.dom4j-jaxen;singleton:=true
 Bundle-Version: 7.3.1.qualifier
 Bundle-Vendor: .Talend SA.
 Bundle-ClassPath: .,
- lib/dom4j-2.1.1.jar,
+ lib/dom4j-2.1.3.jar,
  lib/jaxen-1.1.6.jar
 Export-Package: org.dom4j,
  org.dom4j.bean,

main/plugins/org.talend.libraries.dom4j-jaxen/build.properties

@@ -2,7 +2,7 @@ bin.includes = META-INF/,\
                .,\
                lib/,\
                licences/,\
-               lib/dom4j-2.1.1.jar,\
+               lib/dom4j-2.1.3.jar,\
                lib/jaxen-1.1.6.jar,\
                plugin.xml
 

main/plugins/org.talend.libraries.dom4j-jaxen/plugin.xml

@@ -5,11 +5,11 @@
          point="org.talend.core.runtime.librariesNeeded">
       <libraryNeeded
               context="plugin:org.talend.libraries.dom4j-jaxen"
-              id="dom4j-2.1.1.jar"
-              mvn_uri="mvn:org.dom4j/dom4j/2.1.1"
-              name="dom4j-2.1.1.jar"
+              id="dom4j-2.1.3.jar"
+              mvn_uri="mvn:org.dom4j/dom4j/2.1.3"
+              name="dom4j-2.1.3.jar"
               required="true"
-              uripath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/dom4j-2.1.1.jar">
+              uripath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/dom4j-2.1.3.jar">
       </libraryNeeded>
       <libraryNeeded
               context="plugin:org.talend.libraries.dom4j-jaxen"

main/plugins/org.talend.libraries.dom4j-jaxen/pom.xml

@@ -41,7 +41,7 @@
     <dependency>
         <groupId>org.dom4j</groupId>
         <artifactId>dom4j</artifactId>
-        <version>2.1.1</version>
+        <version>2.1.3</version>
     </dependency>
   </dependencies>
   <build>

main/plugins/org.talend.librariesmanager.ui/plugin.xml

@@ -83,7 +83,7 @@
       <systemRoutine
             name="Document">
          <library
-               name="dom4j-2.1.1.jar">
+               name="dom4j-2.1.3.jar">
             <bundleId
                   bundleId="org.apache.servicemix.bundles.dom4j">
             </bundleId>
@@ -92,7 +92,7 @@
       <systemRoutine
             name="DocumentHelper">
          <library
-               name="dom4j-2.1.1.jar">
+               name="dom4j-2.1.3.jar">
             <bundleId
                   bundleId="org.apache.servicemix.bundles.dom4j">
             </bundleId>
@@ -101,7 +101,7 @@
       <systemRoutine
             name="DocumentToFlat">
          <library
-               name="dom4j-2.1.1.jar">
+               name="dom4j-2.1.3.jar">
             <bundleId
                   bundleId="org.apache.servicemix.bundles.dom4j">
             </bundleId>
@@ -110,7 +110,7 @@
       <systemRoutine
             name="NameSpaceTool">
          <library
-               name="dom4j-2.1.1.jar">
+               name="dom4j-2.1.3.jar">
             <bundleId
                   bundleId="org.apache.servicemix.bundles.dom4j">
             </bundleId>
@@ -119,7 +119,7 @@
       <systemRoutine
             name="ParserUtils">
          <library
-               name="dom4j-2.1.1.jar">
+               name="dom4j-2.1.3.jar">
             <bundleId
                   bundleId="org.apache.servicemix.bundles.dom4j">
             </bundleId>
@@ -128,7 +128,7 @@
       <systemRoutine
             name="TalendMDMJob">
          <library
-               name="dom4j-2.1.1.jar">
+               name="dom4j-2.1.3.jar">
             <bundleId
                   bundleId="org.apache.servicemix.bundles.dom4j">
             </bundleId>

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/views/ModulesViewComposite.java

@@ -40,6 +40,7 @@ import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
+import org.talend.librariesmanager.model.service.CustomUriManager;
 import org.talend.librariesmanager.ui.dialogs.CustomURITextCellEditor;
 import org.talend.librariesmanager.ui.dialogs.InstallModuleDialog;
 import org.talend.librariesmanager.ui.i18n.Messages;
@@ -298,6 +299,7 @@ public class ModulesViewComposite extends Composite {
      * @see org.talend.designer.codegen.perlmodule.ui.views.IModulesViewComposite#refresh()
      */
     public void refresh() {
+        CustomUriManager.getInstance().setForeceReloadCustomUri();
         List<ModuleNeeded> modulesNeeded = new ArrayList<ModuleNeeded>();
         modulesNeeded.addAll(ModulesNeededProvider.getAllManagedModules());
         ModulesViewComposite.getTableViewerCreator().init(modulesNeeded);

main/plugins/org.talend.librariesmanager/resources/java/routines/TalendDate.java

@@ -263,26 +263,31 @@ public class TalendDate {
 
     /**
      * Tests string value as a date with right pattern using strict rules.
-     * This validation uses Java 8 time tools such {@link DateTimeFormatter#parse }
-     * and {@link DateTimeFormatter#format }
-     * </br>
-     * </br>
-     * Examples:
-     * </br>
-     * <code>isDateStrict("20110327 121711", "yyyyMMdd HHmmss")</code> return <code>true</code></br>
-     * <code>isDateStrict("01100327 121711", "yyyyMMdd HHmmss")</code> return <code>false</code></br>
-     * <code>isDateStrict("20180229 221711", "yyyyMMdd HHmmss")</code> return <code>false</code></br>
-     * <code>isDateStrict("2016-02-29 22:17:11", "yyyy-MM-dd HH:mm:ss")</code> return <code>true</code></br>
-     * <code>isDateStrict("2011/03/27 22:17:11+0100", "yyyy/MM/dd HH:mm:ssZ")</code> return <code>true</code></br>
-     * <code>isDateStrict("20110327 021711+1900", "yyyyMMdd HHmmssZ")</code> return <code>false</code></br>
-     * </br>
+     * This validation uses Java 8 time tools.
+     * 
      * The range of time-zone offsets is restricted to -18:00 to 18:00 inclusive.
-     *
-     * @param stringDate the date to judge
-     * @param pattern the specified pattern, like: "yyyy-MM-dd HH:mm:ss")
-     * @return whether the stringDate is a date string with a right pattern.
+     * 
+     * @param stringDate (The <code>String</code> of the date to judge)
+     * @param pattern (The <code>String</code> of a specified pattern, like: "yyyy-MM-dd HH:mm:ss")
+     * @return A boolean value that whether the stringDate is a date string with a right pattern.
      * @throws IllegalArgumentException if pattern is not defined.
-     *
+     * 
+     * {talendTypes} Boolean
+     * 
+     * {Category} TalendDate
+     * 
+     * {param} String(mydate) stringDate : the date to judge
+     * 
+     * {param} String("yyyy-MM-dd HH:mm:ss") pattern : the specified pattern
+     * 
+     * {examples}
+     * 
+     * ->> isDateStrict("20110327 121711", "yyyyMMdd HHmmss") return true
+     * ->> isDateStrict("01100327 121711", "yyyyMMdd HHmmss") return false
+     * ->> isDateStrict("20180229 221711", "yyyyMMdd HHmmss") return false
+     * ->> isDateStrict("2016-02-29 22:17:11", "yyyy-MM-dd HH:mm:ss") return true
+     * ->> isDateStrict("2011/03/27 22:17:11+0100", "yyyy/MM/dd HH:mm:ssZ") return true
+     * ->> isDateStrict("20110327 021711+1900", "yyyyMMdd HHmmssZ") return false
      */
     public static boolean isDateStrict(String stringDate, String pattern) {
         if (stringDate == null) {

main/plugins/org.talend.librariesmanager/resources/java/routines/system/JobStructureCatcherUtils.java

@@ -21,17 +21,19 @@
 // ============================================================================
 package routines.system;
 
+import java.io.UnsupportedEncodingException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 //TODO split to several classes by the level when have a clear requirement or design : job, component, connection
 public class JobStructureCatcherUtils {
 
-	private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSZ");
+	private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSZ");
 
 	// TODO split it as too big, even for storing the reference only which point
 	// null
@@ -59,8 +61,6 @@ public class JobStructureCatcherUtils {
 
 		public String job_version;
 
-		public Long systemPid = JobStructureCatcherUtils.getPid();
-
 		public boolean current_connector_as_input;
 
 		public String current_connector_type;
@@ -84,11 +84,17 @@ public class JobStructureCatcherUtils {
 
 		public long end_time;
 
-		public String moment = sdf.format(new Date());
+		public String moment;
 
 		public String status;
 		
 		public LogType log_type;
+		
+		//process uuid
+		public String pid = ProcessIdAndThreadId.getProcessId();
+				
+		//thread uuid
+		public String tid = ProcessIdAndThreadId.getThreadId();
 
 		public JobStructureCatcherMessage() {
 		}
@@ -101,7 +107,10 @@ public class JobStructureCatcherUtils {
 		RUNCOMPONENT,
 		FLOWINPUT,
 		FLOWOUTPUT,
-		PERFORMANCE
+		PERFORMANCE,
+		
+		RUNTIMEPARAMETER,
+		RUNTIMESCHEMA
 	}
 
 	java.util.List<JobStructureCatcherMessage> messages = java.util.Collections
@@ -118,6 +127,42 @@ public class JobStructureCatcherUtils {
 		this.job_id = jobId;
 		this.job_version = jobVersion;
 	}
+	
+	public void addComponentParameterMessage(String component_id, String component_name, Map<String, String> component_parameters) {
+		JobStructureCatcherMessage scm = new JobStructureCatcherMessage();
+		scm.job_name = this.job_name;
+		scm.job_id = this.job_id;
+		scm.job_version = this.job_version;
+		
+		scm.component_id = component_id;
+		scm.component_name = component_name;
+		
+		scm.component_parameters = component_parameters;
+		
+		scm.log_type = LogType.RUNTIMEPARAMETER;
+		
+		messages.add(scm);
+	}
+	
+	public void addConnectionSchemaMessage(String source_component_id, String source_component_name, String target_component_id, String target_component_name, 
+			String current_connector, List<Map<String, String>> component_schema) {
+		JobStructureCatcherMessage scm = new JobStructureCatcherMessage();
+		scm.job_name = this.job_name;
+		scm.job_id = this.job_id;
+		scm.job_version = this.job_version;
+		
+		scm.current_connector = current_connector;
+		scm.sourceId = source_component_id;
+		scm.sourceComponentName = source_component_name;
+		scm.targetId = target_component_id;
+		scm.targetComponentName = target_component_name;
+		
+		scm.component_schema = component_schema;
+		
+		scm.log_type = LogType.RUNTIMESCHEMA;
+		
+		messages.add(scm);
+	}
 
 	public void addConnectionMessage(String component_id, String component_label, String component_name, boolean current_connector_as_input,
 			String current_connector_type, String current_connector, long total_row_number, long start_time,
@@ -163,6 +208,8 @@ public class JobStructureCatcherUtils {
 
 	public void addJobStartMessage() {
 		JobStructureCatcherMessage scm = new JobStructureCatcherMessage();
+		scm.moment = sdf.format(new Date());
+		
 		scm.job_name = this.job_name;
 		scm.job_id = this.job_id;
 		scm.job_version = this.job_version;
@@ -174,6 +221,8 @@ public class JobStructureCatcherUtils {
 
 	public void addJobEndMessage(long start_time, long end_time, String status) {
 		JobStructureCatcherMessage scm = new JobStructureCatcherMessage();
+		scm.moment = sdf.format(new Date());
+		
 		scm.job_name = this.job_name;
 		scm.job_id = this.job_id;
 		scm.job_version = this.job_version;
@@ -198,16 +247,6 @@ public class JobStructureCatcherUtils {
 		return messagesToSend;
 	}
 
-	public static long getPid() {
-		RuntimeMXBean mx = ManagementFactory.getRuntimeMXBean();
-		String[] mxNameTable = mx.getName().split("@");
-		if (mxNameTable.length == 2) {
-			return Long.parseLong(mxNameTable[0]);
-		} else {
-			return Thread.currentThread().getId();
-		}
-	}
-
 	public void addConnectionMessage4PerformanceMonitor(String current_connector, String sourceId, String sourceLabel,
 			String sourceComponentName, String targetId, String targetLabel, String targetComponentName, int row_count,
 			long start_time, long end_time) {

main/plugins/org.talend.librariesmanager/resources/java/routines/system/ProcessIdAndThreadId.java

@@ -0,0 +1,44 @@
+package routines.system;
+
+import java.io.UnsupportedEncodingException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.util.UUID;
+
+public class ProcessIdAndThreadId {
+	
+	private static class PTId {
+		String processId;
+		String threadId;
+	}
+
+	private static final ThreadLocal<PTId> Id = new ThreadLocal<PTId>() {
+		@Override
+		protected PTId initialValue() {
+			PTId id = new PTId();
+			id.processId = getPid();
+			id.threadId = UUID.randomUUID().toString();
+			return id;
+		}
+		
+	};
+	
+	private static String getPid() {
+		RuntimeMXBean mx = ManagementFactory.getRuntimeMXBean();
+		String processName = mx.getName();
+		try {
+			return UUID.nameUUIDFromBytes(processName.getBytes("UTF8")).toString();
+		} catch (UnsupportedEncodingException e) {
+		}
+		return null;
+	}
+	
+	public static String getProcessId() {
+		return Id.get().processId;
+	}
+	
+	public static String getThreadId() {
+		return Id.get().threadId;
+	}
+	
+}

main/plugins/org.talend.librariesmanager/resources/java/routines/system/RunStat.java

@@ -58,8 +58,16 @@ public class RunStat implements Runnable {
     
     private final JobStructureCatcherUtils jscu;
     
-    public RunStat(JobStructureCatcherUtils jscu) {
-    	this.jscu = jscu;
+    public RunStat(JobStructureCatcherUtils jscu, String interval) {
+        this.jscu = jscu;
+        
+        if(interval!=null) {
+            try {
+                this.interval = Long.valueOf(interval);
+            } catch(Exception e) {
+                //do nothing
+            }
+        }
     }
 
     private class StatBean {
@@ -342,7 +350,7 @@ public class RunStat implements Runnable {
 
     private Map<String, StatBean> processStats4Meter = new HashMap<String, StatBean>();
 
-    private final static long INTERVAL = 500;
+    private long interval = 500;
     
     private long lastLogUpdate = 0;
     
@@ -354,7 +362,7 @@ public class RunStat implements Runnable {
         StatBean stateBean = log(connectionId, mode, nbLine);
         
         long currentLogUpdate = System.currentTimeMillis();
-        if (lastLogUpdate == 0 || lastLogUpdate + INTERVAL < currentLogUpdate) {
+        if (lastLogUpdate == 0 || lastLogUpdate + interval < currentLogUpdate) {
             lastLogUpdate = currentLogUpdate;
             jscu.addConnectionMessage4PerformanceMonitor(
                 connectionId, sourceId, sourceLabel, sourceComponentName, targetId, targetLabel, targetComponentName, stateBean.nbLine, stateBean.startTime, currentLogUpdate);

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/librarydata/LibraryDataJsonProvider.java

@@ -23,8 +23,8 @@ import org.apache.log4j.Logger;
 import org.talend.commons.exception.ExceptionHandler;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
@@ -54,7 +54,7 @@ public class LibraryDataJsonProvider {
                 ExceptionHandler.process(e);
             }
         } else {
-            log.error("Can't find license data file:" + dataFile.getAbsolutePath());
+            log.warn("Can't find license data file:" + dataFile.getAbsolutePath());
         }
 
         return mvnToLibraryMap;

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/librarydata/LibraryDataService.java

@@ -90,16 +90,28 @@ public class LibraryDataService {
         buildLibraryIfLicenseMissing = Boolean
                 .valueOf(System.getProperty(KEY_BUILD_LIBRARY_IF_LICENSE_MISSING, Boolean.FALSE.toString()));
         buildLibraryJarFile = Boolean.valueOf(System.getProperty(KEY_LIBRARIES_BUILD_JAR, Boolean.FALSE.toString()));
-        File libraryDataFile = getLibraryDataFile();
+        File studioLibraryDataFile = getStudioLibraryDataFile();
         if (buildLibraryLicense) {
-            if (libraryDataFile.exists()) {
-                libraryDataFile.delete();
+            if (studioLibraryDataFile.exists()) {
+                studioLibraryDataFile.delete();
             }
         }
         unknownLicense = new LibraryLicense();
         unknownLicense.setName(UNRESOLVED_LICENSE_NAME);
-        dataProvider = new LibraryDataJsonProvider(libraryDataFile);
-        mvnToLibraryMap.putAll(dataProvider.loadLicenseData());
+        dataProvider = new LibraryDataJsonProvider(studioLibraryDataFile);
+        File currentUserDataFile = getCurrentUserLibraryDataFile();
+        Map<String, Library> studioLibraryDataMap = dataProvider.loadLicenseData();
+        if (!StringUtils.equals(currentUserDataFile.getAbsolutePath(), studioLibraryDataFile.getAbsolutePath())) {
+            dataProvider = new LibraryDataJsonProvider(currentUserDataFile);
+            Map<String, Library> userLibraryDataMap = dataProvider.loadLicenseData();
+            if (userLibraryDataMap.size() == 0) {
+                mvnToLibraryMap.putAll(studioLibraryDataMap);
+            } else {
+                mvnToLibraryMap.putAll(userLibraryDataMap);
+            }
+        } else {
+            mvnToLibraryMap.putAll(studioLibraryDataMap);
+        }
     }
 
     public static LibraryDataService getInstance() {
@@ -321,13 +333,22 @@ public class LibraryDataService {
         if (buildLibraryLicense) {
             return true;
         }
-        if (buildLibraryIfFileMissing && !getLibraryDataFile().exists()) {
+        if (buildLibraryIfFileMissing && !getStudioLibraryDataFile().exists()) {
             return true;
         }
         return false;
     }
 
-    private File getLibraryDataFile() {
+    /**
+     * For shared studio, user's library data file not same with studio one
+     * 
+     * @return
+     */
+    private File getCurrentUserLibraryDataFile() {
+        return new File(Platform.getConfigurationLocation().getURL().getPath(), LIBRARIES_DATA_FILE_NAME);
+    }
+
+    private File getStudioLibraryDataFile() {
         String folder = System.getProperty(KEY_LIBRARIES_DATA_FOLDER);
         if (folder == null) {
             folder = new File(Platform.getInstallLocation().getURL().getPath(), "configuration").getAbsolutePath(); //$NON-NLS-1$

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/AbstractLibrariesService.java

@@ -318,4 +318,8 @@ public abstract class AbstractLibrariesService implements ILibrariesService {
             }
         }
     }
+
+	@Override
+	public void setForceReloadCustomUri() {		
+	}  
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/CustomUriManager.java

@@ -46,6 +46,8 @@ public class CustomUriManager {
     private static final String CUSTOM_URI_MAP = "custom_uri_mapping.json";
 
     private static long lastModified = 0;
+    
+    private static boolean isNeedReload = false;
 
     private CustomUriManager() {
         try {
@@ -159,15 +161,19 @@ public class CustomUriManager {
         try {
             File file = new File(getResourcePath(), CUSTOM_URI_MAP);
             long modifyDate = file.lastModified();
-            if (modifyDate > lastModified) {
+            if (isNeedReload || modifyDate > lastModified) {
                 customURIObject.clear();
                 JSONObject loadResources = loadResources(getResourcePath(), CUSTOM_URI_MAP);
                 customURIObject.putAll(loadResources);
                 lastModified = modifyDate;
+                isNeedReload = false;
             }
         } catch (IOException e) {
             ExceptionHandler.process(e);
         }
     }
-
+    
+    public void setForeceReloadCustomUri() {
+    	isNeedReload = true;
+    }
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LibrariesService.java

@@ -267,5 +267,9 @@ public class LibrariesService implements ILibrariesService {
     public void deployLibrary(URL source, String mavenUri, boolean refresh, boolean updateNexusJar) throws IOException {
         getLibrariesService().deployLibrary(source, mavenUri, refresh, updateNexusJar);
     }
+    
+    public void setForceReloadCustomUri() {
+    	CustomUriManager.getInstance().setForeceReloadCustomUri();
+    }
 
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -82,6 +82,7 @@ import org.talend.core.runtime.services.IMavenUIService;
 import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.designer.maven.tools.BuildCacheManager;
 import org.talend.designer.maven.utils.PomUtil;
+import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.librariesmanager.maven.MavenArtifactsHandler;
 import org.talend.librariesmanager.model.ExtensionModuleManager;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
@@ -1054,7 +1055,13 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
             MavenArtifact ma = MavenUrlHelper.parseMvnUrl(mvnUri);
             if (ma != null) {
                 String repositoryUrl = ma.getRepositoryUrl();
-                if (repositoryUrl == null || repositoryUrl.trim().isEmpty()
+                boolean isCIMode = false;
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+                    IRunProcessService runProcessService = GlobalServiceRegister.getDefault()
+                            .getService(IRunProcessService.class);
+                    isCIMode = runProcessService.isCIMode();
+                }
+                if (isCIMode || repositoryUrl == null || repositoryUrl.trim().isEmpty()
                         || MavenConstants.LOCAL_RESOLUTION_URL.equalsIgnoreCase(repositoryUrl)) {
                     return;
                 }
@@ -1385,7 +1392,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                 snapshotResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, false, true);
                 if (snapshotResult != null) {
                     for (MavenArtifact result : snapshotResult) {
-                        ShareLibrariesUtil.putArtifactToMap(result, releaseArtifactMap, true);
+                        ShareLibrariesUtil.putArtifactToMap(result, snapshotArtifactMap, true);
                     }
                 }
                 releaseResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, true, false);