From f38ca7f344ec7febcf7fd3979ccc90ce9a087bd0 Mon Sep 17 00:00:00 2001 From: bhe-talendbj Date: Thu, 26 Oct 2023 09:44:27 +0800 Subject: [PATCH] feat(TUP-40390): run old migrations as lazy --- .../model/ProxyRepositoryFactory.java | 19 +++ .../migration/AbstractItemMigrationTask.java | 6 +- .../migration/IMigrationToolService.java | 72 +++++++- .../migration/AbstractMigrationTask.java | 10 ++ .../org/talend/migration/IMigrationTask.java | 4 + .../migrationtool/MigrationToolService.java | 155 +++++++++++++----- .../migrationtool/model/GetTasksHelper.java | 17 +- 7 files changed, 241 insertions(+), 42 deletions(-) diff --git a/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java b/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java index 9b3c1a3a9a..696ca76498 100644 --- a/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java +++ b/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -2129,6 +2130,21 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory { } } } + + private void checkLazyMigrations() { + IMigrationToolService ms = null; + if (GlobalServiceRegister.getDefault().isServiceRegistered(IMigrationToolService.class)) { + ms = GlobalServiceRegister.getDefault().getService(IMigrationToolService.class); + } + if (ms != null) { + Set invalids = ms.validateLazyMigrations(); + if (!invalids.isEmpty()) { + RuntimeException e = new RuntimeException("Invalid lazy migrations found! " + invalids.toString()); + ExceptionHandler.process(e); + throw e; + } + } + } /** * DOC tang Comment method "logOnProject". @@ -2248,6 +2264,9 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory { } catch (Throwable e) { ExceptionHandler.process(e); } + + // check lazy migrations + checkLazyMigrations(); // Check reference project setting problems checkReferenceProjectsProblems(project); diff --git a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java index 61a9867c23..60f125e842 100644 --- a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java +++ b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java @@ -103,6 +103,11 @@ public abstract class AbstractItemMigrationTask extends AbstractMigrationTask im @Override public ExecutionResult execute(Project project, Item item) { + // remove lazy types from old task + if (IMigrationToolService.removeLazyTypesFromOldTask(this)) { + log.info("removed lazy types from old task: " + this.getId()); + } + if (!getAllTypes().contains(ERepositoryObjectType.getItemType(item))) { return ExecutionResult.NOTHING_TO_DO; } @@ -160,7 +165,6 @@ public abstract class AbstractItemMigrationTask extends AbstractMigrationTask im protected List getAllTypes() { List declaredTypes = getTypes(); ArrayList allTypes = new ArrayList(declaredTypes.size()); - allTypes.addAll(declaredTypes); allTypes.addAll(getExtendedTypes()); return allTypes; } diff --git a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/IMigrationToolService.java b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/IMigrationToolService.java index f549cb236b..716117a60a 100644 --- a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/IMigrationToolService.java +++ b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/IMigrationToolService.java @@ -12,13 +12,21 @@ // ============================================================================ package org.talend.core.model.migration; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.eclipse.core.runtime.IProgressMonitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.talend.commons.utils.VersionUtils; import org.talend.core.IService; import org.talend.core.model.general.Project; import org.talend.core.model.properties.Item; import org.talend.core.model.properties.MigrationTask; +import org.talend.migration.IProjectMigrationTask; +import org.talend.core.model.repository.ERepositoryObjectType; /** * DOC smallet class global comment. Detailled comment
@@ -27,7 +35,7 @@ import org.talend.core.model.properties.MigrationTask; * */ public interface IMigrationToolService extends IService { - + public void executeWorspaceTasks(); public void initNewProjectTasks(Project project); @@ -74,5 +82,67 @@ public interface IMigrationToolService extends IService { * @param item given item */ public void executeLazyMigrations(Project project, Item item) throws Exception; + + /** + * Validate all of lazy migration tasks whether lazy migration task was performed on other types of item except + * process items. + * + * @return + */ + public Set validateLazyMigrations(); + + public static final String SYS_PROP_CHECK_LAZY_MIGRATIONS = "lazyMigrationCheck"; + + public static final String SYS_PROP_CHECK_LAZY_MIGRATIONS_DEFAULT = "true"; + + public static final String SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS = "execOldTaskAsLazy"; + + public static final String SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS_DEFAULT = "true"; + + public static final String SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS_BREAKS = "execOldTaskAsLazyBreaks"; + + public static final String SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS_BREAKS_DEFAULT = "8.0.0"; + + public static boolean checkLazyMigrations() { + String val = System.getProperty(SYS_PROP_CHECK_LAZY_MIGRATIONS, SYS_PROP_CHECK_LAZY_MIGRATIONS_DEFAULT); + return Boolean.parseBoolean(val); + } + + public static boolean execOldTaskAsLazy() { + String val = System.getProperty(SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS, SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS_DEFAULT); + return Boolean.parseBoolean(val); + } + + public static String getExecOldTaskAsLazyBreaks() { + return System.getProperty(SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS_BREAKS, SYS_PROP_EXEC_OLD_AS_LAZY_MIGRATIONS_BREAKS_DEFAULT); + } + + public static boolean isLazyTypes(Collection types) { + Set ts = new HashSet(types); + List lts = ERepositoryObjectType.getAllTypesOfProcess2(); + for (ERepositoryObjectType t : lts) { + if (ts.contains(t)) { + return true; + } + } + return false; + } + + public static boolean canRunAsLazy(IProjectMigrationTask t) { + if (t instanceof AbstractItemMigrationTask) { + if (VersionUtils.compareTo(t.getBreaks(), getExecOldTaskAsLazyBreaks()) >= 0 && isLazyTypes(((AbstractItemMigrationTask) t).getTypes())) { + return true; + } + } + return false; + } + + public static boolean removeLazyTypesFromOldTask(IProjectMigrationTask t) { + if (!t.isLazy() && canRunAsLazy(t)) { + ((AbstractItemMigrationTask) t).getTypes().removeAll(ERepositoryObjectType.getAllTypesOfProcess2()); + return true; + } + return false; + } } diff --git a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/AbstractMigrationTask.java b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/AbstractMigrationTask.java index 0e50290c8d..4007565be4 100644 --- a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/AbstractMigrationTask.java +++ b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/AbstractMigrationTask.java @@ -35,6 +35,8 @@ public abstract class AbstractMigrationTask implements IMigrationTask{ private ExecutionResult status; private boolean isLazy = false; + + private String bundleName; /** * DOC smallet AbstractMigrationTask constructor comment. @@ -116,4 +118,12 @@ public abstract class AbstractMigrationTask implements IMigrationTask{ public void setLazy(boolean isLazy) { this.isLazy = isLazy; } + + public String getBundleSymbolicName() { + return bundleName; + } + + public void setBundleSymbolicName(String bundleName) { + this.bundleName = bundleName; + } } diff --git a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/IMigrationTask.java b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/IMigrationTask.java index 3321b32e99..9b84a7b23e 100644 --- a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/IMigrationTask.java +++ b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/migration/IMigrationTask.java @@ -101,4 +101,8 @@ public interface IMigrationTask { public void setLazy(boolean isLazy); + public String getBundleSymbolicName(); + + public void setBundleSymbolicName(String bundleName); + } diff --git a/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java b/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java index 5ed2bd8483..f8a67edd62 100644 --- a/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java +++ b/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java @@ -20,11 +20,15 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.GregorianCalendar; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import java.util.HashSet; import java.util.stream.Collectors; import java.util.stream.Stream; import java.text.ParseException; @@ -47,6 +51,8 @@ import org.eclipse.core.runtime.SubProgressMonitor; import org.eclipse.core.runtime.jobs.ISchedulingRule; import org.eclipse.emf.common.util.EList; import org.eclipse.emf.ecore.resource.Resource; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; import org.talend.commons.exception.ExceptionHandler; import org.talend.commons.exception.LoginException; import org.talend.commons.exception.PersistenceException; @@ -57,6 +63,7 @@ import org.talend.commons.utils.VersionUtils; import org.talend.core.GlobalServiceRegister; import org.talend.core.ICoreService; import org.talend.core.model.general.Project; +import org.talend.core.model.migration.AbstractItemMigrationTask; import org.talend.core.model.migration.IMigrationToolService; import org.talend.core.model.process.JobInfo; import org.talend.core.model.properties.Item; @@ -125,6 +132,8 @@ public class MigrationToolService implements IMigrationToolService { private static final String MIGRATION_FAILED_PROP = "migration_failed"; + private static final GregorianCalendar LAZY_MIGRATION_RELEASE_TIME = new GregorianCalendar(2023, 10, 10, 12, 0, 0); + public MigrationToolService() { doneThisSession = new ArrayList(); } @@ -951,6 +960,11 @@ public class MigrationToolService implements IMigrationToolService { } public void executeLazyMigrations(Project project, Item item) throws Exception { + if (ProcessorUtilities.isCIMode()) { + log.info("CI mode, lazy migration is disabled"); + return; + } + if (project == null) { project = ProjectManager.getInstance().getCurrentProject(); } @@ -958,6 +972,37 @@ public class MigrationToolService implements IMigrationToolService { throw new IllegalArgumentException("item is not process item or joblet process item"); } + // get all lazy tasks + List allLazyTasks = getLazyMigrationTasks(); + if (allLazyTasks == null || allLazyTasks.isEmpty()) { + log.info("No lazy migration tasks found!"); + return; + } + + // execute old migration tasks as lazy migrations + if (IMigrationToolService.execOldTaskAsLazy()) { + log.info("size of allLazyTasks: " + allLazyTasks.size()); + // load old migration tasks + List allNonLazyTasks = getNonLazyMigrationTasks(); + for (IProjectMigrationTask t : allNonLazyTasks) { + if (IMigrationToolService.canRunAsLazy(t)) { + allLazyTasks.add(t); + } + } + log.info("size of allLazyTasks after loading old migrations: " + allLazyTasks.size()); + } + + // filling bundleId + for (IProjectMigrationTask t : allLazyTasks) { + Bundle b = FrameworkUtil.getBundle(t.getClass()); + if (b == null) { + t.setBundleSymbolicName(MIGRATION_ORDER_PROP); + log.info("Can not find bundle for task: " + t.getId()); + } else { + t.setBundleSymbolicName(b.getSymbolicName()); + } + } + // get all of children final List targetedItems = new ArrayList(); targetedItems.add(item); @@ -973,51 +1018,46 @@ public class MigrationToolService implements IMigrationToolService { } for (Item targetItem : targetedItems) { - executeLazyMigrationsInternal(project, targetItem); + executeLazyMigrationsInternal(allLazyTasks, project, targetItem); } } - protected void executeLazyMigrationsInternal(Project project, Item item) throws Exception { + protected void executeLazyMigrationsInternal(final List allLazyTasks, Project project, Item item) throws Exception { if (project == null) { project = ProjectManager.getInstance().getCurrentProject(); } if (!(item instanceof ProcessItem || item instanceof JobletProcessItem)) { throw new IllegalArgumentException("item is not process item or joblet process item"); } - - // get all lazy tasks - List allLazyTasks = getLazyMigrationTasks(); + // check whether needs to migrate or not - List nonExecutedLazyMigrationTasks = null; - Object migrationOrderObj = item.getProperty().getAdditionalProperties().get(MIGRATION_ORDER_PROP); - if (migrationOrderObj != null) { - try { - Date migrationOrder = DATE_FORMATTER.parse(migrationOrderObj.toString()); - nonExecutedLazyMigrationTasks = getNonExecutedLazyMigrationTasks(allLazyTasks, migrationOrder); - } catch (ParseException e) { - ExceptionHandler.process(e); - } - } else { - nonExecutedLazyMigrationTasks = allLazyTasks; - } - if (nonExecutedLazyMigrationTasks == null || nonExecutedLazyMigrationTasks.isEmpty()) { +// List nonExecutedLazyMigrationTasks = null; +// Object migrationOrderObj = item.getProperty().getAdditionalProperties().get(MIGRATION_ORDER_PROP); +// if (migrationOrderObj != null) { +// try { +// Date migrationOrder = DATE_FORMATTER.parse(migrationOrderObj.toString()); +// nonExecutedLazyMigrationTasks = getNonExecutedLazyMigrationTasks(allLazyTasks, migrationOrder); +// } catch (ParseException e) { +// ExceptionHandler.process(e); +// } +// } else { +// nonExecutedLazyMigrationTasks = allLazyTasks; +// } + if (allLazyTasks == null || allLazyTasks.isEmpty()) { log .info("No lazy migration tasks for project: " + project.getTechnicalLabel() + ", item id: " + item.getProperty().getId() + ", item display name: " + item.getProperty().getDisplayName()); return; } // execute migrations - // IRepositoryService migRepoService = (IRepositoryService) - // GlobalServiceRegister.getDefault().getService(IRepositoryService.class); - // final IProxyRepositoryFactory migRepoFactory = migRepoService.getProxyRepositoryFactory(); - - ProxyRepositoryFactory migRepoFactory = ProxyRepositoryFactory.getInstance(); + IRepositoryService migRepoService = (IRepositoryService) GlobalServiceRegister.getDefault().getService(IRepositoryService.class); + final IProxyRepositoryFactory migRepoFactory = migRepoService.getProxyRepositoryFactory(); final Project projectMig = project; - final List nonExecutedLazyMigrationList = nonExecutedLazyMigrationTasks; +// final List nonExecutedLazyMigrationList = nonExecutedLazyMigrationTasks; log .info("lazy migration tasks for project: " + project.getTechnicalLabel() + ", item id: " + item.getProperty().getId() + ", item display name: " + item.getProperty().getDisplayName() - + ", size: " + nonExecutedLazyMigrationList.size()); + + ", size: " + allLazyTasks.size()); String taskDesc = "executeLazyMigrations on project: " + project.getTechnicalLabel(); log.trace(taskDesc); try { @@ -1045,13 +1085,27 @@ public class MigrationToolService implements IMigrationToolService { ExceptionHandler.process(e); } } - - for (IProjectMigrationTask t : nonExecutedLazyMigrationList) { + Map migrationOrderMap = new HashMap(); + for (IProjectMigrationTask t : allLazyTasks) { + Object migrationOrderObj = item.getProperty().getAdditionalProperties().get(t.getBundleSymbolicName()); + if (migrationOrderObj != null) { + try { + Date migrationOrder = DATE_FORMATTER.parse(migrationOrderObj.toString()); + if (!t.getOrder().after(migrationOrder)) { + // already executed + continue; + } + } catch (ParseException e) { + ExceptionHandler.process(e); + } + } try { ExecutionResult res = t.execute(projectMig, tempItem); t.setStatus(res); if (t.getStatus() == ExecutionResult.FAILURE) { failedMigrations.add(t.getId()); + } else { + migrationOrderMap.put(t.getBundleSymbolicName(), t.getOrder()); } } catch (Exception e) { ExceptionHandler.process(e); @@ -1070,7 +1124,10 @@ public class MigrationToolService implements IMigrationToolService { Property prop = tempItem.getProperty(); // save latest migration order into properties - prop.getAdditionalProperties().put(MIGRATION_ORDER_PROP, DATE_FORMATTER.format(nonExecutedLazyMigrationList.get(nonExecutedLazyMigrationList.size() - 1).getOrder())); + migrationOrderMap.forEach((k, v) -> { + prop.getAdditionalProperties().put(k, DATE_FORMATTER.format(v)); + }); + if (!failedMigrations.isEmpty()) { log .info("lazy migration tasks for project: " + projectMig.getTechnicalLabel() + ", item id: " + item.getProperty().getId() + ", item display name: " @@ -1079,7 +1136,7 @@ public class MigrationToolService implements IMigrationToolService { } try { - migRepoFactory.save(tempItem); + migRepoFactory.save(tempItem.getProperty()); } catch (PersistenceException e) { ExceptionHandler.process(e); } @@ -1115,16 +1172,40 @@ public class MigrationToolService implements IMigrationToolService { } private static List getLazyMigrationTasks() { - List ret = new ArrayList(); List lazyTasks = GetTasksHelper.getProjectTasks(null, true); - if (lazyTasks != null && !lazyTasks.isEmpty()) { - ret.addAll(lazyTasks); - } - // sort - sortMigrationTasks(ret); - - return ret; + sortMigrationTasks(lazyTasks); + return lazyTasks; + } + + private static List getNonLazyMigrationTasks() { + List nonLazyTasks = GetTasksHelper.getProjectTasks(null, false); + // sort + sortMigrationTasks(nonLazyTasks); + return nonLazyTasks; } + public Set validateLazyMigrations() { + Set invalids = new HashSet(); + if (!IMigrationToolService.checkLazyMigrations()) { + return invalids; + } + List allLazyTasks = getLazyMigrationTasks(); + for (IProjectMigrationTask t : allLazyTasks) { + if (t instanceof AbstractItemMigrationTask) { + // only validate new lazy migrations added after lazy migration system releases + if (!t.getOrder().after(LAZY_MIGRATION_RELEASE_TIME.getTime())) { + continue; + } + AbstractItemMigrationTask at = (AbstractItemMigrationTask) t; + // though we can not guarantee that the migration task is lazy, + // at least this migration task may modify other things such as context or metadata etc. + if (!IMigrationToolService.isLazyTypes(at.getTypes())) { + invalids.add(at.getId()); + } + } + } + + return invalids; + } } diff --git a/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/model/GetTasksHelper.java b/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/model/GetTasksHelper.java index e9b939371a..c5c2209e3b 100644 --- a/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/model/GetTasksHelper.java +++ b/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/model/GetTasksHelper.java @@ -26,6 +26,7 @@ import org.talend.commons.utils.workbench.extensions.ExtensionPointLimiterImpl; import org.talend.commons.utils.workbench.extensions.IExtensionPointLimiter; import org.talend.core.model.properties.MigrationTask; import org.talend.core.model.utils.MigrationUtil; +import org.talend.designer.runprocess.ProcessorUtilities; import org.talend.migration.IMigrationTask; import org.talend.migration.IProjectMigrationTask; import org.talend.migration.IWorkspaceMigrationTask; @@ -112,7 +113,7 @@ public class GetTasksHelper { String version = configurationElement.getAttribute("version"); //$NON-NLS-1$ String breaks = configurationElement.getAttribute("breaks"); //$NON-NLS-1$ String isLazy = configurationElement.getAttribute("isLazy"); - if (Boolean.parseBoolean(isLazy)) { + if (!isCIMode() && Boolean.parseBoolean(isLazy)) { return null; } return MigrationUtil.createMigrationTask(id, version, breaks, MigrationUtil.DEFAULT_STATUS); @@ -162,8 +163,14 @@ public class GetTasksHelper { } } }; - - return provider.createInstances().stream().filter(t -> t.isLazy() == isLazy).collect(Collectors.toList()); + return provider.createInstances().stream().filter(t -> { + if (isCIMode()) { + return true; + } else { + return t.isLazy() == isLazy; + } + }).collect(Collectors.toList()); +// return provider.createInstances().stream().filter(t -> t.isLazy() == isLazy).collect(Collectors.toList()); } public static List getWorkspaceTasks() { @@ -209,4 +216,8 @@ public class GetTasksHelper { return currentAction; } + + private static boolean isCIMode() { + return ProcessorUtilities.isCIMode(); + } }