diff --git a/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java b/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java index 01c442b635..474bb9250a 100644 --- a/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java +++ b/main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java @@ -2283,6 +2283,9 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory { IRunProcessService runProcessService = getRunProcessService(); if (runProcessService != null) { runProcessService.initMavenJavaProject(monitor, project); + + // before afterLogon migration execute, check and update daikon dependency + runProcessService.checkAndUpdateDaikonDependencies(); } currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE); diff --git a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java index 22763e052c..afd1704783 100644 --- a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java +++ b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java @@ -728,6 +728,46 @@ public final class ProcessUtils { } return null; } + + private static boolean isRoute(Property property) { + return property!= null && (ERepositoryObjectType.getType(property).equals(ERepositoryObjectType.PROCESS_ROUTE) || + ERepositoryObjectType.getType(property).equals(ERepositoryObjectType.PROCESS_ROUTE_MICROSERVICE)); + } + + public static boolean isRouteWithChildJobs(IProcess process) { + + if (process instanceof IProcess2) { + Property p = ((IProcess2) process).getProperty(); + if (isRoute(p)) { + return false; + } + Item item = p.getItem(); + return isRouteWithChildJobs(item); + } else { + for (INode node : process.getGraphicalNodes()) { + if (node.getComponent().getName().equals("cTalendJob")) { + return true; + } + } + } + return false; + } + + + public static boolean isRouteWithChildJobs(Item item) { + + if (item!= null && item instanceof ProcessItem) { + for (Object obj : ((ProcessItem) item).getProcess().getNode()) { + if (obj instanceof NodeType) { + if (((NodeType) obj).getComponentName().equals("cTalendJob")) { + return true; + } + } + } + } + return false; + + } public static int getAssertAmount(IProcess process) { int count = 0; diff --git a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java index 6d7805a1ab..31bc4c3a8d 100644 --- a/main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java +++ b/main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java @@ -275,4 +275,6 @@ public interface IRunProcessService extends IService { public void deleteOldVersionTalendJobProject(IRepositoryViewObject object); + public void checkAndUpdateDaikonDependencies(); + } diff --git a/main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java b/main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java index b04f94c4fd..0130a0b6c2 100644 --- a/main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java +++ b/main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java @@ -198,6 +198,8 @@ public class ProcessorUtilities { private static boolean isDynamicJobAndCITest = false; private static JobInfo mainJobInfo; + + private static boolean needExportItemsForDQ = false; public static void addOpenEditor(IEditorPart editor) { openedEditors.add(editor); @@ -699,7 +701,7 @@ public class ProcessorUtilities { processor.setArguments(argumentsMap); - copyDQDroolsToSrc(selectedProcessItem); + handelDQComponents(selectedProcessItem, currentProcess); // generate the code of the father after the childrens // so the code won't have any error during the check, and it will help to check // if the generation is really needed. @@ -1243,7 +1245,7 @@ public class ProcessorUtilities { processor.setArguments(argumentsMap); - copyDQDroolsToSrc(selectedProcessItem); + handelDQComponents(selectedProcessItem, currentProcess); generateContextInfo(jobInfo, selectedContextName, statistics, trace, needContext, progressMonitor, currentProcess, currentJobName, processor, isMainJob, codeGenerationNeeded); @@ -1380,12 +1382,13 @@ public class ProcessorUtilities { } /** - * - * copy the current item's drools file from 'workspace/metadata/survivorship' to '.Java/src/resources' - * + * + * Specail operation for DQ components: + * - For tdqReportRun, set 'needExportItemsForDQ'to true so as the item folder can be exported + * - For tRuleSurvivorship, copy the current item's drools file from 'workspace/metadata/survivorship' to '.Java/src/resources' * @param processItem */ - private static void copyDQDroolsToSrc(ProcessItem processItem) { + private static void handelDQComponents(ProcessItem processItem,IProcess currentProcess) { // 1.TDQ-12474 copy the "metadata/survivorship/rulePackage" to ".Java/src/main/resources/". so that it will be // used by // maven command 'include-survivorship-rules' to export. @@ -1397,6 +1400,20 @@ public class ProcessorUtilities { return; } try { + // TDQ-19637 if it includes 'tDqReportRun',must export item folder + if (!needExportItemsForDQ) { + for (INode node : currentProcess.getGeneratingNodes()) { + String componentName = node.getComponent().getName(); + if ("tDqReportRun".equals(componentName)) { + needExportItemsForDQ = true; + break; + } + } + } + /* 1.TDQ-12474 copy the "metadata/survivorship/rulePackage" to ".Java/src/main/resources/". so that it will be + used by maven command 'include-survivorship-rules' to export. + 2.TDQ-14308 current drools file in 'src/resourcesmetadata/survivorship/' should be included to job jar. + */ ExportFileResource resouece = new ExportFileResource(); BuildExportManager.getInstance().exportDependencies(resouece, processItem); if (resouece.getAllResources().isEmpty()) { @@ -2920,4 +2937,8 @@ public class ProcessorUtilities { public static boolean isDynamicJobAndCITest() { return isDynamicJobAndCITest; } + + public static boolean isNeedExportItemsForDQ() { + return needExportItemsForDQ; + } } diff --git a/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml b/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml index de6fdcf03c..785bd2ed5d 100644 --- a/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml +++ b/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml @@ -35,6 +35,11 @@ commons-io 2.8.0 + + org.apache.maven + maven-core + 3.8.1 + diff --git a/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-7/pom.xml b/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-7/pom.xml index 0380819731..6d23b45323 100644 --- a/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-7/pom.xml +++ b/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-7/pom.xml @@ -85,6 +85,11 @@ plexus-archiver 3.6.0 + + org.apache.maven + maven-core + 3.8.1 + diff --git a/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/talend-compiler-plugin/pom.xml b/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/talend-compiler-plugin/pom.xml index 0477074564..efce270f74 100644 --- a/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/talend-compiler-plugin/pom.xml +++ b/main/plugins/org.talend.designer.maven.tos/resources/build/plugins/talend-compiler-plugin/pom.xml @@ -10,6 +10,11 @@ talend-compiler-plugin-tos pom + + org.apache.maven + maven-core + 3.8.1 + org.eclipse.tycho tycho-compiler-jdt @@ -23,6 +28,10 @@ org.codehaus.plexus plexus-archiver + + org.apache.maven + maven-core + diff --git a/main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml b/main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml index 67fb4a8d49..5aaffc3189 100644 --- a/main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml +++ b/main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml @@ -30,6 +30,11 @@ + + org.apache.maven + maven-core + 3.8.1 + org.apache.maven.plugins maven-compiler-plugin @@ -39,6 +44,10 @@ org.apache.maven.shared maven-shared-utils + + org.apache.maven + maven-core + @@ -50,6 +59,10 @@ org.apache.maven.shared maven-shared-utils + + org.apache.maven + maven-core + @@ -61,6 +74,10 @@ org.apache.maven.shared maven-shared-utils + + org.apache.maven + maven-core + @@ -75,11 +92,15 @@ org.apache.maven.shared maven-shared-utils - + + + org.apache.maven + maven-core + org.apache.maven maven-compat - + diff --git a/main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml b/main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml index 81fd9a888e..4d7a3a6a44 100644 --- a/main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml +++ b/main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml @@ -66,6 +66,11 @@ plexus-archiver 3.6.0 + + org.apache.maven + maven-core + 3.8.1 + @@ -83,6 +88,11 @@ maven-shared-utils 3.3.3 + + org.apache.maven + maven-core + 3.8.1 + @@ -115,6 +125,11 @@ plexus-io 3.0.0 + + org.apache.maven + maven-core + 3.8.1 + @@ -152,6 +167,11 @@ plexus-archiver 3.6.0 + + org.apache.maven + maven-core + 3.8.1 + @@ -164,6 +184,11 @@ plexus-utils 3.0.24 + + org.apache.maven + maven-core + 3.8.1 + diff --git a/main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java b/main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java index d0fa61addf..ccbd540727 100644 --- a/main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java +++ b/main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java @@ -15,6 +15,7 @@ package org.talend.designer.maven.tools; import static org.talend.designer.maven.model.TalendJavaProjectConstants.*; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -28,7 +29,9 @@ import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.maven.artifact.versioning.DefaultArtifactVersion; import org.apache.maven.model.Activation; +import org.apache.maven.model.Dependency; import org.apache.maven.model.Model; import org.apache.maven.model.Profile; import org.eclipse.core.resources.IContainer; @@ -1053,4 +1056,45 @@ public class AggregatorPomsHelper { return null; } + public void checkAndUpdateDaikonDependencies() { + try { + IFile rootPomFile = getProjectRootPom(); + if (rootPomFile.exists()) { + boolean isRegeneratePoms = false; + Model model = MavenPlugin.getMaven().readModel(rootPomFile.getLocation().toFile()); + if (model.getModules() != null) { + for (String module : model.getModules()) { + java.nio.file.Path modPath = Paths.get(rootPomFile.getLocation().toFile().getParent(), module, "pom.xml"); + Model modModel = MavenPlugin.getMaven().readModel(modPath.toFile()); + List deps = modModel.getDependencies(); + if (deps != null && !deps.isEmpty()) { + DefaultArtifactVersion targetVersion = new DefaultArtifactVersion("0.31.12"); + + for (Dependency dep : deps) { + DefaultArtifactVersion actualVersion = new DefaultArtifactVersion(dep.getVersion()); + int cmp = targetVersion.compareTo(actualVersion); + + if (StringUtils.equals("org.talend.daikon", dep.getGroupId()) && cmp > 0) { + if (StringUtils.equals("crypto-utils", dep.getArtifactId()) + || StringUtils.equals("daikon", dep.getArtifactId())) { + isRegeneratePoms = true; + } + } + } + } + if (isRegeneratePoms) { + break; + } + } + } + if (isRegeneratePoms) { + syncAllPomsWithoutProgress(new NullProgressMonitor()); + BuildCacheManager.getInstance().clearAllCodesCache(); + } + } + } catch (Exception e) { + ExceptionHandler.process(e); + } + } + } diff --git a/main/plugins/org.talend.librariesmanager/resources/java/routines/system/ResumeUtil.java b/main/plugins/org.talend.librariesmanager/resources/java/routines/system/ResumeUtil.java index 8399b36f79..68a7f67e44 100644 --- a/main/plugins/org.talend.librariesmanager/resources/java/routines/system/ResumeUtil.java +++ b/main/plugins/org.talend.librariesmanager/resources/java/routines/system/ResumeUtil.java @@ -13,10 +13,13 @@ package routines.system; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.io.Writer; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -53,7 +56,7 @@ public class ResumeUtil { private static Object lock = new Object(); // step1: init the log file name - public ResumeUtil(String logFileName, boolean createNewFile, String root_pid) { + public ResumeUtil(String logFileName, boolean append, String root_pid) { if (logFileName == null || logFileName.equals("null")) { return; } @@ -70,7 +73,13 @@ public class ResumeUtil { File file = new File(logFileName); try { if (sharedWriter == null) { - this.csvWriter = new SimpleCsvWriter(new FileWriter(logFileName, createNewFile)); + FileChannel fc = new RandomAccessFile(logFileName, "rw").getChannel(); + if(append){ + fc.position(fc.size()); + }else { + fc.truncate(0); + } + this.csvWriter = new SimpleCsvWriter(fc); // output the header part if (file.length() == 0) { @@ -97,9 +106,6 @@ public class ResumeUtil { csvWriter.write("dynamicData");// dynamicData csvWriter.endRecord(); csvWriter.flush(); - csvWriter.close(); - // To avoid use File.delete() as it cannot make sure file being deleted. - this.csvWriter = new SimpleCsvWriter(new FileWriter(logFileName, true)); } // shared sharedWriterMap.put(this.root_pid, this.csvWriter); @@ -138,8 +144,10 @@ public class ResumeUtil { JobLogItem item = new JobLogItem(eventDate, type, partName, parentPart, threadId, logPriority, errorCode, message, stackTrace, dynamicData); + FileLock fileLock = null; try { synchronized (csvWriter) { + fileLock = csvWriter.getlock(); if (genDynamicPart) { csvWriter.write(item.eventDate);// eventDate--------------->??? csvWriter.write(commonInfo.pid);// pid @@ -164,6 +172,7 @@ public class ResumeUtil { csvWriter.write(item.dynamicData);// dynamicData--->it is the 17th field. @see:feature:11296 csvWriter.endRecord(); csvWriter.flush(); + fileLock.release(); } // for test the order // System.out.print(item.partName + ",");// partName @@ -173,6 +182,15 @@ public class ResumeUtil { // System.out.println(); } catch (Exception e) { e.printStackTrace(); + }finally { + if (fileLock != null) { + try { + fileLock.release(); + fileLock = null; + } catch (IOException e) { + e.printStackTrace(); + } + } } } @@ -457,7 +475,11 @@ public class ResumeUtil { */ public class SimpleCsvWriter { - private Writer writer = null; + private FileChannel channel = null; + + private ByteBuffer buf = null; + + private boolean firstColumn = true; @@ -467,11 +489,11 @@ public class ResumeUtil { private static final int EscapeMode = ESCAPE_MODE_DOUBLED; - private static final char TextQualifier = '"'; + private static final String TextQualifier = "\""; - private static final char BACKSLASH = '\\'; + private static final String BACKSLASH = "\\"; - private static final char Delimiter = ','; + private static final String Delimiter = ","; // JDK1.5 can't pass compile // private String lineSeparator = (String) @@ -481,8 +503,9 @@ public class ResumeUtil { private String lineSeparator = System.getProperty("line.separator"); - public SimpleCsvWriter(Writer writer) { - this.writer = writer; + public SimpleCsvWriter(FileChannel channel) { + this.channel = channel; + buf = ByteBuffer.allocate(2<<14);//32k buffer size } /** @@ -495,10 +518,10 @@ public class ResumeUtil { } if (!firstColumn) { - writer.write(Delimiter); + buf.put(Delimiter.getBytes()); } - writer.write(TextQualifier); + buf.put(TextQualifier.getBytes()); // support backslash mode if (EscapeMode == ESCAPE_MODE_BACKSLASH) { @@ -508,18 +531,22 @@ public class ResumeUtil { content = replace(content, "" + TextQualifier, "" + TextQualifier + TextQualifier); } - writer.write(content); + buf.put(content.getBytes()); - writer.write(TextQualifier); + buf.put(TextQualifier.getBytes()); firstColumn = false; } + public FileLock getlock() throws IOException { + return channel.lock(); + } + /** * finish a record, prepare the next one */ public void endRecord() throws IOException { - writer.write(lineSeparator); + buf.put(lineSeparator.getBytes()); firstColumn = true; } @@ -528,7 +555,13 @@ public class ResumeUtil { */ public void flush() { try { - writer.flush(); + buf.flip(); + channel.position(channel.size()); + while(buf.hasRemaining()) { + channel.write(buf); + } + channel.force(true); + buf.clear(); } catch (IOException e) { e.printStackTrace(); } @@ -539,7 +572,7 @@ public class ResumeUtil { */ public void close() { try { - writer.close(); + channel.close(); } catch (IOException e) { e.printStackTrace(); } diff --git a/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java b/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java index 62f8b92b59..37345593eb 100644 --- a/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java +++ b/main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java @@ -267,35 +267,20 @@ public class MigrationToolService implements IMigrationToolService { nbMigrationsToDo++; } - - int migrationInEveryLogon = 0; - if (beforeLogon) { - // UpdateDaikonCryptoUtilsMigrationTask to execute every logon - MigrationUtil.removeMigrationTaskById(done, - "org.talend.repository.model.migration.UpdateDaikonCryptoUtilsMigrationTask"); - migrationInEveryLogon++; - } - - if (nbMigrationsToDo == 0 && migrationInEveryLogon == 0) { + if (nbMigrationsToDo == 0) { return; } - if (nbMigrationsToDo > 0) { - // only execute when migration exist - if (beforeLogon) { - // for migration exist only, force reset to default maven template - MigrationUtil.removeMigrationTaskById(done, - "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask"); - } else { - // force execute migration in case user copy-past items with diffrent path on the file system and - // refresh - // the studio,it may cause bug TDI-19229 - MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink"); - // force to re-generate all job poms - MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask"); - } - } + // force execute migration in case user copy-past items with diffrent path on the file system and refresh + // the studio,it may cause bug TDI-19229 + MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink"); + // force to re-generate all job poms + MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask"); + if (beforeLogon) { + // for every migration, force reset to default maven template + MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask"); + } boolean haveAnyBinFolder = false; // to avoid some problems of migration, sometimes for (ERepositoryObjectType type : (ERepositoryObjectType[]) ERepositoryObjectType.values()) {