jprdonnelly/tcommon-studio-se
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/maintenance/7.3' into wwang-talend/TDI-46185

Browse Source
This commit is contained in:
wwang-talend
2021-08-13 16:50:31 +08:00
parent 6c4e5c80f4 baf574ea55
commit 6a95abfa47
12 changed files with 246 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java
View File

@@ -2283,6 +2283,9 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
IRunProcessService runProcessService = getRunProcessService();
if (runProcessService != null) {
runProcessService.initMavenJavaProject(monitor, project);
// before afterLogon migration execute, check and update daikon dependency
runProcessService.checkAndUpdateDaikonDependencies();
}
currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);

40
main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java
View File

@@ -728,6 +728,46 @@ public final class ProcessUtils {
}
return null;
}
private static boolean isRoute(Property property) {
return property!= null && (ERepositoryObjectType.getType(property).equals(ERepositoryObjectType.PROCESS_ROUTE) ||
ERepositoryObjectType.getType(property).equals(ERepositoryObjectType.PROCESS_ROUTE_MICROSERVICE));
}
public static boolean isRouteWithChildJobs(IProcess process) {
if (process instanceof IProcess2) {
Property p = ((IProcess2) process).getProperty();
if (isRoute(p)) {
return false;
}
Item item = p.getItem();
return isRouteWithChildJobs(item);
} else {
for (INode node : process.getGraphicalNodes()) {
if (node.getComponent().getName().equals("cTalendJob")) {
return true;
}
}
}
return false;
}
public static boolean isRouteWithChildJobs(Item item) {
if (item!= null && item instanceof ProcessItem) {
for (Object obj : ((ProcessItem) item).getProcess().getNode()) {
if (obj instanceof NodeType) {
if (((NodeType) obj).getComponentName().equals("cTalendJob")) {
return true;
}
}
}
}
return false;
}
public static int getAssertAmount(IProcess process) {
int count = 0;

2
main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java
View File

@@ -275,4 +275,6 @@ public interface IRunProcessService extends IService {
public void deleteOldVersionTalendJobProject(IRepositoryViewObject object);
public void checkAndUpdateDaikonDependencies();
}

33
main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java
View File

@@ -198,6 +198,8 @@ public class ProcessorUtilities {
private static boolean isDynamicJobAndCITest = false;
private static JobInfo mainJobInfo;
private static boolean needExportItemsForDQ = false;
public static void addOpenEditor(IEditorPart editor) {
openedEditors.add(editor);
@@ -699,7 +701,7 @@ public class ProcessorUtilities {
processor.setArguments(argumentsMap);
copyDQDroolsToSrc(selectedProcessItem);
handelDQComponents(selectedProcessItem, currentProcess);
// generate the code of the father after the childrens
// so the code won't have any error during the check, and it will help to check
// if the generation is really needed.
@@ -1243,7 +1245,7 @@ public class ProcessorUtilities {
processor.setArguments(argumentsMap);
copyDQDroolsToSrc(selectedProcessItem);
handelDQComponents(selectedProcessItem, currentProcess);
generateContextInfo(jobInfo, selectedContextName, statistics, trace, needContext, progressMonitor,
currentProcess, currentJobName, processor, isMainJob, codeGenerationNeeded);
@@ -1380,12 +1382,13 @@ public class ProcessorUtilities {
}
/**
*
* copy the current item's drools file from 'workspace/metadata/survivorship' to '.Java/src/resources'
*
*
* Specail operation for DQ components:
* - For tdqReportRun, set 'needExportItemsForDQ'to true so as the item folder can be exported
* - For tRuleSurvivorship, copy the current item's drools file from 'workspace/metadata/survivorship' to '.Java/src/resources'
* @param processItem
*/
private static void copyDQDroolsToSrc(ProcessItem processItem) {
private static void handelDQComponents(ProcessItem processItem,IProcess currentProcess) {
// 1.TDQ-12474 copy the "metadata/survivorship/rulePackage" to ".Java/src/main/resources/". so that it will be
// used by
// maven command 'include-survivorship-rules' to export.
@@ -1397,6 +1400,20 @@ public class ProcessorUtilities {
return;
}
try {
// TDQ-19637 if it includes 'tDqReportRun',must export item folder
if (!needExportItemsForDQ) {
for (INode node : currentProcess.getGeneratingNodes()) {
String componentName = node.getComponent().getName();
if ("tDqReportRun".equals(componentName)) {
needExportItemsForDQ = true;
break;
}
}
}
/* 1.TDQ-12474 copy the "metadata/survivorship/rulePackage" to ".Java/src/main/resources/". so that it will be
used by maven command 'include-survivorship-rules' to export.
2.TDQ-14308 current drools file in 'src/resourcesmetadata/survivorship/' should be included to job jar.
*/
ExportFileResource resouece = new ExportFileResource();
BuildExportManager.getInstance().exportDependencies(resouece, processItem);
if (resouece.getAllResources().isEmpty()) {
@@ -2920,4 +2937,8 @@ public class ProcessorUtilities {
public static boolean isDynamicJobAndCITest() {
return isDynamicJobAndCITest;
}
public static boolean isNeedExportItemsForDQ() {
return needExportItemsForDQ;
}
}

5
main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml
View File

@@ -35,6 +35,11 @@
<artifactId>commons-io</artifactId>
<version>2.8.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
<build>
<plugins>

5
main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-7/pom.xml
View File

@@ -85,6 +85,11 @@
<artifactId>plexus-archiver</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
<build>
<plugins>

9
main/plugins/org.talend.designer.maven.tos/resources/build/plugins/talend-compiler-plugin/pom.xml
View File

@@ -10,6 +10,11 @@
<artifactId>talend-compiler-plugin-tos</artifactId>
<packaging>pom</packaging>
<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
<dependency>
<groupId>org.eclipse.tycho</groupId>
<artifactId>tycho-compiler-jdt</artifactId>
@@ -23,6 +28,10 @@
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-archiver</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>

25
main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml
View File

@@ -30,6 +30,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
@@ -39,6 +44,10 @@
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-shared-utils</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@@ -50,6 +59,10 @@
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-shared-utils</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@@ -61,6 +74,10 @@
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-shared-utils</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@@ -75,11 +92,15 @@
<exclusion>
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-shared-utils</artifactId>
</exclusion>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
</exclusion>
</exclusion>
</exclusions>
</dependency>
<dependency>

25
main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml
View File

@@ -66,6 +66,11 @@
<artifactId>plexus-archiver</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
</plugin>
<plugin>
@@ -83,6 +88,11 @@
<artifactId>maven-shared-utils</artifactId>
<version>3.3.3</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
</plugin>
<plugin>
@@ -115,6 +125,11 @@
<artifactId>plexus-io</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
<configuration>
<archive>
@@ -152,6 +167,11 @@
<artifactId>plexus-archiver</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
</plugin>
<plugin>
@@ -164,6 +184,11 @@
<artifactId>plexus-utils</artifactId>
<version>3.0.24</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
</plugin>
<plugin>

44
main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java
View File

@@ -15,6 +15,7 @@ package org.talend.designer.maven.tools;
import static org.talend.designer.maven.model.TalendJavaProjectConstants.*;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -28,7 +29,9 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.maven.artifact.versioning.DefaultArtifactVersion;
import org.apache.maven.model.Activation;
import org.apache.maven.model.Dependency;
import org.apache.maven.model.Model;
import org.apache.maven.model.Profile;
import org.eclipse.core.resources.IContainer;
@@ -1053,4 +1056,45 @@ public class AggregatorPomsHelper {
return null;
}
public void checkAndUpdateDaikonDependencies() {
try {
IFile rootPomFile = getProjectRootPom();
if (rootPomFile.exists()) {
boolean isRegeneratePoms = false;
Model model = MavenPlugin.getMaven().readModel(rootPomFile.getLocation().toFile());
if (model.getModules() != null) {
for (String module : model.getModules()) {
java.nio.file.Path modPath = Paths.get(rootPomFile.getLocation().toFile().getParent(), module, "pom.xml");
Model modModel = MavenPlugin.getMaven().readModel(modPath.toFile());
List<Dependency> deps = modModel.getDependencies();
if (deps != null && !deps.isEmpty()) {
DefaultArtifactVersion targetVersion = new DefaultArtifactVersion("0.31.12");
for (Dependency dep : deps) {
DefaultArtifactVersion actualVersion = new DefaultArtifactVersion(dep.getVersion());
int cmp = targetVersion.compareTo(actualVersion);
if (StringUtils.equals("org.talend.daikon", dep.getGroupId()) && cmp > 0) {
if (StringUtils.equals("crypto-utils", dep.getArtifactId())
|| StringUtils.equals("daikon", dep.getArtifactId())) {
isRegeneratePoms = true;
}
}
}
}
if (isRegeneratePoms) {
break;
}
}
}
if (isRegeneratePoms) {
syncAllPomsWithoutProgress(new NullProgressMonitor());
BuildCacheManager.getInstance().clearAllCodesCache();
}
}
} catch (Exception e) {
ExceptionHandler.process(e);
}
}
}

73
main/plugins/org.talend.librariesmanager/resources/java/routines/system/ResumeUtil.java
View File

@@ -13,10 +13,13 @@
package routines.system;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -53,7 +56,7 @@ public class ResumeUtil {
private static Object lock = new Object();
// step1: init the log file name
public ResumeUtil(String logFileName, boolean createNewFile, String root_pid) {
public ResumeUtil(String logFileName, boolean append, String root_pid) {
if (logFileName == null || logFileName.equals("null")) {
return;
}
@@ -70,7 +73,13 @@ public class ResumeUtil {
File file = new File(logFileName);
try {
if (sharedWriter == null) {
this.csvWriter = new SimpleCsvWriter(new FileWriter(logFileName, createNewFile));
FileChannel fc = new RandomAccessFile(logFileName, "rw").getChannel();
if(append){
fc.position(fc.size());
}else {
fc.truncate(0);
}
this.csvWriter = new SimpleCsvWriter(fc);
// output the header part
if (file.length() == 0) {
@@ -97,9 +106,6 @@ public class ResumeUtil {
csvWriter.write("dynamicData");// dynamicData
csvWriter.endRecord();
csvWriter.flush();
csvWriter.close();
// To avoid use File.delete() as it cannot make sure file being deleted.
this.csvWriter = new SimpleCsvWriter(new FileWriter(logFileName, true));
}
// shared
sharedWriterMap.put(this.root_pid, this.csvWriter);
@@ -138,8 +144,10 @@ public class ResumeUtil {
JobLogItem item = new JobLogItem(eventDate, type, partName, parentPart, threadId, logPriority, errorCode, message,
stackTrace, dynamicData);
FileLock fileLock = null;
try {
synchronized (csvWriter) {
fileLock = csvWriter.getlock();
if (genDynamicPart) {
csvWriter.write(item.eventDate);// eventDate--------------->???
csvWriter.write(commonInfo.pid);// pid
@@ -164,6 +172,7 @@ public class ResumeUtil {
csvWriter.write(item.dynamicData);// dynamicData--->it is the 17th field. @see:feature:11296
csvWriter.endRecord();
csvWriter.flush();
fileLock.release();
}
// for test the order
// System.out.print(item.partName + ",");// partName
@@ -173,6 +182,15 @@ public class ResumeUtil {
// System.out.println();
} catch (Exception e) {
e.printStackTrace();
}finally {
if (fileLock != null) {
try {
fileLock.release();
fileLock = null;
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
@@ -457,7 +475,11 @@ public class ResumeUtil {
*/
public class SimpleCsvWriter {
private Writer writer = null;
private FileChannel channel = null;
private ByteBuffer buf = null;
private boolean firstColumn = true;
@@ -467,11 +489,11 @@ public class ResumeUtil {
private static final int EscapeMode = ESCAPE_MODE_DOUBLED;
private static final char TextQualifier = '"';
private static final String TextQualifier = "\"";
private static final char BACKSLASH = '\\';
private static final String BACKSLASH = "\\";
private static final char Delimiter = ',';
private static final String Delimiter = ",";
// JDK1.5 can't pass compile
// private String lineSeparator = (String)
@@ -481,8 +503,9 @@ public class ResumeUtil {
private String lineSeparator = System.getProperty("line.separator");
public SimpleCsvWriter(Writer writer) {
this.writer = writer;
public SimpleCsvWriter(FileChannel channel) {
this.channel = channel;
buf = ByteBuffer.allocate(2<<14);//32k buffer size
}
/**
@@ -495,10 +518,10 @@ public class ResumeUtil {
}
if (!firstColumn) {
writer.write(Delimiter);
buf.put(Delimiter.getBytes());
}
writer.write(TextQualifier);
buf.put(TextQualifier.getBytes());
// support backslash mode
if (EscapeMode == ESCAPE_MODE_BACKSLASH) {
@@ -508,18 +531,22 @@ public class ResumeUtil {
content = replace(content, "" + TextQualifier, "" + TextQualifier + TextQualifier);
}
writer.write(content);
buf.put(content.getBytes());
writer.write(TextQualifier);
buf.put(TextQualifier.getBytes());
firstColumn = false;
}
public FileLock getlock() throws IOException {
return channel.lock();
}
/**
* finish a record, prepare the next one
*/
public void endRecord() throws IOException {
writer.write(lineSeparator);
buf.put(lineSeparator.getBytes());
firstColumn = true;
}
@@ -528,7 +555,13 @@ public class ResumeUtil {
*/
public void flush() {
try {
writer.flush();
buf.flip();
channel.position(channel.size());
while(buf.hasRemaining()) {
channel.write(buf);
}
channel.force(true);
buf.clear();
} catch (IOException e) {
e.printStackTrace();
}
@@ -539,7 +572,7 @@ public class ResumeUtil {
*/
public void close() {
try {
writer.close();
channel.close();
} catch (IOException e) {
e.printStackTrace();
}

35
main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java
View File

@@ -267,35 +267,20 @@ public class MigrationToolService implements IMigrationToolService {
nbMigrationsToDo++;
}
int migrationInEveryLogon = 0;
if (beforeLogon) {
// UpdateDaikonCryptoUtilsMigrationTask to execute every logon
MigrationUtil.removeMigrationTaskById(done,
"org.talend.repository.model.migration.UpdateDaikonCryptoUtilsMigrationTask");
migrationInEveryLogon++;
}
if (nbMigrationsToDo == 0 && migrationInEveryLogon == 0) {
if (nbMigrationsToDo == 0) {
return;
}
if (nbMigrationsToDo > 0) {
// only execute when migration exist
if (beforeLogon) {
// for migration exist only, force reset to default maven template
MigrationUtil.removeMigrationTaskById(done,
"org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
} else {
// force execute migration in case user copy-past items with diffrent path on the file system and
// refresh
// the studio,it may cause bug TDI-19229
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
// force to re-generate all job poms
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
}
}
// force execute migration in case user copy-past items with diffrent path on the file system and refresh
// the studio,it may cause bug TDI-19229
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
// force to re-generate all job poms
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
if (beforeLogon) {
// for every migration, force reset to default maven template
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
}
boolean haveAnyBinFolder = false; // to avoid some problems of migration, sometimes
for (ERepositoryObjectType type : (ERepositoryObjectType[]) ERepositoryObjectType.values()) {