Compare commits

..

5 Commits

Author SHA1 Message Date
SunChaoqun
08505284c7 Merge branch 'master' into feature/master/TESB-28330 2020-11-12 15:37:35 +08:00
SunChaoqun
ba3708fa97 TESB-28330:cConfig/Beans - Support customer groupid and artifact name
(additional to custom version)
2020-11-03 11:16:45 +08:00
SunChaoqun
9e3fa73f2f Merge branch 'master' into feature/master/TESB-28330 2020-11-02 10:35:30 +08:00
SunChaoqun
dbee8af3e6 TESB-28330:cConfig/Beans - Support customer groupid and artifact name
(additional to custom version)
2020-10-30 18:47:35 +08:00
bhe-talendbj
b17cc125a3 feat(TESB-28330): remove workaround for cConfig 2020-10-23 17:58:21 +08:00
16 changed files with 90 additions and 843 deletions

View File

@@ -42,7 +42,6 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -51,16 +50,11 @@ import org.eclipse.core.resources.IContainer;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IFolder;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspace;
import org.eclipse.core.resources.IWorkspaceRunnable;
import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.OperationCanceledException;
import org.eclipse.core.runtime.jobs.ISchedulingRule;
import org.eclipse.core.runtime.jobs.Job;
import org.osgi.framework.Bundle;
import org.talend.commons.exception.CommonExceptionHandler;
import org.talend.commons.exception.ExceptionHandler;
@@ -1179,33 +1173,4 @@ public class FilesUtils {
}
}
public static void executeFolderAction(IProgressMonitor monitor, IResource parentFolder, IWorkspaceRunnable run)
throws CoreException {
if (Job.getJobManager().currentRule() == null) {
IWorkspace workspace = ResourcesPlugin.getWorkspace();
ISchedulingRule defaultRule = workspace.getRuleFactory().modifyRule(parentFolder);
ISchedulingRule noBlockRule = new ISchedulingRule() {
@Override
public boolean isConflicting(ISchedulingRule rule) {
return this.contains(rule);
}
@Override
public boolean contains(ISchedulingRule rule) {
if (this.equals(rule)) {
return true;
}
if (defaultRule.contains(rule)) {
return true;
}
return false;
}
};
workspace.run(run, noBlockRule, IWorkspace.AVOID_UPDATE, monitor);
} else {
run.run(monitor);
}
}
}

View File

@@ -575,7 +575,7 @@ public class ContextUtils {
return false;
}
// need check the raw value, because in sourceParam, it's raw
if (!StringUtils.equals(sourceParam.getValue(), targetParamType.getRawValue())) {
if (!sourceParam.getValue().equals(targetParamType.getRawValue())) {
return false;
}
@@ -1112,4 +1112,3 @@ public class ContextUtils {
return list;
}
}

View File

@@ -20,7 +20,6 @@ import java.util.Set;
import org.eclipse.core.runtime.IProgressMonitor;
import org.talend.commons.exception.BusinessException;
import org.talend.core.GlobalServiceRegister;
import org.talend.core.IService;
import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
import org.talend.core.model.process.IElement;
@@ -134,14 +133,4 @@ public interface ILibrariesService extends IService {
public void afterChangingLibraries();
}
public static ILibrariesService get() {
GlobalServiceRegister gsr = GlobalServiceRegister.getDefault();
if (gsr != null) {
if (gsr.isServiceRegistered(ILibrariesService.class)) {
return gsr.getService(ILibrariesService.class);
}
}
return null;
}
}

View File

@@ -1003,9 +1003,7 @@ public final class ProcessUtils {
}
return false;
}
public static boolean isChildRouteProcess(IProcess process) {
List n = process.getNodesOfType("tRouteInput");
if (n!=null && n.size()!=0) {

View File

@@ -320,28 +320,24 @@ public class RepositoryUpdateManagerHelper {
// all the jobs
for (IRepositoryViewObject process : processRep) {
try {
Item item = process.getProperty().getItem();
boolean found = false;
for (IProcess2 open : openedProcessList) {
if (open.getId().equals(item.getProperty().getId())) {
found = true;
}
Item item = process.getProperty().getItem();
boolean found = false;
for (IProcess2 open : openedProcessList) {
if (open.getId().equals(item.getProperty().getId())) {
found = true;
}
if (found) {
continue;
}
checkMonitorCanceled(parentMonitor);
parentMonitor.subTask(RepositoryUpdateManager.getUpdateJobInfor(item.getProperty()));
}
if (found) {
continue;
}
checkMonitorCanceled(parentMonitor);
parentMonitor.subTask(RepositoryUpdateManager.getUpdateJobInfor(item.getProperty()));
// List<UpdateResult> resultFromProcess = getResultFromProcess(process, types, onlySimpleShow);
// List<UpdateResult> resultFromProcess = getResultFromProcess(process, types, onlySimpleShow);
List<UpdateResult> resultFromProcess = getUpdatesNeededFromItems(parentMonitor, item, types);
if (resultFromProcess != null) {
resultList.addAll(resultFromProcess);
}
} catch (Exception ex) {
ExceptionHandler.process(ex);
List<UpdateResult> resultFromProcess = getUpdatesNeededFromItems(parentMonitor, item, types);
if (resultFromProcess != null) {
resultList.addAll(resultFromProcess);
}
parentMonitor.worked(1);
}

View File

@@ -47,8 +47,6 @@ public class RepositoryConstants {
public static final String JOBLET_DOCUMENTATION_PATH = "documentations/generated/joblets"; //$NON-NLS-1$
public static final String PROJECT_SETTINGS_CUSTOM_URI_MAP = "custom_uri_mapping.json"; //$NON-NLS-1$
public static final String SYSTEM_DIRECTORY = "system"; //$NON-NLS-1$
public static final String USER_DEFINED = "UserDefined"; //$NON-NLS-1$

View File

@@ -45,7 +45,6 @@ import org.talend.core.model.utils.JavaResourcesHelper;
import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.core.runtime.maven.MavenConstants;
import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.core.runtime.process.LastGenerationInfo;
import org.talend.core.runtime.process.TalendProcessArgumentConstant;
import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
import org.talend.core.runtime.repository.build.IMavenPomCreator;
@@ -53,7 +52,6 @@ import org.talend.core.ui.ITestContainerProviderService;
import org.talend.designer.maven.model.TalendMavenConstants;
import org.talend.designer.maven.template.ETalendMavenVariables;
import org.talend.designer.maven.tools.ProcessorDependenciesManager;
import org.talend.designer.maven.utils.JobUtils;
import org.talend.designer.maven.utils.PomIdsHelper;
import org.talend.designer.maven.utils.PomUtil;
import org.talend.designer.runprocess.IBigDataProcessor;
@@ -136,16 +134,9 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
Map<ETalendMavenVariables, String> variablesValuesMap = new HashMap<ETalendMavenVariables, String>();
// no need check property is null or not, because if null, will get default ids.
if (JobUtils.isJob(property) && ProcessUtils.isChildRouteProcess(process)) {
JobInfo lastMainJob = LastGenerationInfo.getInstance().getLastMainJob();
variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(lastMainJob.getProcessor().getProperty()));
variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(lastMainJob.getProcessor().getProperty()));
}else {
variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(property));
variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(property));
}
variablesValuesMap.put(ETalendMavenVariables.JobArtifactId, PomIdsHelper.getJobArtifactId(property));
variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(property));
variablesValuesMap.put(ETalendMavenVariables.JobArtifactId, PomIdsHelper.getJobArtifactId(property));
variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(property));
variablesValuesMap.put(ETalendMavenVariables.TalendJobVersion, property.getVersion());
final String jobName = JavaResourcesHelper.escapeFileName(process.getName());
variablesValuesMap.put(ETalendMavenVariables.JobName, jobName);

View File

@@ -19,7 +19,6 @@ import java.util.Set;
import org.talend.commons.exception.PersistenceException;
import org.talend.core.GlobalServiceRegister;
import org.talend.core.model.process.INode;
import org.talend.core.model.process.IProcess;
import org.talend.core.model.process.JobInfo;
import org.talend.core.model.process.ProcessUtils;
@@ -100,13 +99,6 @@ public class JobUtils {
}
return clonedJobInfos;
}
public static boolean isRoute(Property p) {
if (p != null) {
return ERepositoryObjectType.getType(p).equals(ERepositoryObjectType.PROCESS_ROUTE);
}
return false;
}
public static boolean isJob(JobInfo job) {
if (job != null && job.getProcessItem() != null) {

View File

@@ -64,6 +64,7 @@ import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.jobs.ISchedulingRule;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.core.runtime.jobs.MultiRule;
import org.eclipse.m2e.core.MavenPlugin;
import org.eclipse.m2e.core.embedder.MavenModelManager;
@@ -72,7 +73,6 @@ import org.talend.commons.exception.PersistenceException;
import org.talend.commons.runtime.utils.io.IOUtils;
import org.talend.commons.utils.VersionUtils;
import org.talend.commons.utils.generation.JavaUtils;
import org.talend.commons.utils.io.FilesUtils;
import org.talend.commons.utils.workbench.resources.ResourceUtils;
import org.talend.core.GlobalServiceRegister;
import org.talend.core.ILibraryManagerService;
@@ -583,7 +583,7 @@ public class PomUtil {
IProject fsProject = ResourceUtils.getProject(project);
IFolder tmpFolder = fsProject.getFolder("temp");
if (!tmpFolder.exists()) {
FilesUtils.executeFolderAction(monitor, fsProject, new IWorkspaceRunnable() {
executeFolderAction(monitor, fsProject, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
@@ -595,7 +595,7 @@ public class PomUtil {
createTempFile.delete();
String tmpFolderName = createTempFile.getName();
IFolder folder = tmpFolder.getFolder(tmpFolderName);
FilesUtils.executeFolderAction(monitor, tmpFolder, new IWorkspaceRunnable() {
executeFolderAction(monitor, tmpFolder, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
@@ -604,7 +604,7 @@ public class PomUtil {
});
IFile pomFile = folder.getFile(TalendMavenConstants.POM_FILE_NAME);
FilesUtils.executeFolderAction(monitor, folder, new IWorkspaceRunnable() {
executeFolderAction(monitor, folder, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
@@ -622,6 +622,35 @@ public class PomUtil {
return null;
}
private static void executeFolderAction(IProgressMonitor monitor, IResource parentFolder, IWorkspaceRunnable run)
throws CoreException {
if (Job.getJobManager().currentRule() == null) {
IWorkspace workspace = ResourcesPlugin.getWorkspace();
ISchedulingRule defaultRule = workspace.getRuleFactory().modifyRule(parentFolder);
ISchedulingRule noBlockRule = new ISchedulingRule() {
@Override
public boolean isConflicting(ISchedulingRule rule) {
return this.contains(rule);
}
@Override
public boolean contains(ISchedulingRule rule) {
if (this.equals(rule)) {
return true;
}
if (defaultRule.contains(rule)) {
return true;
}
return false;
}
};
workspace.run(run, noBlockRule, IWorkspace.AVOID_UPDATE, monitor);
} else {
run.run(monitor);
}
}
private static Model createModel(MavenArtifact artifact) {
Model pomModel = new Model();
pomModel.setModelVersion(TalendMavenConstants.POM_VERSION);

View File

@@ -13,38 +13,25 @@
package org.talend.librariesmanager.model.service;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Set;
import net.sf.json.JSONObject;
import org.codehaus.jackson.map.ObjectMapper;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IFolder;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspaceRunnable;
import org.eclipse.core.runtime.Assert;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.talend.commons.exception.ExceptionHandler;
import org.talend.commons.exception.LoginException;
import org.talend.commons.exception.PersistenceException;
import org.talend.commons.utils.io.FilesUtils;
import org.talend.commons.utils.workbench.resources.ResourceUtils;
import org.talend.core.model.general.Project;
import org.talend.core.runtime.CoreRuntimePlugin;
import org.talend.repository.ProjectManager;
import org.talend.repository.RepositoryWorkUnit;
import org.talend.repository.model.IProxyRepositoryFactory;
import org.talend.repository.model.RepositoryConstants;
import net.sf.json.JSONObject;
/**
* created by wchen on Aug 18, 2017 Detailled comment
@@ -56,30 +43,17 @@ public class CustomUriManager {
private static CustomUriManager manager = new CustomUriManager();;
private static final String CUSTOM_URI_MAP = "custom_uri_mapping.json";
private static long lastModified = 0;
private static boolean isNeedReload = false;
private final Object reloadingLock = new Object();
private CustomUriManager() {
IFolder resourcePath = getResourcePath();
try {
Assert.isNotNull(resourcePath);
customURIObject = loadResources(resourcePath, RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP);
customURIObject = loadResources(getResourcePath(), CUSTOM_URI_MAP);
} catch (Exception e) {
ExceptionHandler.process(e);
try {
if (resourcePath != null) {
customURIObject = loadResources(resourcePath.getLocation().toPortableString(),
RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP);
}
} catch (Exception e1) {
ExceptionHandler.process(e1);
/**
* Seems dangrous to load an empty json here, because it may overwrite the existing one
*/
// customURIObject = new JSONObject();
}
}
}
@@ -87,121 +61,34 @@ public class CustomUriManager {
return manager;
}
private JSONObject loadResources(String path, String fileName) throws IOException {
JSONObject jsonObj = new JSONObject();
InputStream in = null;
try {
File file = new File(path, fileName);
if (file.exists()) {
jsonObj = loadResources(new FileInputStream(file), fileName);
}
} finally {
if (in != null) {
try {
in.close();
} catch (Exception e) {
}
}
}
return jsonObj;
}
private JSONObject loadResources(IFolder path, String fileName) throws CoreException, IOException {
JSONObject jsonObj = new JSONObject();
InputStream in = null;
try {
IFile file = path.getFile(fileName);
FilesUtils.executeFolderAction(new NullProgressMonitor(), path, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
file.refreshLocal(IResource.DEPTH_ZERO, monitor);
}
});
if (file.isAccessible()) {
in = file.getContents(true);
return loadResources(in, fileName);
}
} finally {
try {
if (in != null) {
in.close();
}
} catch (Exception e) {
}
}
return jsonObj;
}
private JSONObject loadResources(InputStream in, String fileName) throws IOException {
private synchronized JSONObject loadResources(String path, String fileName) throws IOException {
BufferedReader br = null;
JSONObject jsonObj = new JSONObject();
try {
br = new BufferedReader(new InputStreamReader(in));
StringBuffer buffer = new StringBuffer();
String line = null;
while ((line = br.readLine()) != null) {
buffer.append(line);
File file = new File(path, fileName);
if (file.exists()) {
br = new BufferedReader(new FileReader(file));
StringBuffer buffer = new StringBuffer();
String line = null;
while ((line = br.readLine()) != null) {
buffer.append(line);
}
jsonObj = JSONObject.fromObject(buffer.toString());
}
jsonObj = JSONObject.fromObject(buffer.toString());
} finally {
if (br != null) {
try {
br.close();
} catch (Exception e) {
}
br.close();
}
}
return jsonObj;
}
private void saveResource(JSONObject customMap, IFolder filePath, String fileName, boolean isExport) {
ByteArrayOutputStream out = null;
ByteArrayInputStream in = null;
try {
IFile file = filePath.getFile(fileName);
out = new ByteArrayOutputStream();
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(out, customMap);
in = new ByteArrayInputStream(out.toByteArray());
final InputStream fin = in;
FilesUtils.executeFolderAction(new NullProgressMonitor(), file.getParent(), new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
file.refreshLocal(IResource.DEPTH_ZERO, monitor);
if (!file.exists()) {
file.create(fin, false, monitor);
} else {
file.setContents(fin, true, false, monitor);
}
}
});
} catch (Exception e) {
ExceptionHandler.process(e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException ex) {
// ExceptionHandler.process(ex);
}
}
if (in != null) {
try {
in.close();
} catch (IOException ex) {
// ExceptionHandler.process(ex);
}
}
}
}
private void saveResource(JSONObject customMap, String filePath, String fileName, boolean isExport) {
try {
File file = new File(filePath, fileName);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(file, customMap);
lastModified = file.lastModified();
} catch (IOException e) {
ExceptionHandler.process(e);
}
@@ -213,7 +100,7 @@ public class CustomUriManager {
@Override
public void run() throws PersistenceException, LoginException {
saveResource(customURIObject, getResourcePath(), RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP, false);
saveResource(customURIObject, getResourcePath(), CUSTOM_URI_MAP, false);
}
};
IProxyRepositoryFactory factory = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory();
@@ -224,11 +111,12 @@ public class CustomUriManager {
}
private IFolder getResourcePath() {
private String getResourcePath() {
try {
Project currentProject = ProjectManager.getInstance().getCurrentProject();
IProject project = ResourceUtils.getProject(currentProject);
return project.getFolder(".settings");
IFolder settingsFolder = project.getFolder(".settings");
return settingsFolder.getLocation().toPortableString();
} catch (PersistenceException e) {
ExceptionHandler.process(e);
}
@@ -236,11 +124,7 @@ public class CustomUriManager {
}
public void put(String key, String value) {
try {
reloadCustomMapping();
} catch (Exception e) {
ExceptionHandler.process(e);
}
reloadCustomMapping();
if (value != null) {
customURIObject.put(key, value);
} else {
@@ -249,11 +133,7 @@ public class CustomUriManager {
}
public String get(String key) {
try {
reloadCustomMapping();
} catch (Exception e) {
ExceptionHandler.process(e);
}
reloadCustomMapping();
if (customURIObject.containsKey(key)) {
return customURIObject.getString(key);
}
@@ -279,22 +159,16 @@ public class CustomUriManager {
public void reloadCustomMapping() {
try {
if (isNeedReload || customURIObject == null) {
synchronized (reloadingLock) {
if (isNeedReload || customURIObject == null) {
if (customURIObject == null) {
customURIObject = new JSONObject();
} else {
customURIObject.clear();
}
JSONObject loadResources = loadResources(getResourcePath(),
RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP);
customURIObject.putAll(loadResources);
isNeedReload = false;
}
}
File file = new File(getResourcePath(), CUSTOM_URI_MAP);
long modifyDate = file.lastModified();
if (isNeedReload || modifyDate > lastModified) {
customURIObject.clear();
JSONObject loadResources = loadResources(getResourcePath(), CUSTOM_URI_MAP);
customURIObject.putAll(loadResources);
lastModified = modifyDate;
isNeedReload = false;
}
} catch (Exception e) {
} catch (IOException e) {
ExceptionHandler.process(e);
}
}

View File

@@ -8,78 +8,5 @@
<relativePath>../../../</relativePath>
</parent>
<artifactId>org.talend.librariesmanager.test</artifactId>
<packaging>eclipse-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
<type>zip</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
<type>dll</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
<type>exe</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
<type>zip</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
<type>dll</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
<type>exe</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>generate-sources</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeGroupIds>org.talend.studio.test</includeGroupIds>
<copyPom>true</copyPom>
<includeArtifactIdIds>nexus.upload.test.old,nexus.upload.test.new</includeArtifactIdIds>
<excludeTransitive>true</excludeTransitive>
<outputDirectory>${project.basedir}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
<packaging>eclipse-plugin</packaging>
</project>

View File

@@ -1,5 +0,0 @@
rep.create.endpoint=/service/rest/v1/repositories/maven/hosted
rep.list.endpoint=/service/rest/v1/repositories/maven/hosted/{repid}
component.search.endpoint=/service/rest/v1/search
component.delete.endpoint=/service/rest/v1/components/{id}
component.upload.command=curl -v -u {user}:{password} -X POST \"{server}/service/rest/v1/components?repository={repid}\" -F maven2.groupId={groupid} -F maven2.artifactId={artifactid} -F maven2.version={version} -F maven2.asset1=@{filepath} -F maven2.asset1.extension={fileext}

View File

@@ -1,18 +0,0 @@
{
"name": "NEWREPNAME",
"online": true,
"storage": {
"blobStoreName": "default",
"strictContentTypeValidation": false,
"writePolicy": "ALLOW"
},
"cleanup": {
"policyNames": [
"string"
]
},
"maven": {
"versionPolicy": "NEWVERSIONPOLICY",
"layoutPolicy": "STRICT"
}
}

View File

@@ -12,72 +12,17 @@
// ============================================================================
package org.talend.librariesmanager.nexus;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Dictionary;
import java.util.Hashtable;
import java.util.List;
import java.util.Properties;
import org.apache.commons.codec.digest.DigestUtils;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.Platform;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
import org.osgi.framework.InvalidSyntaxException;
import org.osgi.framework.ServiceReference;
import org.osgi.service.cm.Configuration;
import org.osgi.service.cm.ConfigurationAdmin;
import org.talend.core.nexus.ArtifactRepositoryBean;
import org.talend.core.nexus.IRepositoryArtifactHandler;
import org.talend.core.nexus.NexusServerUtils;
import org.talend.core.nexus.RepositoryArtifactHandlerManager;
import org.talend.core.nexus.TalendMavenResolver;
import org.talend.core.nexus.ArtifactRepositoryBean.NexusType;
import org.talend.core.runtime.CoreRuntimePlugin;
import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.librariesmanager.model.service.LocalLibraryManager;
import org.talend.librariesmanager.nexus.utils.NexusServerManagerProxy;
import org.talend.librariesmanager.nexus.utils.RestAPIUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
/**
* created by wchen on Aug 18, 2017 Detailled comment
*
*/
public class Nexus3RepositoryHandlerTest {
private static Properties nexusprops = new Properties();
private static ArtifactRepositoryBean customNexusServer;
private static IRepositoryArtifactHandler repHandler;
private static String[] types = new String[] {"jar", "pom", "exe", "zip", "dll"};
@BeforeClass
public static void init() throws FileNotFoundException, IOException {
URL entry = Platform.getBundle("org.talend.librariesmanager.test").getEntry("resources/nexus/nexus3.properties");
nexusprops.load(new FileInputStream(FileLocator.toFileURL(entry).getFile()));
customNexusServer = NexusServerManagerProxy.getInstance().getCustomNexusServer();
repHandler = RepositoryArtifactHandlerManager.getRepositoryHandler(customNexusServer);
createNexusRepository(customNexusServer.getRepositoryId(),"RELEASE");
createNexusRepository(customNexusServer.getSnapshotRepId(),"SNAPSHOT");
}
@Test
public void testGetRepositoryURL() {
ArtifactRepositoryBean serverBean = new ArtifactRepositoryBean();
@@ -92,329 +37,5 @@ public class Nexus3RepositoryHandlerTest {
String snapshotUrl = handler.getRepositoryURL(false);
Assert.assertEquals("http://localhost:8081/repository/snapshot-repository/", snapshotUrl);
}
@Test
public void testCheckConnectionOK() {
Assert.assertTrue(repHandler.checkConnection());
}
@Test
public void testCheckConnectionFalse() {
ArtifactRepositoryBean dummyNexusServer = NexusServerManagerProxy.getInstance().getDummyNexusServer();
IRepositoryArtifactHandler repHandler = RepositoryArtifactHandlerManager.getRepositoryHandler(dummyNexusServer);
Assert.assertFalse(repHandler.checkConnection());
}
@Test
public void testDeployOnSnapshot() throws Exception {
for ( String type : types ) {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/" + type;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
clearLocalFile(uri);
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File originalJarFile = new File(FileLocator.toFileURL(entry).getFile());
String originalSHA1 = getSha1(originalJarFile);
// deploy the file to nexus
repHandler.deploy(originalJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type, artifact.getVersion());
File reolved = repHandler.resolve(artifact);
assertEquals(originalSHA1,getSha1(reolved));
} finally {
deleteNexusArtifact(artifact, false, true);
}
}
}
@Test
public void testDeployOnRelease() throws Exception {
for ( String type : types ) {
String uri = "mvn:org.talend.libraries/test/6.0.0/" + type ;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
clearLocalFile(uri);
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File originalJarFile = new File(FileLocator.toFileURL(entry).getFile());
String originalSHA1 = getSha1(originalJarFile);
// deploy file to nexus
repHandler.deploy(originalJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type , artifact.getVersion());
File resolved = repHandler.resolve(artifact);
assertEquals(originalSHA1,getSha1(resolved));
} finally {
deleteNexusArtifact(artifact, true, true);
}
}
}
@Test
public void testUpdateOnSnapshot() throws Exception {
for ( String type : types ) {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/" + type ;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
clearLocalFile(uri);
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File originalJarFile = new File(FileLocator.toFileURL(entry).getFile());
entry = bundle.getEntry("/lib/nexus.upload.test.new-1.0.0." + type);
File newJarFile = new File(FileLocator.toFileURL(entry).getFile());
String originalSHA1 = getSha1(originalJarFile);
String newJarSHA1 = getSha1(newJarFile);
// deploy original jar without resolving
repHandler.deploy(originalJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type, artifact.getVersion());
//deploy new jar
repHandler.deploy(newJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type, artifact.getVersion());
//resolve and check the local file
File resolved = repHandler.resolve(artifact);
assertEquals(newJarSHA1,getSha1(resolved));
} finally {
deleteNexusArtifact(artifact, false, true);
}
}
}
@Test
public void testResolveSha1NotExist() throws Exception {
String uri = "mvn:org.talend.libraries/not-existing/6.0.0-SNAPSHOT/jar";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
File f = repHandler.resolve(artifact);
assertNull(f);
} catch (FileNotFoundException fnfe) {
//It is one expected exception.
return;
}
}
@Test
public void testSearchOnRelease() throws Exception {
for (String type: types) {
String uri = "mvn:org.talend.libraries/test/6.0.0/" + type;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
//Step1:Prepare the installed artifact
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
final URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File fileToBeInstalled = new File(FileLocator.toFileURL(entry).getFile());
//Step2:Search empty
List<MavenArtifact> searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), true, false);
if ( searchRet != null && searchRet.size() > 0 ) {
Assert.fail("The artifact:" + uri + " is not expected to intalled before the test");
}
//Step3:Deploy the artifact
repHandler.deploy(fileToBeInstalled, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), artifact.getType(), artifact.getVersion());
//Step4:Verify the search result
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), true, false);
int i = 30;
while ( searchRet.size() < 1 && i > 0 ) {
Thread.sleep(1000);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), true, false);
i--;
}
if (NexusServerUtils.IGNORED_TYPES.contains(type)) {
assertEquals("Should get 0 artifact when searching for ignored type of " + uri , 0, searchRet.size());
} else {
assertEquals("Should get 1 artifact when searching for " + uri , 1, searchRet.size());
assertEquals(artifact.getArtifactId(), searchRet.get(0).getArtifactId());
assertEquals(artifact.getGroupId(), searchRet.get(0).getGroupId());
assertEquals(artifact.getVersion(), searchRet.get(0).getVersion());
assertEquals(artifact.getClassifier(), searchRet.get(0).getClassifier());
assertEquals(artifact.getType(), searchRet.get(0).getType());
}
} finally {
deleteNexusArtifact(artifact, true, true);
Thread.sleep(1000);
}
}
}
@Test
public void testSearchOnSnapshot() throws Exception {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/jar";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
//Step1:Prepare the installed artifact
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
final URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0.jar");
File fileToBeInstalled = new File(FileLocator.toFileURL(entry).getFile());
//Step2:Search empty
List<MavenArtifact> searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
if ( searchRet != null && searchRet.size() > 0 ) {
Assert.fail("The artifact is not expected to intalled before the test");
}
//Step3:Deploy the artifact
repHandler.deploy(fileToBeInstalled, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), "jar", artifact.getVersion());
//Step4:Verify the search result
//In snapshot repository, Timestamp is added in item version like 6.0.0-20201102.054042-1,
//the search result for given version like 6.0.0 will be empty. Currently the version will not be given in search criteria.
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
int i = 30;
while ( searchRet.size() < 1 && i > 0 ) {
Thread.sleep(1000);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
i--;
}
debug("testSearchSnapshot:" + searchRet);
assertEquals(1, searchRet.size());
assertEquals(artifact.getArtifactId(), searchRet.get(0).getArtifactId());
assertEquals(artifact.getGroupId(), searchRet.get(0).getGroupId());
//assertEquals(artifact.getVersion(), searchRet.get(0).getVersion());
assertEquals(artifact.getClassifier(), searchRet.get(0).getClassifier());
assertEquals(artifact.getType(), searchRet.get(0).getType());
} finally {
deleteNexusArtifact(artifact, false, true);
Thread.sleep(1000);
}
}
@Test
public void testPaginationSearchOnSnapshot() throws Exception {
//Pagination is enabled by Nexus when a search returns 50 results
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/zip";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
List<MavenArtifact> searchRet = new ArrayList<MavenArtifact>();
try {
//Step1:Prepare the installed artifact
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
final URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0.zip");
File fileToBeInstalled = new File(FileLocator.toFileURL(entry).getFile());
//Step2:Deploy the artifacts for 101 times on snapshot repository .
int i = 51;
while ( i> 0 ) {
repHandler.deploy(fileToBeInstalled, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), "zip", artifact.getVersion());
i--;
}
//Step3:Verify the search result. It will take some time for the deployment and wait for most 60 seconds
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
i = 6;
while ( searchRet.size() < 51 && i > 0 ) {
Thread.sleep(10000);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
i--;
}
assertEquals(51, searchRet.size());
} finally {
deleteNexusArtifacts(searchRet, false, true);
}
}
//@Test
public void testUpdateMavenResolver() throws InvalidSyntaxException, IOException {
Dictionary<String, String> props = new Hashtable<String, String>();
props.put("org.ops4j.pax.url.mvn.socket.readTimeout", "59");
props.put("org.ops4j.pax.url.mvn.socket.connectionTimeout", "119");
repHandler.updateMavenResolver(TalendMavenResolver.TALEND_ARTIFACT_LIBRARIES_RESOLVER, props);
BundleContext context = CoreRuntimePlugin.getInstance().getBundle().getBundleContext();
ServiceReference ca = context.getServiceReference(ConfigurationAdmin.class);
debug("" + ca);
ConfigurationAdmin configAdmin = (ConfigurationAdmin) context.getService(ca);
if ( configAdmin == null ) {
Configuration conf = configAdmin.getConfiguration("(service.pid=org.ops4j.pax.url.mvn)");
assertEquals("59", conf.getProperties().get("org.ops4j.pax.url.mvn.socket.readTimeout"));
assertEquals("119", conf.getProperties().get("org.ops4j.pax.url.mvn.socket.connectionTimeout"));
}
}
@Before
@After
public void cleanup() {
try {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/jar";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
List<MavenArtifact> searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
deleteNexusArtifacts(searchRet,false,true);
uri = "mvn:org.talend.libraries/test/6.0.0/jar";
artifact = MavenUrlHelper.parseMvnUrl(uri);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, true, false);
deleteNexusArtifacts(searchRet,true,true);
} catch ( Exception ex ) {
}
}
private void clearLocalFile(String uri) {
LocalLibraryManager localLibraryManager = new LocalLibraryManager();
String localJarPath = localLibraryManager.getJarPathFromMaven(uri);
debug("localJarPath:" + localJarPath);
// force to delete the file to have a valid test
if (localJarPath != null) {
org.talend.utils.io.FilesUtils.deleteFolder(new File(localJarPath).getParentFile().getParentFile(), true);
}
// file should not exist anymore
assertNull(localLibraryManager.getJarPathFromMaven(uri));
}
private static void createNexusRepository(String repId,String versionPolicy) throws IOException {
if (!"SNAPSHOT".equalsIgnoreCase(versionPolicy) && !"RELEASE".equalsIgnoreCase(versionPolicy)) {
Assert.fail("Repository Version Policy must be SNAPSHOT OR RELEASE, but got " + versionPolicy);
}
String getstmt = customNexusServer.getServer() +
nexusprops.getProperty("rep.list.endpoint").replace("{repid}", repId);
try {
String[] response = RestAPIUtil.doRequest(getstmt, "GET", customNexusServer.getUserName(), customNexusServer.getPassword(), null);
if ( response[0].equals("200")) return;
} catch (Exception ex ) {
//Can not find and will create a new one
debug("Can not find the repository named as " + repId + " and will create a new one :" + ex.getMessage());
}
String createstmt = customNexusServer.getServer() + nexusprops.getProperty("rep.create.endpoint");
String jsonfilepath = FileLocator.toFileURL(Platform.getBundle("org.talend.librariesmanager.test").getEntry("resources/nexus/nexus3_create_rep.json")).getFile();
String data = new String(Files.readAllBytes(new File(jsonfilepath).toPath())).replace("NEWREPNAME", repId).replace("NEWVERSIONPOLICY", versionPolicy.toUpperCase());
try {
String[] response = RestAPIUtil.doRequest(createstmt, "POST", customNexusServer.getUserName(), customNexusServer.getPassword(), data);
if ( response[0].equals("201")) debug("Created the test repository successfully!");
} catch (Exception ex ) {
debug("Exception when create the repository of " + repId + ":" + ex );
}
}
private static void deleteNexusArtifacts( List<MavenArtifact> artifacts,boolean isRelease, boolean skiperror) throws Exception{
if (artifacts == null) return;
for (MavenArtifact artifact : artifacts) {
deleteNexusArtifact(artifact,isRelease,skiperror);
}
}
private static void deleteNexusArtifact(MavenArtifact artifact, boolean fromRelease, boolean skiperror) throws Exception {
try {
StringBuffer searchstmt = new StringBuffer();
searchstmt.append(customNexusServer.getServer())
.append(nexusprops.getProperty("component.search.endpoint"))
.append("?repository=").append( fromRelease ? customNexusServer.getRepositoryId() : customNexusServer.getSnapshotRepId())
.append("&group=").append(artifact.getGroupId())
.append("&name=").append(artifact.getArtifactId());
if ( !artifact.getVersion().endsWith(MavenUrlHelper.VERSION_SNAPSHOT))
searchstmt.append("&version=").append(artifact.getVersion());
String[] resp = RestAPIUtil.doRequest(searchstmt.toString(), "GET", customNexusServer.getUserName(), customNexusServer.getPassword(), null);
if ( !resp[0].equals("200")) return;
JSONObject obj = JSONObject.fromObject(resp[1]);
JSONArray items = obj.getJSONArray("items");
if ( items.size() < 1) return;
JSONObject item = (JSONObject) items.get(0);
String componentid = item.getString("id");
String deltestmt = customNexusServer.getServer() + nexusprops.getProperty("component.delete.endpoint").replace("{id}", componentid);
RestAPIUtil.doRequest(deltestmt, "DELETE", customNexusServer.getUserName(), customNexusServer.getPassword(), null);
} catch (Exception ex) {
if (!skiperror) throw ex;
}
}
private String getSha1(File file) throws IOException {
FileInputStream fis = new FileInputStream(file);
String sha1 = DigestUtils.shaHex(fis);
fis.close();
return sha1;
}
private String getSha1(String file) throws IOException {
return getSha1(new File(file));
}
private static void debug(String message) {
System.out.println("[DEBUG]:" + message);
}
}

View File

@@ -1,52 +0,0 @@
package org.talend.librariesmanager.nexus.utils;
import java.util.Properties;
import org.talend.core.nexus.ArtifactRepositoryBean;
import org.talend.core.nexus.IRepositoryArtifactHandler;
import org.talend.core.nexus.RepositoryArtifactHandlerManager;
public class NexusServerManagerProxy {
private static String NEXUS_USER = "nexus.user";
private static String NEXUS_PASSWORD = "nexus.password";
private static String NEXUS_URL = "nexus.url";
private static String NEXUS_LIB_REPO = "nexus.lib.repo";
private static String NEXUS_LIB_SNAPSHOT_REPO = "nexus.lib.repo.snapshot";
private static String DEFAULT_LIB_REPO = "rep-for-test-releases";
private static String DEFAULT_LIB_SNAPSHOT_REPO = "rep-for-test-snapshots";
private static String NEXUS_LIB_SERVER_TYPE = "nexus.lib.server.type";
private static Properties props;
private static NexusServerManagerProxy manager = null;
public static synchronized NexusServerManagerProxy getInstance() {
if (manager == null) {
manager = new NexusServerManagerProxy();
}
return manager;
}
public ArtifactRepositoryBean getCustomNexusServer() {
String nexus_url = System.getProperty(NEXUS_URL);
String nexus_user = System.getProperty(NEXUS_USER);
String nexus_pass = System.getProperty(NEXUS_PASSWORD);
String repositoryId = System.getProperty(NEXUS_LIB_REPO, DEFAULT_LIB_REPO);
String snapshotRepId = System.getProperty(NEXUS_LIB_SNAPSHOT_REPO, DEFAULT_LIB_SNAPSHOT_REPO);
String serverType = System.getProperty(NEXUS_LIB_SERVER_TYPE, "NEXUS_3");
ArtifactRepositoryBean serverBean = new ArtifactRepositoryBean();
serverBean.setServer(nexus_url);
serverBean.setUserName(nexus_user);
serverBean.setPassword(nexus_pass);
serverBean.setRepositoryId(repositoryId);
serverBean.setSnapshotRepId(snapshotRepId);
serverBean.setType(serverType);
return serverBean;
}
public ArtifactRepositoryBean getDummyNexusServer() {
ArtifactRepositoryBean serverBean = getCustomNexusServer();
serverBean.setServer("http://localhost:0000");
return serverBean;
}
}

View File

@@ -1,57 +0,0 @@
package org.talend.librariesmanager.nexus.utils;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
public class RestAPIUtil {
public static String[] doRequest(String uri, String httpmethod, String user, String password, String data)
throws Exception {
String[] ret = new String[2];
StringBuffer response = new StringBuffer();
URL url = new URL(uri);
String auth = user + ":" + password;
try {
byte[] encodedAuth = Base64.getEncoder().encode((auth.getBytes(StandardCharsets.UTF_8)));
String authHeaderValue = "Basic " + new String(encodedAuth);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Authorization", authHeaderValue);
conn.setRequestProperty("Content-Type", "application/json");
conn.setRequestProperty("Accept", "application/json");
conn.setRequestMethod(httpmethod);
if (data != null) {
conn.setDoOutput(true);
conn.setDoInput(true);
conn.connect();
try (OutputStreamWriter streamWriter = new OutputStreamWriter(conn.getOutputStream());) {
streamWriter.write(data);
streamWriter.flush();
}
}
int responsecode = conn.getResponseCode();
debug(responsecode + " is returned for " + httpmethod + " on the endpoint:" + uri);
ret[0] = Integer.toString(responsecode);
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
String responseLine = null;
while ((responseLine = br.readLine()) != null) {
response.append(responseLine.trim());
}
}
ret[1] = response.toString();
conn.disconnect();
} catch (Exception ex) {
debug(ex.getMessage());
throw ex;
}
return ret;
}
private static void debug(String message) {
System.out.println("[DEBUG]:" + message);
}
}