mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-19 18:05:41 -05:00
fix(cli): also fetch parent namespaces resources for metadata migration
part of https://github.com/kestra-io/kestra-ee/issues/6019#event-21325612781
This commit is contained in:
committed by
brian-mulier-p
parent
56fb304ff6
commit
8f3a5058b1
@@ -1,5 +1,6 @@
|
|||||||
package io.kestra.cli.commands.migrations.metadata;
|
package io.kestra.cli.commands.migrations.metadata;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import io.kestra.core.models.kv.PersistedKvMetadata;
|
import io.kestra.core.models.kv.PersistedKvMetadata;
|
||||||
import io.kestra.core.models.namespaces.files.NamespaceFileMetadata;
|
import io.kestra.core.models.namespaces.files.NamespaceFileMetadata;
|
||||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||||
@@ -11,18 +12,16 @@ import io.kestra.core.storages.StorageInterface;
|
|||||||
import io.kestra.core.storages.kv.InternalKVStore;
|
import io.kestra.core.storages.kv.InternalKVStore;
|
||||||
import io.kestra.core.storages.kv.KVEntry;
|
import io.kestra.core.storages.kv.KVEntry;
|
||||||
import io.kestra.core.tenant.TenantService;
|
import io.kestra.core.tenant.TenantService;
|
||||||
import jakarta.inject.Inject;
|
import io.kestra.core.utils.NamespaceUtils;
|
||||||
import jakarta.inject.Singleton;
|
import jakarta.inject.Singleton;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.NoSuchFileException;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.Collections;
|
import java.util.*;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@@ -30,25 +29,18 @@ import static io.kestra.core.utils.Rethrow.throwConsumer;
|
|||||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
|
@AllArgsConstructor
|
||||||
public class MetadataMigrationService {
|
public class MetadataMigrationService {
|
||||||
@Inject
|
protected FlowRepositoryInterface flowRepository;
|
||||||
private TenantService tenantService;
|
protected TenantService tenantService;
|
||||||
|
protected KvMetadataRepositoryInterface kvMetadataRepository;
|
||||||
|
protected NamespaceFileMetadataRepositoryInterface namespaceFileMetadataRepository;
|
||||||
|
protected StorageInterface storageInterface;
|
||||||
|
|
||||||
@Inject
|
@VisibleForTesting
|
||||||
private FlowRepositoryInterface flowRepository;
|
public Map<String, List<String>> namespacesPerTenant() {
|
||||||
|
|
||||||
@Inject
|
|
||||||
private KvMetadataRepositoryInterface kvMetadataRepository;
|
|
||||||
|
|
||||||
@Inject
|
|
||||||
private NamespaceFileMetadataRepositoryInterface namespaceFileMetadataRepository;
|
|
||||||
|
|
||||||
@Inject
|
|
||||||
private StorageInterface storageInterface;
|
|
||||||
|
|
||||||
protected Map<String, List<String>> namespacesPerTenant() {
|
|
||||||
String tenantId = tenantService.resolveTenant();
|
String tenantId = tenantService.resolveTenant();
|
||||||
return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId));
|
return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId).stream().map(NamespaceUtils::asTree).flatMap(Collection::stream).distinct().toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void kvMigration() throws IOException {
|
public void kvMigration() throws IOException {
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package io.kestra.cli.commands.migrations.metadata;
|
||||||
|
|
||||||
|
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||||
|
import io.kestra.core.tenant.TenantService;
|
||||||
|
import io.kestra.core.utils.NamespaceUtils;
|
||||||
|
import io.kestra.core.utils.TestsUtils;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
public class MetadataMigrationServiceTest<T extends MetadataMigrationService> {
|
||||||
|
private static final String TENANT_ID = TestsUtils.randomTenant();
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void namespacesPerTenant() {
|
||||||
|
Map<String, List<String>> expected = getNamespacesPerTenant();
|
||||||
|
Map<String, List<String>> result = metadataMigrationService(
|
||||||
|
expected
|
||||||
|
).namespacesPerTenant();
|
||||||
|
|
||||||
|
assertThat(result).hasSize(expected.size());
|
||||||
|
expected.forEach((tenantId, namespaces) -> {;
|
||||||
|
assertThat(result.get(tenantId)).containsExactlyInAnyOrderElementsOf(namespaces.stream().map(NamespaceUtils::asTree).flatMap(Collection::stream).distinct().toList());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, List<String>> getNamespacesPerTenant() {
|
||||||
|
return Map.of(TENANT_ID, List.of("my.first.namespace", "my.second.namespace", "another.namespace"));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected T metadataMigrationService(Map<String, List<String>> namespacesPerTenant) {
|
||||||
|
FlowRepositoryInterface mockedFlowRepository = Mockito.mock(FlowRepositoryInterface.class);
|
||||||
|
Mockito.doAnswer((params) -> namespacesPerTenant.get(params.getArgument(0).toString())).when(mockedFlowRepository).findDistinctNamespace(Mockito.anyString());
|
||||||
|
//noinspection unchecked
|
||||||
|
return ((T) new MetadataMigrationService(mockedFlowRepository, new TenantService() {
|
||||||
|
@Override
|
||||||
|
public String resolveTenant() {
|
||||||
|
return TENANT_ID;
|
||||||
|
}
|
||||||
|
}, null, null, null));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -80,7 +80,7 @@ abstract public class TestsUtils {
|
|||||||
}
|
}
|
||||||
var tenantRegex = "^[a-z0-9][a-z0-9_-]*";
|
var tenantRegex = "^[a-z0-9][a-z0-9_-]*";
|
||||||
var validTenantPrefixes = Arrays.stream(prefix)
|
var validTenantPrefixes = Arrays.stream(prefix)
|
||||||
.map(s -> s.replace(".", "-").replace("$", "-"))
|
.map(s -> s.replaceAll("[.$<>]", "-"))
|
||||||
.map(String::toLowerCase)
|
.map(String::toLowerCase)
|
||||||
.peek(p -> {
|
.peek(p -> {
|
||||||
if (!p.matches(tenantRegex)) {
|
if (!p.matches(tenantRegex)) {
|
||||||
|
|||||||
Reference in New Issue
Block a user