fix(cli): also fetch parent namespaces resources for metadata migration

part of https://github.com/kestra-io/kestra-ee/issues/6019#event-21325612781
This commit is contained in:
brian.mulier
2025-12-03 12:47:29 +01:00
committed by brian-mulier-p
parent 56fb304ff6
commit 8f3a5058b1
3 changed files with 62 additions and 23 deletions

View File

@@ -1,5 +1,6 @@
package io.kestra.cli.commands.migrations.metadata; package io.kestra.cli.commands.migrations.metadata;
import com.google.common.annotations.VisibleForTesting;
import io.kestra.core.models.kv.PersistedKvMetadata; import io.kestra.core.models.kv.PersistedKvMetadata;
import io.kestra.core.models.namespaces.files.NamespaceFileMetadata; import io.kestra.core.models.namespaces.files.NamespaceFileMetadata;
import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.repositories.FlowRepositoryInterface;
@@ -11,18 +12,16 @@ import io.kestra.core.storages.StorageInterface;
import io.kestra.core.storages.kv.InternalKVStore; import io.kestra.core.storages.kv.InternalKVStore;
import io.kestra.core.storages.kv.KVEntry; import io.kestra.core.storages.kv.KVEntry;
import io.kestra.core.tenant.TenantService; import io.kestra.core.tenant.TenantService;
import jakarta.inject.Inject; import io.kestra.core.utils.NamespaceUtils;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.AllArgsConstructor;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.nio.file.Path; import java.nio.file.NoSuchFileException;
import java.time.Instant; import java.time.Instant;
import java.util.Collections; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -30,25 +29,18 @@ import static io.kestra.core.utils.Rethrow.throwConsumer;
import static io.kestra.core.utils.Rethrow.throwFunction; import static io.kestra.core.utils.Rethrow.throwFunction;
@Singleton @Singleton
@AllArgsConstructor
public class MetadataMigrationService { public class MetadataMigrationService {
@Inject protected FlowRepositoryInterface flowRepository;
private TenantService tenantService; protected TenantService tenantService;
protected KvMetadataRepositoryInterface kvMetadataRepository;
protected NamespaceFileMetadataRepositoryInterface namespaceFileMetadataRepository;
protected StorageInterface storageInterface;
@Inject @VisibleForTesting
private FlowRepositoryInterface flowRepository; public Map<String, List<String>> namespacesPerTenant() {
@Inject
private KvMetadataRepositoryInterface kvMetadataRepository;
@Inject
private NamespaceFileMetadataRepositoryInterface namespaceFileMetadataRepository;
@Inject
private StorageInterface storageInterface;
protected Map<String, List<String>> namespacesPerTenant() {
String tenantId = tenantService.resolveTenant(); String tenantId = tenantService.resolveTenant();
return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId)); return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId).stream().map(NamespaceUtils::asTree).flatMap(Collection::stream).distinct().toList());
} }
public void kvMigration() throws IOException { public void kvMigration() throws IOException {

View File

@@ -0,0 +1,47 @@
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.tenant.TenantService;
import io.kestra.core.utils.NamespaceUtils;
import io.kestra.core.utils.TestsUtils;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class MetadataMigrationServiceTest<T extends MetadataMigrationService> {
private static final String TENANT_ID = TestsUtils.randomTenant();
@Test
void namespacesPerTenant() {
Map<String, List<String>> expected = getNamespacesPerTenant();
Map<String, List<String>> result = metadataMigrationService(
expected
).namespacesPerTenant();
assertThat(result).hasSize(expected.size());
expected.forEach((tenantId, namespaces) -> {;
assertThat(result.get(tenantId)).containsExactlyInAnyOrderElementsOf(namespaces.stream().map(NamespaceUtils::asTree).flatMap(Collection::stream).distinct().toList());
});
}
protected Map<String, List<String>> getNamespacesPerTenant() {
return Map.of(TENANT_ID, List.of("my.first.namespace", "my.second.namespace", "another.namespace"));
}
protected T metadataMigrationService(Map<String, List<String>> namespacesPerTenant) {
FlowRepositoryInterface mockedFlowRepository = Mockito.mock(FlowRepositoryInterface.class);
Mockito.doAnswer((params) -> namespacesPerTenant.get(params.getArgument(0).toString())).when(mockedFlowRepository).findDistinctNamespace(Mockito.anyString());
//noinspection unchecked
return ((T) new MetadataMigrationService(mockedFlowRepository, new TenantService() {
@Override
public String resolveTenant() {
return TENANT_ID;
}
}, null, null, null));
}
}

View File

@@ -80,7 +80,7 @@ abstract public class TestsUtils {
} }
var tenantRegex = "^[a-z0-9][a-z0-9_-]*"; var tenantRegex = "^[a-z0-9][a-z0-9_-]*";
var validTenantPrefixes = Arrays.stream(prefix) var validTenantPrefixes = Arrays.stream(prefix)
.map(s -> s.replace(".", "-").replace("$", "-")) .map(s -> s.replaceAll("[.$<>]", "-"))
.map(String::toLowerCase) .map(String::toLowerCase)
.peek(p -> { .peek(p -> {
if (!p.matches(tenantRegex)) { if (!p.matches(tenantRegex)) {