jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): have separate commands for KV & secrets metadata migrations (#12585)

Browse Source
This commit is contained in:
brian-mulier-p
2025-11-03 09:33:26 +01:00
committed by GitHub
parent e116186201
commit 3ab6d6a94f
8 changed files with 124 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
cli/src/main/java/io/kestra/cli/commands/migrations/MetadataMigrationCommand.java
View File

@@ -1,26 +0,0 @@
package io.kestra.cli.commands.migrations;
import io.kestra.cli.AbstractCommand;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;
@CommandLine.Command(
name = "metadata",
description = "populate metadata for entities"
)
@Slf4j
public class MetadataMigrationCommand extends AbstractCommand {
@Inject
private MetadataMigrationService metadataMigrationService;
@Override
public Integer call() throws Exception {
super.call();
int returnCode = metadataMigrationService.migrateMetadata();
if (returnCode == 0) {
System.out.println("✅ Metadata migration complete.");
}
return returnCode;
}
}

1
cli/src/main/java/io/kestra/cli/commands/migrations/MigrationCommand.java
View File

@@ -2,6 +2,7 @@ package io.kestra.cli.commands.migrations;
import io.kestra.cli.AbstractCommand;
import io.kestra.cli.App;
import io.kestra.cli.commands.migrations.metadata.MetadataMigrationCommand;
import io.micronaut.configuration.picocli.PicocliRunner;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;

30
cli/src/main/java/io/kestra/cli/commands/migrations/metadata/KvMetadataMigrationCommand.java Normal file
View File

@@ -0,0 +1,30 @@
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.cli.AbstractCommand;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;
@CommandLine.Command(
name = "kv",
description = "populate metadata for KV"
)
@Slf4j
public class KvMetadataMigrationCommand extends AbstractCommand {
@Inject
private MetadataMigrationService metadataMigrationService;
@Override
public Integer call() throws Exception {
super.call();
try {
metadataMigrationService.kvMigration();
} catch (Exception e) {
System.err.println("❌ KV Metadata migration failed: " + e.getMessage());
e.printStackTrace();
return 1;
}
System.out.println("✅ KV Metadata migration complete.");
return 0;
}
}

23
cli/src/main/java/io/kestra/cli/commands/migrations/metadata/MetadataMigrationCommand.java Normal file
View File

@@ -0,0 +1,23 @@
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.cli.AbstractCommand;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;
@CommandLine.Command(
name = "metadata",
description = "populate metadata for entities",
subcommands = {
KvMetadataMigrationCommand.class,
SecretsMetadataMigrationCommand.class
}
)
@Slf4j
public class MetadataMigrationCommand extends AbstractCommand {
@Override
public Integer call() throws Exception {
super.call();
return 0;
}
}

20
cli/src/main/java/io/kestra/cli/commands/migrations/MetadataMigrationService.java → cli/src/main/java/io/kestra/cli/commands/migrations/metadata/MetadataMigrationService.java
View File

@@ -1,4 +1,4 @@
package io.kestra.cli.commands.migrations;
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.core.models.kv.PersistedKvMetadata;
import io.kestra.core.repositories.FlowRepositoryInterface;
@@ -43,19 +43,7 @@ public class MetadataMigrationService {
return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId));
}
public int migrateMetadata() {
try {
kvMigration();
} catch (IOException e) {
System.err.println("❌ KV metadata migration failed: " + e.getMessage());
e.printStackTrace();
return 1;
}
return 0;
}
private void kvMigration() throws IOException {
public void kvMigration() throws IOException {
this.namespacesPerTenant().entrySet().stream()
.flatMap(namespacesForTenant -> namespacesForTenant.getValue().stream().map(namespace -> Map.entry(namespacesForTenant.getKey(), namespace)))
.flatMap(throwFunction(namespaceForTenant -> {
@@ -82,6 +70,10 @@ public class MetadataMigrationService {
.forEach(kvMetadataRepository::save);
}
public void secretMigration() throws Exception {
throw new UnsupportedOperationException("Secret migration is not needed in the OSS version");
}
private static List<FileAttributes> listAllFromStorage(StorageInterface storage, String tenant, String namespace) throws IOException {
try {
return storage.list(tenant, namespace, URI.create(StorageContext.KESTRA_PROTOCOL + StorageContext.kvPrefix(namespace)));

30
cli/src/main/java/io/kestra/cli/commands/migrations/metadata/SecretsMetadataMigrationCommand.java Normal file
View File

@@ -0,0 +1,30 @@
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.cli.AbstractCommand;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;
@CommandLine.Command(
name = "secrets",
description = "populate metadata for secrets"
)
@Slf4j
public class SecretsMetadataMigrationCommand extends AbstractCommand {
@Inject
private MetadataMigrationService metadataMigrationService;
@Override
public Integer call() throws Exception {
super.call();
try {
metadataMigrationService.secretMigration();
} catch (Exception e) {
System.err.println("❌ Secrets Metadata migration failed: " + e.getMessage());
e.printStackTrace();
return 1;
}
System.out.println("✅ Secrets Metadata migration complete.");
return 0;
}
}

10
cli/src/test/java/io/kestra/cli/commands/migrations/MetadataMigrationCommandTest.java → cli/src/test/java/io/kestra/cli/commands/migrations/metadata/KvMetadataMigrationCommandTest.java
View File

@@ -1,4 +1,4 @@
package io.kestra.cli.commands.migrations;
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.cli.App;
import io.kestra.core.exceptions.ResourceExpiredException;
@@ -33,7 +33,7 @@ import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
public class MetadataMigrationCommandTest {
public class KvMetadataMigrationCommandTest {
@Test
void run() throws IOException, ResourceExpiredException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -65,12 +65,12 @@ public class MetadataMigrationCommandTest {
assertThat(kvMetadataRepository.findByName(tenantId, namespace, key).isPresent()).isFalse();
String[] kvMetadataMigrationCommand = {
"migrate", "metadata"
"migrate", "metadata", "kv"
};
PicocliRunner.call(App.class, ctx, kvMetadataMigrationCommand);
assertThat(out.toString()).contains("✅ Metadata migration complete.");
assertThat(out.toString()).contains("✅ KV Metadata migration complete.");
// Still it's not in the metadata repository because no flow exist to find that kv
assertThat(kvMetadataRepository.findByName(tenantId, namespace, key).isPresent()).isFalse();
assertThat(kvMetadataRepository.findByName(tenantId, anotherNamespace, anotherKey).isPresent()).isFalse();
@@ -86,7 +86,7 @@ public class MetadataMigrationCommandTest {
out.reset();
PicocliRunner.call(App.class, ctx, kvMetadataMigrationCommand);
assertThat(out.toString()).contains("✅ Metadata migration complete.");
assertThat(out.toString()).contains("✅ KV Metadata migration complete.");
Optional<PersistedKvMetadata> foundKv = kvMetadataRepository.findByName(tenantId, namespace, key);
assertThat(foundKv.isPresent()).isTrue();
assertThat(foundKv.get().getDescription()).isEqualTo(description);

29
cli/src/test/java/io/kestra/cli/commands/migrations/metadata/SecretsMetadataMigrationCommandTest.java Normal file
View File

@@ -0,0 +1,29 @@
package io.kestra.cli.commands.migrations.metadata;
import io.kestra.cli.App;
import io.micronaut.configuration.picocli.PicocliRunner;
import io.micronaut.context.ApplicationContext;
import io.micronaut.context.env.Environment;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import static org.assertj.core.api.Assertions.assertThat;
public class SecretsMetadataMigrationCommandTest {
@Test
void run() {
ByteArrayOutputStream err = new ByteArrayOutputStream();
System.setErr(new PrintStream(err));
try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
String[] secretMetadataMigrationCommand = {
"migrate", "metadata", "secrets"
};
PicocliRunner.call(App.class, ctx, secretMetadataMigrationCommand);
assertThat(err.toString()).contains("❌ Secrets Metadata migration failed: Secret migration is not needed in the OSS version");
}
}
}