jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(): use string instead of path to be Windows compatible for namespace files (#4471)

Browse Source
This commit is contained in:
YannC
2024-07-30 11:39:53 +02:00
committed by GitHub
parent 079d3ec9fb
commit e5cd7d2c2f
6 changed files with 42 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/src/main/java/io/kestra/core/storages/Namespace.java
View File

@@ -89,6 +89,14 @@ public interface Namespace {
NamespaceFile putFile(Path path, InputStream content, Conflicts onAlreadyExist) throws IOException;
default NamespaceFile putFile(NamespaceFile file, InputStream content) throws IOException {
return putFile(file, content, Conflicts.OVERWRITE);
}
default NamespaceFile putFile(NamespaceFile file, InputStream content, Conflicts onAlreadyExist) throws IOException {
return putFile(Path.of(file.path()), content, onAlreadyExist);
}
/**
* Creates a new directory for the current namespace.
*
@@ -105,7 +113,7 @@ public interface Namespace {
* @throws IOException if an error happens while performing the delete operation.
*/
default boolean delete(NamespaceFile file) throws IOException {
return delete(file.path());
return delete(Path.of(file.path()));
}
/**

17
core/src/main/java/io/kestra/core/storages/NamespaceFile.java
View File

@@ -14,11 +14,15 @@ import java.util.Objects;
* @param namespace The namespace of the file.
*/
public record NamespaceFile(
Path path,
String path,
URI uri,
String namespace
) {
public NamespaceFile(Path path, URI uri, String namespace) {
this(path.toString(), uri, namespace);
}
/**
* Static factory method for constructing a new {@link NamespaceFile} object.
* <p>
@@ -86,7 +90,7 @@ public record NamespaceFile(
Objects.requireNonNull(namespace, "namespace cannot be null");
if (path == null || path.equals(Path.of("/"))) {
return new NamespaceFile(
Path.of(""),
"",
URI.create(StorageContext.KESTRA_PROTOCOL + StorageContext.namespaceFilePrefix(namespace) + "/"),
namespace
);
@@ -97,9 +101,12 @@ public record NamespaceFile(
if (filePath.isAbsolute()) {
filePath = filePath.getRoot().relativize(filePath);
}
// Need to remove starting trailing slash for Windows
String pathWithoutTrailingSlash = path.toString().replaceFirst("^[.]*[\\\\|/]*", "");
return new NamespaceFile(
filePath,
URI.create(StorageContext.KESTRA_PROTOCOL + namespacePrefixPath.resolve(filePath)),
pathWithoutTrailingSlash,
URI.create(StorageContext.KESTRA_PROTOCOL + namespacePrefixPath.resolve(pathWithoutTrailingSlash).toString().replace("\\","/")),
namespace
);
}
@@ -121,7 +128,7 @@ public record NamespaceFile(
return Path.of("/").resolve(path);
}
}
return path;
return Path.of(path);
}
/**

2
core/src/main/java/io/kestra/plugin/core/flow/WorkingDirectory.java
View File

@@ -269,7 +269,7 @@ public class WorkingDirectory extends Sequential implements NamespaceFilesInterf
.findAllFilesMatching(this.namespaceFiles.getInclude(), this.namespaceFiles.getExclude())
.forEach(Rethrow.throwConsumer(namespaceFile -> {
InputStream content = runContext.storage().getFile(namespaceFile.uri());
runContext.workingDir().putFile(namespaceFile.path(), content);
runContext.workingDir().putFile(Path.of(namespaceFile.path()), content);
}));
}

24
core/src/test/java/io/kestra/core/storages/InternalNamespaceTest.java
View File

@@ -65,7 +65,7 @@ class InternalNamespaceTest {
// Then
assertThat(namespaceFile, is(NamespaceFile.of(namespaceId, Path.of("sub/dir/file.txt"))));
// Then
try (InputStream is = namespace.getFileContent(namespaceFile.path())) {
try (InputStream is = namespace.getFileContent(Path.of(namespaceFile.path()))) {
assertThat(new String(is.readAllBytes()), is("1"));
}
}
@@ -78,13 +78,13 @@ class InternalNamespaceTest {
NamespaceFile namespaceFile = namespace.get(Path.of("/sub/dir/file.txt"));
namespace.putFile(namespaceFile.path(), new ByteArrayInputStream("1".getBytes()));
namespace.putFile(namespaceFile, new ByteArrayInputStream("1".getBytes()));
// When
namespace.putFile(namespaceFile.path(), new ByteArrayInputStream("2".getBytes()), Namespace.Conflicts.OVERWRITE);
namespace.putFile(namespaceFile, new ByteArrayInputStream("2".getBytes()), Namespace.Conflicts.OVERWRITE);
// Then
try (InputStream is = namespace.getFileContent(namespaceFile.path())) {
try (InputStream is = namespace.getFileContent(Path.of(namespaceFile.path()))) {
assertThat(new String(is.readAllBytes()), is("2"));
}
}
@@ -97,12 +97,12 @@ class InternalNamespaceTest {
NamespaceFile namespaceFile = namespace.get(Path.of("/sub/dir/file.txt"));
namespace.putFile(namespaceFile.path(), new ByteArrayInputStream("1".getBytes()));
namespace.putFile(namespaceFile, new ByteArrayInputStream("1".getBytes()));
// When - Then
Assertions.assertThrows(
IOException.class,
() -> namespace.putFile(namespaceFile.path(), new ByteArrayInputStream("2".getBytes()), Namespace.Conflicts.ERROR)
() -> namespace.putFile(namespaceFile, new ByteArrayInputStream("2".getBytes()), Namespace.Conflicts.ERROR)
);
}
@@ -114,13 +114,13 @@ class InternalNamespaceTest {
NamespaceFile namespaceFile = namespace.get(Path.of("/sub/dir/file.txt"));
namespace.putFile(namespaceFile.path(), new ByteArrayInputStream("1".getBytes()));
namespace.putFile(namespaceFile, new ByteArrayInputStream("1".getBytes()));
// When
namespace.putFile(namespaceFile.path(), new ByteArrayInputStream("2".getBytes()), Namespace.Conflicts.SKIP);
namespace.putFile(namespaceFile, new ByteArrayInputStream("2".getBytes()), Namespace.Conflicts.SKIP);
// Then
try (InputStream is = namespace.getFileContent(namespaceFile.path())) {
try (InputStream is = namespace.getFileContent(Path.of(namespaceFile.path()))) {
assertThat(new String(is.readAllBytes()), is("1"));
}
}
@@ -146,9 +146,9 @@ class InternalNamespaceTest {
// Then
assertThat(namespaceFiles.stream().map(NamespaceFile::path).toList(), containsInAnyOrder(
is(Path.of("a/b/c/1.sql")),
is(Path.of("b/c/d/3.sql")),
is(Path.of("c/5.sql"))
is("a/b/c/1.sql"),
is("b/c/d/3.sql"),
is("c/5.sql")
));
}

9
core/src/test/java/io/kestra/plugin/core/namespace/UploadFilesTest.java
View File

@@ -20,6 +20,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -84,7 +85,7 @@ public class UploadFilesTest {
List<NamespaceFile> namespaceFiles = namespaceStorage.all();
assertThat(namespaceFiles.size(), is(1));
String previousFile = IOUtils.toString(namespaceStorage.getFileContent(namespaceFiles.getFirst().path()), StandardCharsets.UTF_8);
String previousFile = IOUtils.toString(namespaceStorage.getFileContent(Path.of(namespaceFiles.getFirst().path())), StandardCharsets.UTF_8);
fileStorage = addToStorage("logback.xml");
uploadFile = uploadFile.toBuilder()
@@ -96,7 +97,7 @@ public class UploadFilesTest {
namespaceFiles = namespaceStorage.all();
assertThat(namespaceFiles.size(), is(1));
String newFile = IOUtils.toString(namespaceStorage.getFileContent(namespaceFiles.getFirst().path()), StandardCharsets.UTF_8);
String newFile = IOUtils.toString(namespaceStorage.getFileContent(Path.of(namespaceFiles.getFirst().path())), StandardCharsets.UTF_8);
assertThat(previousFile.equals(newFile), is(false));
}
@@ -123,7 +124,7 @@ public class UploadFilesTest {
List<NamespaceFile> namespaceFiles = namespaceStorage.all();
assertThat(namespaceFiles.size(), is(1));
String previousFile = IOUtils.toString(namespaceStorage.getFileContent(namespaceFiles.getFirst().path()), StandardCharsets.UTF_8);
String previousFile = IOUtils.toString(namespaceStorage.getFileContent(Path.of(namespaceFiles.getFirst().path())), StandardCharsets.UTF_8);
fileStorage = addToStorage("logback.xml");
uploadFile = uploadFile.toBuilder()
@@ -135,7 +136,7 @@ public class UploadFilesTest {
namespaceFiles = namespaceStorage.all();
assertThat(namespaceFiles.size(), is(1));
String newFile = IOUtils.toString(namespaceStorage.getFileContent(namespaceFiles.getFirst().path()), StandardCharsets.UTF_8);
String newFile = IOUtils.toString(namespaceStorage.getFileContent(Path.of(namespaceFiles.getFirst().path())), StandardCharsets.UTF_8);
assertThat(previousFile.equals(newFile), is(true));
}

5
storage-local/src/main/java/io/kestra/storage/local/LocalStorage.java
View File

@@ -78,10 +78,11 @@ public class LocalStorage implements StorageInterface {
Files.walkFileTree(fsPath, new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
String dirPath = dir.toString().replace("\\", "/");
if (includeDirectories) {
uris.add(URI.create(dir + "/"));
uris.add(URI.create(dirPath + "/"));
}
return super.preVisitDirectory(dir, attrs);
return super.preVisitDirectory(Path.of(dirPath), attrs);
}
@Override