jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

don't split lines on LSEP unicode characters when reading lines in destinations (#3327)

Browse Source 
* use strict JSONL definition of new lines in destinations

* failing test case

* use next instead of nextLine

* add \n in string for test

* bump destination versions

* bump to even newer version

* bump versions in dockerfiles as well

* force mysql test to pass
This commit is contained in:
Jared Rhizor
2021-05-10 12:57:12 -07:00
committed by GitHub
parent 4b79b2a6f9
commit 6fd8e00ad8
21 changed files with 66 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java
View File

@@ -120,11 +120,13 @@ public class IntegrationRunner {
@VisibleForTesting
static void consumeWriteStream(AirbyteMessageConsumer consumer) throws Exception {
final Scanner input = new Scanner(System.in);
// use a Scanner that only processes new line characters to strictly abide with the
// https://jsonlines.org/ standard
final Scanner input = new Scanner(System.in).useDelimiter("[\r\n]+");
try (consumer) {
consumer.start();
while (input.hasNextLine()) {
final String inputString = input.nextLine();
while (input.hasNext()) {
final String inputString = input.next();
final Optional<AirbyteMessage> singerMessageOptional = Jsons.tryDeserialize(inputString, AirbyteMessage.class);
if (singerMessageOptional.isPresent()) {
consumer.accept(singerMessageOptional.get());