Connector builder: E2e tests (#21122)

* wip

* wip

* e2e tests for connector builder server

* rename function

* clean up

* clean up a bit more

* fix path

* fix and add documentation

* more documentation

* stabilze

* review comments

airbyte-webapp-e2e-tests/README.md

@@ -8,12 +8,13 @@ Except as noted, all commands are written as if run from inside the `airbyte-web
 Steps:
 1) If you have not already done so, run `npm install` to install the e2e test dependencies.
 2) Build the OSS backend for the current commit with `SUB_BUILD=PLATFORM ../gradlew clean build`.
-3) Create the test database: `npm run createdbsource`.
-4) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker-compose --file ../docker-compose.yaml up`. If you want, follow this with `docker-compose stop webapp` to turn off the dockerized frontend build; interactive cypress sessions don't use it.
-5) The following two commands will start a separate long-running server, so open another terminal window. In it, `cd` into the `airbyte-webapp/` directory.
-6) If you have not already done so, run `npm install` to install the frontend app's dependencies.
-7) Start the frontend development server with `npm start`.
-8) Back in the `airbyte-webapp-e2e-tests/` directory, start the cypress test runner with `npm run cypress:open`.
+3) Create the test database: `npm run createdbsource` and `npm run createdbdestination`.
+4) When running the connector builder tests, start the dummy API server: `npm run createdummyapi`
+5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker-compose --file ../docker-compose.yaml up`. If you want, follow this with `docker-compose stop webapp` to turn off the dockerized frontend build; interactive cypress sessions don't use it.
+6) The following two commands will start a separate long-running server, so open another terminal window. In it, `cd` into the `airbyte-webapp/` directory.
+7) If you have not already done so, run `npm install` to install the frontend app's dependencies.
+8) Start the frontend development server with `npm start`.
+9) Back in the `airbyte-webapp-e2e-tests/` directory, start the cypress test runner with `npm run cypress:open`.
 
 ## Reproducing CI test results with `npm run cypress:ci` or `npm run cypress:ci:record`
 Unlike `npm run cypress:open`, `npm run cypress:ci` and `npm run cypress:ci:record` use the dockerized UI (i.e. they expect the UI at port 8000, rather than port 3000). If the OSS backend is running but you have run `docker-compose stop webapp`, you'll have to re-enable it with `docker-compose start webapp`. These trigger headless runs: you won't have a live browser to interact with, just terminal output.
@@ -23,6 +24,15 @@ Except as noted, all commands are written as if run from inside the `airbyte-web
 Steps:
 1) If you have not already done so, run `npm install` to install the e2e test dependencies.
 2) Build the OSS backend for the current commit with `SUB_BUILD=PLATFORM ../gradlew clean build`.
-3) Create the test database: `npm run createdbsource`.
-4) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker-compose --file ../docker-compose.yaml up`.
-5) Start the cypress test run with `npm run cypress:ci` or `npm run cypress:ci:record`.
+3) Create the test database: `npm run createdbsource` and `npm run createdbdestination`.
+4) When running the connector builder tests, start the dummy API server: `npm run createdummyapi`
+5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker-compose --file ../docker-compose.yaml up`.
+6) Start the cypress test run with `npm run cypress:ci` or `npm run cypress:ci:record`.
+
+## Test setup
+
+When the tests are run as described above, the platform under test is started via docker compose on the local docker host. To test connections from real sources and destinations, additional docker containers are started for hosting these. For basic connections, additional postgres instances are started (`createdbsource` and `createdbdestination`).
+
+For testing the connector builder UI, a dummy api server based on a node script is started (`createdummyapi`). It is providing a simple http API with bearer authentication returning a few records of hardcoded data. By running it in the internal airbyte network, the connector builder server can access it under its container name.
+
+The tests in here are instrumenting a Chrome instance to test the full functionality of Airbyte from the frontend, so other components of the platform (scheduler, worker, connector builder server) are also tested in a rudimentary way.

airbyte-webapp-e2e-tests/cypress/commands/connectorBuilder.ts

@@ -0,0 +1,69 @@
+import {
+  addStream,
+  configureOffsetPagination,
+  enterName,
+  enterRecordSelector,
+  enterStreamName,
+  enterTestInputs,
+  enterUrlBase,
+  enterUrlPath,
+  goToTestPage,
+  goToView,
+  openTestInputs,
+  selectAuthMethod,
+  submitForm,
+  togglePagination
+} from "pages/connectorBuilderPage";
+
+export const configureGlobals = () => {
+  goToView("global");
+  enterName("Dummy API");
+  enterUrlBase("http://dummy_api:6767/");
+}
+
+export const configureStream = () => {
+  addStream();
+  enterStreamName("Items");
+  enterUrlPath("items/");
+  submitForm();
+  enterRecordSelector("items");
+}
+
+export const configureAuth = () => {
+  goToView("global");
+  selectAuthMethod("Bearer");
+  openTestInputs();
+  enterTestInputs({ apiKey: "theauthkey" })
+  submitForm();
+}
+
+export const configurePagination = () => {
+  goToView("0");
+  togglePagination();
+  configureOffsetPagination("2", "header", "offset");
+}
+
+const testPanelContains = (str: string) => {
+  cy.get("pre").contains(str).should("exist");
+}
+
+export const assertTestReadAuthFailure = () => {
+  testPanelContains('"error": "Bad credentials"');
+};
+
+export const assertTestReadItems = () => {
+  testPanelContains('"name": "abc"');
+  testPanelContains('"name": "def"');
+};
+
+export const assertMultiPageReadItems = () => {
+  goToTestPage(1);
+  assertTestReadItems();
+
+  goToTestPage(2);
+  testPanelContains('"name": "xxx"');
+  testPanelContains('"name": "yyy"');
+
+  goToTestPage(3);
+  testPanelContains('[]');
+};

airbyte-webapp-e2e-tests/cypress/integration/connectorBuilder.spec.ts

@@ -0,0 +1,30 @@
+import { goToConnectorBuilderPage, testStream } from "pages/connectorBuilderPage";
+import { assertTestReadItems, assertTestReadAuthFailure, configureAuth, configureGlobals, configureStream, configurePagination, assertMultiPageReadItems } from "commands/connectorBuilder";
+
+describe("Connector builder", () => {
+  before(() => {
+    goToConnectorBuilderPage();
+  });
+
+  it("Configure basic connector", () => {
+    configureGlobals();
+    configureStream();
+  });
+
+  it("Fail on missing auth", () => {
+    testStream();
+    assertTestReadAuthFailure();
+  });
+
+  it("Succeed on provided auth", () => {
+    configureAuth();
+    testStream();
+    assertTestReadItems();
+  });
+
+  it("Pagination", () => {
+    configurePagination();
+    testStream();
+    assertMultiPageReadItems();
+  });
+});

airbyte-webapp-e2e-tests/cypress/pages/connectorBuilderPage.ts

@@ -0,0 +1,91 @@
+const nameInput = "input[name='global.connectorName']";
+const urlBaseInput = "input[name='global.urlBase']";
+const addStreamButton = "button[data-testid='add-stream']";
+const apiKeyInput = "input[name='connectionConfiguration.api_key']";
+const toggleInput = "input[data-testid='toggle']";
+const streamNameInput = "input[name='streamName']";
+const streamUrlPath = "input[name='urlPath']";
+const recordSelectorInput = "[data-testid='tag-input'] input";
+const authType = "[data-testid='global.authenticator.type']";
+const testInputsButton = "[data-testid='test-inputs']";
+const limitInput = "[name='streams[0].paginator.strategy.page_size']";
+const injectOffsetInto = "[data-testid$='paginator.pageTokenOption.inject_into']";
+const injectOffsetFieldName = "[name='streams[0].paginator.pageTokenOption.field_name']";
+const testPageItem = "[data-testid='test-pages'] li";
+const submit = "button[type='submit']"
+const testStreamButton = "button[data-testid='read-stream']";
+
+export const goToConnectorBuilderPage = () => {
+  cy.visit("/connector-builder");
+  cy.wait(3000);
+};
+
+export const enterName = (name: string) => {
+  cy.get(nameInput).type(name);
+};
+
+export const enterUrlBase = (urlBase: string) => {
+  cy.get(urlBaseInput).type(urlBase);
+};
+
+export const enterRecordSelector = (recordSelector: string) => {
+  cy.get(recordSelectorInput).first().type(recordSelector, { force: true }).type("{enter}", { force: true });
+};
+
+const selectFromDropdown = (selector: string, value: string) => {
+  cy.get(`${selector} .react-select__dropdown-indicator`).last().click({ force: true });
+
+  cy.get(`.react-select__option`).contains(value).click();
+}
+
+export const selectAuthMethod = (value: string) => {
+  selectFromDropdown(authType, value);
+};
+
+export const goToView = (view: string) => {
+  cy.get(`button[data-testid=navbutton-${view}]`).click();
+}
+
+export const openTestInputs = () => {
+  cy.get(testInputsButton).click();
+}
+
+export const enterTestInputs = ({ apiKey }: { apiKey: string }) => {
+  cy.get(apiKeyInput).type(apiKey);
+}
+
+export const goToTestPage = (page: number) => {
+  cy.get(testPageItem).contains(page).click();
+}
+
+export const togglePagination = () => {
+  cy.get(toggleInput).first().click({ force: true });
+}
+
+export const configureOffsetPagination = (limit: string, into: string, fieldName: string) => {
+  cy.get(limitInput).type(limit);
+  selectFromDropdown(injectOffsetInto, into);
+  cy.get(injectOffsetFieldName).type(fieldName);
+}
+
+export const addStream = () => {
+  cy.get(addStreamButton).click();
+};
+
+export const enterStreamName = (streamName: string) => {
+  cy.get(streamNameInput).type(streamName);
+};
+
+export const enterUrlPath = (urlPath: string) => {
+  cy.get(streamUrlPath).type(urlPath);
+};
+
+export const submitForm = () => {
+  cy.get(submit).click();
+};
+
+export const testStream = () => {
+  // wait for debounced form
+  cy.wait(500);
+  cy.get(testStreamButton).click();
+};

airbyte-webapp-e2e-tests/dummy_api.js

@@ -0,0 +1,28 @@
+// Script starting a basic webserver returning mocked data over an authenticated API to test the connector builder UI and connector builder server in an
+// end to end fashion.
+
+// Start with `npm run createdummyapi`
+
+const http = require('http');
+
+const items = [{ name: "abc" }, { name: "def" }, { name: "xxx" }, { name: "yyy" }];
+
+const requestListener = function (req, res) {
+  if (req.headers["authorization"] !== "Bearer theauthkey") {
+    res.writeHead(403); res.end(JSON.stringify({ error: "Bad credentials" })); return;
+  }
+  if (req.url !== "/items") {
+    res.writeHead(404); res.end(JSON.stringify({ error: "Not found" })); return;
+  }
+  // Add more dummy logic in here
+  res.setHeader("Content-Type", "application/json");
+  res.writeHead(200);
+  res.end(JSON.stringify({ items: [...items].splice(req.headers["offset"] ? Number(req.headers["offset"]) : 0, 2) }));
+}
+
+const server = http.createServer(requestListener);
+server.listen(6767);
+
+process.on('SIGINT', function () {
+  process.exit()
+})

airbyte-webapp-e2e-tests/package.json

@@ -11,6 +11,7 @@
     "cypress:ci:record": "CYPRESS_BASE_URL=http://localhost:8000 cypress run --record --key $CYPRESS_KEY",
     "createdbsource": "docker run --rm -d -p 5433:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_source --name airbyte_ci_pg_source postgres",
     "createdbdestination": "docker run --rm -d -p 5434:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_destination --name airbyte_ci_pg_destination postgres",
+    "createdummyapi": "docker run --rm -d -p 6767:6767 --network=airbyte_airbyte_internal --mount type=bind,source=\"$(pwd)\"/dummy_api.js,target=/index.js --name=dummy_api node:16-alpine \"index.js\"",
     "lint": "eslint --ext js,ts,tsx cypress"
   },
   "devDependencies": {

airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx

@@ -25,9 +25,15 @@ interface AddStreamButtonProps {
   onAddStream: (addedStreamNum: number) => void;
   button?: React.ReactElement;
   initialValues?: Partial<BuilderStream>;
+  "data-testid"?: string;
 }
 
-export const AddStreamButton: React.FC<AddStreamButtonProps> = ({ onAddStream, button, initialValues }) => {
+export const AddStreamButton: React.FC<AddStreamButtonProps> = ({
+  onAddStream,
+  button,
+  initialValues,
+  "data-testid": testId,
+}) => {
   const { formatMessage } = useIntl();
   const [isOpen, setIsOpen] = useState(false);
   const [streamsField, , helpers] = useField<BuilderStream[]>("streams");
@@ -42,9 +48,10 @@ export const AddStreamButton: React.FC<AddStreamButtonProps> = ({ onAddStream, b
       {button ? (
         React.cloneElement(button, {
           onClick: buttonClickHandler,
+          "data-testid": testId,
         })
       ) : (
-        <Button className={styles.addButton} onClick={buttonClickHandler} icon={<PlusIcon />} />
+        <Button className={styles.addButton} onClick={buttonClickHandler} icon={<PlusIcon />} data-testid={testId} />
       )}
       {isOpen && (
         <Formik

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx

@@ -44,6 +44,7 @@ export const BuilderCard: React.FC<React.PropsWithChildren<BuilderCardProps>> =
       {toggleConfig && (
         <div className={styles.toggleContainer}>
           <CheckBox
+            data-testid="toggle"
             checked={toggleConfig.toggledOn}
             onChange={(event) => {
               toggleConfig.onToggle(event.target.checked);

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx

@@ -127,7 +127,13 @@ const InnerBuilderField: React.FC<BuilderFieldProps & FastFieldProps<unknown>> =
         />
       )}
       {props.type === "enum" && (
-        <EnumField options={props.options} value={field.value as string} setValue={setValue} error={hasError} />
+        <EnumField
+          options={props.options}
+          value={field.value as string}
+          setValue={setValue}
+          error={hasError}
+          data-testid={path}
+        />
       )}
       {hasError && (
         <Text className={styles.error}>

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx

@@ -25,6 +25,7 @@ interface ViewSelectButtonProps {
   selected: boolean;
   showErrorIndicator: boolean;
   onClick: () => void;
+  "data-testid": string;
 }
 
 const ViewSelectButton: React.FC<React.PropsWithChildren<ViewSelectButtonProps>> = ({
@@ -33,9 +34,11 @@ const ViewSelectButton: React.FC<React.PropsWithChildren<ViewSelectButtonProps>>
   selected,
   showErrorIndicator,
   onClick,
+  "data-testid": testId,
 }) => {
   return (
     <button
+      data-testid={testId}
       className={classnames(className, styles.viewButton, {
         [styles.selectedViewButton]: selected,
         [styles.unselectedViewButton]: !selected,
@@ -93,6 +96,7 @@ export const BuilderSidebar: React.FC<BuilderSidebarProps> = React.memo(({ class
       </div>
 
       <ViewSelectButton
+        data-testid="navbutton-global"
         className={styles.globalConfigButton}
         selected={selectedView === "global"}
         showErrorIndicator={hasErrors(true, ["global"])}
@@ -103,6 +107,7 @@ export const BuilderSidebar: React.FC<BuilderSidebarProps> = React.memo(({ class
       </ViewSelectButton>
 
       <ViewSelectButton
+        data-testid="navbutton-inputs"
         showErrorIndicator={false}
         className={styles.globalConfigButton}
         selected={selectedView === "inputs"}
@@ -122,13 +127,14 @@ export const BuilderSidebar: React.FC<BuilderSidebarProps> = React.memo(({ class
           <FormattedMessage id="connectorBuilder.streamsHeading" values={{ number: values.streams.length }} />
         </Text>
 
-        <AddStreamButton onAddStream={(addedStreamNum) => handleViewSelect(addedStreamNum)} />
+        <AddStreamButton onAddStream={(addedStreamNum) => handleViewSelect(addedStreamNum)} data-testid="add-stream" />
       </div>
 
       <div className={styles.streamList}>
         {values.streams.map(({ name }, num) => (
           <ViewSelectButton
             key={num}
+            data-testid={`navbutton-${String(num)}`}
             selected={selectedView === num}
             showErrorIndicator={hasErrors(true, [num])}
             onClick={() => handleViewSelect(num)}

airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx

@@ -58,6 +58,7 @@ export const ConfigMenu: React.FC<ConfigMenuProps> = ({ className, testInputJson
             <Button
               size="sm"
               variant="secondary"
+              data-testid="test-inputs"
               onClick={() => setIsOpen(true)}
               disabled={
                 !jsonManifest.spec ||

airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ResultDisplay.tsx

@@ -34,7 +34,7 @@ export const ResultDisplay: React.FC<ResultDisplayProps> = ({ slices, className
       )}
       <PageDisplay className={styles.pageDisplay} page={page} />
       {slice.pages.length > 1 && (
-        <div className={styles.paginator}>
+        <div className={styles.paginator} data-testid="test-pages">
           <Text className={styles.pageLabel}>Page:</Text>
           <Paginator numPages={numPages} onPageChange={setSelectedPage} selectedPage={selectedPage} />
         </div>

airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx

@@ -60,6 +60,7 @@ export const StreamTestButton: React.FC<StreamTestButtonProps> = ({
       size="sm"
       onClick={handleClick}
       disabled={buttonDisabled}
+      data-testid="read-stream"
       icon={
         showWarningIcon ? (
           <FontAwesomeIcon icon={faWarning} />

tools/bin/e2e_test.sh

@@ -21,6 +21,7 @@ VERSION=dev BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" TRACKING_STRATEGY=logg
 
 docker run --rm -d -p 5433:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_source --name airbyte_ci_pg_source postgres
 docker run --rm -d -p 5434:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_destination --name airbyte_ci_pg_destination postgres
+docker run --rm -d -p 6767:6767 --network=airbyte_airbyte_internal --mount type=bind,source="$(pwd)"/airbyte-webapp-e2e-tests/dummy_api.js,target=/index.js --name=dummy_api node:16-alpine "index.js"
 
 echo "Waiting for health API to be available..."
 # Retry loading the health API of the server to check that the server is fully available