mirror of
https://github.com/turbot/steampipe.git
synced 2025-12-19 09:58:53 -05:00
Update embedded postgresql version to 14.19 (#4671)
This commit is contained in:
@@ -9,12 +9,12 @@ on:
|
|||||||
Version number for the OCI image for this release - usually the same as the
|
Version number for the OCI image for this release - usually the same as the
|
||||||
postgres version
|
postgres version
|
||||||
required: true
|
required: true
|
||||||
default: 14.17.0
|
default: 14.19.0
|
||||||
|
|
||||||
postgres_version:
|
postgres_version:
|
||||||
description: "Postgres Version to package (eg 14.2.0)"
|
description: "Postgres Version to package (eg 14.2.0)"
|
||||||
required: true
|
required: true
|
||||||
default: 14.17.0
|
default: 14.19.0
|
||||||
|
|
||||||
env:
|
env:
|
||||||
PROJECT_ID: steampipe
|
PROJECT_ID: steampipe
|
||||||
@@ -78,32 +78,32 @@ jobs:
|
|||||||
- name: Pull & Extract - darwin amd64
|
- name: Pull & Extract - darwin amd64
|
||||||
run: |-
|
run: |-
|
||||||
EXTRACT_DIR=extracted-darwin-amd64
|
EXTRACT_DIR=extracted-darwin-amd64
|
||||||
# new link (darwin-amd64.txz) - https://drive.google.com/file/d/12fxd_jZxdP4Ilwd0BnefbVCLZV957TNJ/view?usp=drive_link
|
# new link (darwin-amd64.txz) - https://drive.google.com/file/d/14t83HKLJ-ckLwgqRnkVPMTeaw5wMoPu2/view?usp=drive_link
|
||||||
curl -L -o darwin-amd64.txz "https://drive.google.com/uc?export=download&id=12fxd_jZxdP4Ilwd0BnefbVCLZV957TNJ"
|
curl -L -o darwin-amd64.txz "https://drive.google.com/uc?export=download&id=14t83HKLJ-ckLwgqRnkVPMTeaw5wMoPu2"
|
||||||
mkdir $EXTRACT_DIR
|
mkdir $EXTRACT_DIR
|
||||||
tar -xf darwin-amd64.txz --directory $EXTRACT_DIR
|
tar -xf darwin-amd64.txz --directory $EXTRACT_DIR
|
||||||
|
|
||||||
- name: Pull & Extract - darwin arm64
|
- name: Pull & Extract - darwin arm64
|
||||||
run: |-
|
run: |-
|
||||||
EXTRACT_DIR=extracted-darwin-arm64
|
EXTRACT_DIR=extracted-darwin-arm64
|
||||||
# new link (darwin-arm64.txz) - https://drive.google.com/file/d/1lG6eNYM5JaywbRxQBubzMDmbCtGVilQh/view?usp=drive_link
|
# new link (darwin-arm64.txz) - https://drive.google.com/file/d/1UXdLn9aNDY_5OVLjgwF_cAPKiAsxHHB-/view?usp=drive_link
|
||||||
curl -L -o darwin-arm64.txz "https://drive.google.com/uc?export=download&id=1lG6eNYM5JaywbRxQBubzMDmbCtGVilQh"
|
curl -L -o darwin-arm64.txz "https://drive.google.com/uc?export=download&id=1UXdLn9aNDY_5OVLjgwF_cAPKiAsxHHB-"
|
||||||
mkdir $EXTRACT_DIR
|
mkdir $EXTRACT_DIR
|
||||||
tar -xf darwin-arm64.txz --directory $EXTRACT_DIR
|
tar -xf darwin-arm64.txz --directory $EXTRACT_DIR
|
||||||
|
|
||||||
- name: Pull & Extract - linux amd64
|
- name: Pull & Extract - linux amd64
|
||||||
run: |-
|
run: |-
|
||||||
EXTRACT_DIR=extracted-linux-amd64
|
EXTRACT_DIR=extracted-linux-amd64
|
||||||
# new link (linux-amd64.txz) - https://drive.google.com/file/d/11pqDcTtlD7RF5LPj-X4gRVfl4mk00fuy/view?usp=drive_link
|
# new link (linux-amd64.txz) - https://drive.google.com/file/d/1CetXFMZZRAtkVr4gCTlcJcOfIm0wyyYn/view?usp=drive_link
|
||||||
curl -L -o linux-amd64.txz "https://drive.google.com/uc?export=download&id=11pqDcTtlD7RF5LPj-X4gRVfl4mk00fuy"
|
curl -L -o linux-amd64.txz "https://drive.google.com/uc?export=download&id=1CetXFMZZRAtkVr4gCTlcJcOfIm0wyyYn"
|
||||||
mkdir $EXTRACT_DIR
|
mkdir $EXTRACT_DIR
|
||||||
tar -xf linux-amd64.txz --directory $EXTRACT_DIR
|
tar -xf linux-amd64.txz --directory $EXTRACT_DIR
|
||||||
|
|
||||||
- name: Pull & Extract - linux arm64
|
- name: Pull & Extract - linux arm64
|
||||||
run: |-
|
run: |-
|
||||||
EXTRACT_DIR=extracted-linux-arm64
|
EXTRACT_DIR=extracted-linux-arm64
|
||||||
# new link (linux-arm64.txz) - https://drive.google.com/file/d/1lpjyHegwd_8574uVWgaPB31yR7vRB8Zh/view?usp=drive_link
|
# new link (linux-arm64.txz) - https://drive.google.com/file/d/1ce43gJu8HPAufcsZEk87uFPyT6fGEair/view?usp=drive_link
|
||||||
curl -L -o linux-arm64.txz "https://drive.google.com/uc?export=download&id=1lpjyHegwd_8574uVWgaPB31yR7vRB8Zh"
|
curl -L -o linux-arm64.txz "https://drive.google.com/uc?export=download&id=1ce43gJu8HPAufcsZEk87uFPyT6fGEair"
|
||||||
mkdir $EXTRACT_DIR
|
mkdir $EXTRACT_DIR
|
||||||
tar -xf linux-arm64.txz --directory $EXTRACT_DIR
|
tar -xf linux-arm64.txz --directory $EXTRACT_DIR
|
||||||
|
|
||||||
@@ -160,10 +160,10 @@ jobs:
|
|||||||
extracted-linux-amd64:application/vnd.turbot.steampipe.db.linux-amd64.layer.v1+tar \
|
extracted-linux-amd64:application/vnd.turbot.steampipe.db.linux-amd64.layer.v1+tar \
|
||||||
extracted-linux-arm64:application/vnd.turbot.steampipe.db.linux-arm64.layer.v1+tar
|
extracted-linux-arm64:application/vnd.turbot.steampipe.db.linux-arm64.layer.v1+tar
|
||||||
|
|
||||||
# check if the version is NOT an rc version before tagging as latest
|
# check if the version is NOT an pre-release version before tagging as latest
|
||||||
if [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
if [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||||
echo "Tagging as latest: $LATEST_REF"
|
echo "Tagging as latest: $LATEST_REF"
|
||||||
oras tag $REF latest
|
oras tag $REF latest
|
||||||
else
|
else
|
||||||
echo "Skipping latest tag for rc version: $VERSION"
|
echo "Skipping latest tag for pre-release version: $VERSION"
|
||||||
fi
|
fi
|
||||||
|
|||||||
32
.github/workflows/11-test-acceptance.yaml
vendored
32
.github/workflows/11-test-acceptance.yaml
vendored
@@ -67,7 +67,9 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
mkdir ~/artifacts
|
mkdir ~/artifacts
|
||||||
mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_linux_amd64.tar.gz ~/artifacts/linux.tar.gz
|
mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_linux_amd64.tar.gz ~/artifacts/linux.tar.gz
|
||||||
|
mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_linux_arm64.tar.gz ~/artifacts/linux-arm64.tar.gz
|
||||||
mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_darwin_arm64.zip ~/artifacts/darwin.zip
|
mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_darwin_arm64.zip ~/artifacts/darwin.zip
|
||||||
|
mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_darwin_amd64.zip ~/artifacts/darwin-amd64.zip
|
||||||
|
|
||||||
- name: List Build Artifacts
|
- name: List Build Artifacts
|
||||||
run: ls -l ~/artifacts
|
run: ls -l ~/artifacts
|
||||||
@@ -80,14 +82,6 @@ jobs:
|
|||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
overwrite: true
|
overwrite: true
|
||||||
|
|
||||||
- name: Save MacOS Build Artifact
|
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
||||||
with:
|
|
||||||
name: build-artifact-darwin
|
|
||||||
path: ~/artifacts/darwin.zip
|
|
||||||
if-no-files-found: error
|
|
||||||
overwrite: true
|
|
||||||
|
|
||||||
acceptance_test:
|
acceptance_test:
|
||||||
name: Test
|
name: Test
|
||||||
needs: goreleaser
|
needs: goreleaser
|
||||||
@@ -146,20 +140,7 @@ jobs:
|
|||||||
name: build-artifact-linux
|
name: build-artifact-linux
|
||||||
path: ~/artifacts
|
path: ~/artifacts
|
||||||
|
|
||||||
- name: Download Darwin Build Artifacts
|
- name: Extract Linux Artifacts and Install Binary
|
||||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
||||||
if: ${{ matrix.platform == 'macos-latest' }}
|
|
||||||
with:
|
|
||||||
name: build-artifact-darwin
|
|
||||||
path: ~/artifacts
|
|
||||||
|
|
||||||
- name: Extract Darwin Artifacts and Install Binary
|
|
||||||
if: ${{ matrix.platform == 'macos-latest' }}
|
|
||||||
run: |
|
|
||||||
mkdir ~/build
|
|
||||||
unzip ~/artifacts/darwin.zip -d ~/build
|
|
||||||
|
|
||||||
- name: Extract Ubuntu Artifacts and Install Binary
|
|
||||||
if: ${{ matrix.platform == 'ubuntu-latest' }}
|
if: ${{ matrix.platform == 'ubuntu-latest' }}
|
||||||
run: |
|
run: |
|
||||||
mkdir ~/build
|
mkdir ~/build
|
||||||
@@ -167,20 +148,21 @@ jobs:
|
|||||||
|
|
||||||
- name: Set PATH
|
- name: Set PATH
|
||||||
run: |
|
run: |
|
||||||
echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV
|
echo "PATH=$PATH:$HOME/build:$GITHUB_WORKSPACE/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV
|
||||||
|
|
||||||
- name: Go install jd
|
- name: Go install jd
|
||||||
run: |
|
run: |-
|
||||||
go install github.com/josephburnett/jd@latest
|
go install github.com/josephburnett/jd@latest
|
||||||
|
|
||||||
- name: Install DB
|
- name: Install DB
|
||||||
id: install-db
|
id: install-db
|
||||||
continue-on-error: false
|
continue-on-error: false
|
||||||
run: |
|
run: |
|
||||||
steampipe query "select 1"
|
STEAMPIPE_LOG_LEVEL=trace steampipe query "select 1"
|
||||||
steampipe plugin install chaos chaosdynamic --progress=false
|
steampipe plugin install chaos chaosdynamic --progress=false
|
||||||
|
|
||||||
- name: Save Install DB Logs
|
- name: Save Install DB Logs
|
||||||
|
if: always()
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: install-db-logs-${{ matrix.test_block }}-${{ matrix.platform }}
|
name: install-db-logs-${{ matrix.test_block }}-${{ matrix.platform }}
|
||||||
|
|||||||
241
design/embedded_postgres_build_instructions.md
Normal file
241
design/embedded_postgres_build_instructions.md
Normal file
@@ -0,0 +1,241 @@
|
|||||||
|
# PostgreSQL Source Build Instructions
|
||||||
|
|
||||||
|
This document provides step-by-step instructions for building the embedded PostgreSQL binaries required by Steampipe for a specific PostgreSQL version. It covers both macOS and Linux environments, including prerequisites, build steps, and packaging guidelines to ensure the resulting binaries are relocatable and suitable for Steampipe's use.
|
||||||
|
|
||||||
|
1. **Source Code:**
|
||||||
|
[https://www.postgresql.org/ftp/source/](https://www.postgresql.org/ftp/source/)
|
||||||
|
|
||||||
|
2. **Build Documentation:**
|
||||||
|
[https://www.postgresql.org/docs/current/install-make.html](https://www.postgresql.org/docs/current/install-make.html)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Download Source Code and Run
|
||||||
|
|
||||||
|
### For MacOS
|
||||||
|
|
||||||
|
#### 3.1. Pre-requisites
|
||||||
|
|
||||||
|
* `openssl`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 3.2. Steps to Build
|
||||||
|
|
||||||
|
1. Change to the PostgreSQL source directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /postgres/source/dir
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Set environment variables:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export MACOSX_DEPLOYMENT_TARGET=11.0
|
||||||
|
export CFLAGS="-mmacosx-version-min=11.0"
|
||||||
|
export LDFLAGS="-mmacosx-version-min=11.0 -Wl,-rpath,@loader_path/../lib/postgresql"
|
||||||
|
```
|
||||||
|
|
||||||
|
*(Rebuild with an older deployment target)*
|
||||||
|
|
||||||
|
3. Configure the build:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./configure --prefix=location/where/you/want/the/files \
|
||||||
|
--libdir=/location/where/you/want/the/files/lib/postgresql \
|
||||||
|
--datadir=/location/where/you/want/the/files/share/postgresql \
|
||||||
|
--with-openssl \
|
||||||
|
--with-includes=$(brew --prefix openssl)/include \
|
||||||
|
--with-libraries=$(brew --prefix openssl)/lib
|
||||||
|
```
|
||||||
|
|
||||||
|
*(Make sure the `libdir` and `datadir` args are passed correctly and point to the `postgresql` dir inside `lib` and `share` — this is needed for Steampipe.)*
|
||||||
|
|
||||||
|
4. Build PostgreSQL:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make -j$(sysctl -n hw.ncpu)
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Install binaries:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make install
|
||||||
|
```
|
||||||
|
|
||||||
|
6. Verify that all binaries are built in the specified location.
|
||||||
|
|
||||||
|
7. Build contrib modules:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make -C contrib
|
||||||
|
```
|
||||||
|
|
||||||
|
8. Install contrib modules:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make -C contrib install
|
||||||
|
```
|
||||||
|
|
||||||
|
9. *(This builds extensions in the contrib directory — needed since we load `ltree` and `tablefunc`.)*
|
||||||
|
|
||||||
|
10. Verify installation structure:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ls -al location/where/you/want/the/files
|
||||||
|
```
|
||||||
|
|
||||||
|
You should see `lib`, `share`, `bin`, and `include` directories under that path.
|
||||||
|
|
||||||
|
11. Remove the `include` directory.
|
||||||
|
|
||||||
|
12. Remove unneeded binaries from `bin`.
|
||||||
|
|
||||||
|
13. Check that all extensions exist.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 3.3. Fix RPATHs
|
||||||
|
|
||||||
|
Run the `fix_rpath.sh` script to fix the rpaths of the binaries (`initdb`, `pg_restore`, `pg_dump`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# --- CONFIGURE ---
|
||||||
|
# Adjust if your libpq lives in lib/ not lib/postgresql
|
||||||
|
LIB_SUBDIR="lib/postgresql"
|
||||||
|
BUNDLE_ROOT="$(pwd)"
|
||||||
|
LIBPQ_PATH="$BUNDLE_ROOT/$LIB_SUBDIR/libpq.5.dylib"
|
||||||
|
|
||||||
|
echo "🔧 Fixing libpq install name..."
|
||||||
|
install_name_tool -id "@rpath/libpq.5.dylib" "$LIBPQ_PATH"
|
||||||
|
|
||||||
|
echo "🔍 Processing binaries in bin/..."
|
||||||
|
for binfile in "$BUNDLE_ROOT"/bin/*; do
|
||||||
|
[[ -x "$binfile" && ! -d "$binfile" ]] || continue
|
||||||
|
echo "➡️ Patching $(basename "$binfile")"
|
||||||
|
|
||||||
|
# Ensure an rpath to ../lib/postgresql exists
|
||||||
|
install_name_tool -add_rpath "@loader_path/../$LIB_SUBDIR" "$binfile" 2>/dev/null || true
|
||||||
|
|
||||||
|
# Rewrite any absolute reference to libpq
|
||||||
|
install_name_tool -change "$BUNDLE_ROOT/$LIB_SUBDIR/libpq.5.dylib" "@rpath/libpq.5.dylib" "$binfile" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "✅ Verification:"
|
||||||
|
for binfile in "$BUNDLE_ROOT"/bin/*; do
|
||||||
|
[[ -x "$binfile" && ! -d "$binfile" ]] || continue
|
||||||
|
echo "--- $(basename "$binfile") ---"
|
||||||
|
otool -L "$binfile" | grep libpq || echo "⚠️ No libpq linkage"
|
||||||
|
otool -l "$binfile" | grep -A2 LC_RPATH | grep path || echo "⚠️ No RPATH"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 3.4. Pack the Built Binaries
|
||||||
|
|
||||||
|
Create a `.txz` archive:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tar --disable-copyfile --exclude='._*' -cJf darwin-arm64.txz -C darwin-arm64 bin lib share
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### For Linux (Ubuntu 24 / amd64 or arm64)
|
||||||
|
|
||||||
|
#### 3.5. Pre-requisites
|
||||||
|
|
||||||
|
```bash
|
||||||
|
apt update
|
||||||
|
apt install -y build-essential wget ca-certificates \
|
||||||
|
libreadline-dev zlib1g-dev flex bison \
|
||||||
|
libssl-dev patchelf file
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 3.6. Steps to Build
|
||||||
|
|
||||||
|
1. Change to the PostgreSQL source directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /postgres/source/dir
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Set installation prefix and linker flags:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export PREFIX=/postgres-binaries-14.19/linux-$(uname -m)
|
||||||
|
mkdir -p "$PREFIX"
|
||||||
|
export LDFLAGS='-Wl,-rpath,$ORIGIN/../lib/postgresql -Wl,--enable-new-dtags'
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Configure:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./configure \
|
||||||
|
--prefix="$PREFIX" \
|
||||||
|
--libdir="$PREFIX/lib/postgresql" \
|
||||||
|
--datadir="$PREFIX/share/postgresql" \
|
||||||
|
--with-openssl \
|
||||||
|
--with-includes=/usr/include \
|
||||||
|
--with-libraries=/usr/lib/$(uname -m)-linux-gnu
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Build and install:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make -j2
|
||||||
|
make install
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Build contrib extensions:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make -C contrib -j2
|
||||||
|
make -C contrib install
|
||||||
|
```
|
||||||
|
|
||||||
|
6. Patch RPATHs for relocatability:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd "$PREFIX"
|
||||||
|
for f in bin/*; do
|
||||||
|
if [ -x "$f" ] && file "$f" | grep -q ELF; then
|
||||||
|
patchelf --set-rpath '$ORIGIN/../lib/postgresql' "$f"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
7. Verify RPATH:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
readelf -d bin/initdb | grep -i rpath
|
||||||
|
# → RUNPATH [$ORIGIN/../lib/postgresql]
|
||||||
|
```
|
||||||
|
|
||||||
|
8. Verify linkage:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ldd bin/initdb | grep libpq
|
||||||
|
# → libpq.so.5 => .../bin/../lib/postgresql/libpq.so.5
|
||||||
|
```
|
||||||
|
|
||||||
|
9. Remove `include` directory and unnecessary binaries:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rm -rf "$PREFIX/include"
|
||||||
|
```
|
||||||
|
|
||||||
|
10. Pack into `.txz`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd $(dirname "$PREFIX")
|
||||||
|
tar -cJf postgres-14.19-$(uname -m).txz $(basename "$PREFIX")
|
||||||
|
```
|
||||||
|
|
||||||
|
✅ **Done.**
|
||||||
@@ -27,12 +27,12 @@ const (
|
|||||||
|
|
||||||
// constants for installing db and fdw images
|
// constants for installing db and fdw images
|
||||||
const (
|
const (
|
||||||
DatabaseVersion = "14.17.0"
|
DatabaseVersion = "14.19.0"
|
||||||
FdwVersion = "2.1.3"
|
FdwVersion = "2.1.3"
|
||||||
|
|
||||||
// PostgresImageRef is the OCI Image ref for the database binaries
|
// PostgresImageRef is the OCI Image ref for the database binaries
|
||||||
PostgresImageRef = "ghcr.io/turbot/steampipe/db:14.17.0"
|
PostgresImageRef = "ghcr.io/turbot/steampipe/db:14.19.0"
|
||||||
PostgresImageDigest = "sha256:4d66131a30139b2b912abe17bcd93c6b4e6a89224874960fea0f141ca51168e7"
|
PostgresImageDigest = "sha256:346d096caf87a6886fd404594ac6b72abee0b166236711fdc6c3d347c099e61f"
|
||||||
|
|
||||||
FdwImageRef = "ghcr.io/turbot/steampipe/fdw:" + FdwVersion
|
FdwImageRef = "ghcr.io/turbot/steampipe/fdw:" + FdwVersion
|
||||||
FdwBinaryFileName = "steampipe_postgres_fdw.so"
|
FdwBinaryFileName = "steampipe_postgres_fdw.so"
|
||||||
|
|||||||
@@ -332,7 +332,7 @@ func runRestoreUsingList(ctx context.Context, info *RunningDBInstanceInfo, listF
|
|||||||
fmt.Sprintf("--username=%s", constants.DatabaseSuperUser),
|
fmt.Sprintf("--username=%s", constants.DatabaseSuperUser),
|
||||||
)
|
)
|
||||||
|
|
||||||
log.Println("[TRACE]", cmd.String())
|
log.Println("[TRACE] pg_restore command:", cmd.String())
|
||||||
|
|
||||||
if output, err := cmd.CombinedOutput(); err != nil {
|
if output, err := cmd.CombinedOutput(); err != nil {
|
||||||
log.Println("[TRACE] runRestoreUsingList process:", string(output))
|
log.Println("[TRACE] runRestoreUsingList process:", string(output))
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import (
|
|||||||
var ensureMux sync.Mutex
|
var ensureMux sync.Mutex
|
||||||
|
|
||||||
func noBackupWarning() string {
|
func noBackupWarning() string {
|
||||||
warningMessage := `Steampipe database has been upgraded from Postgres 14.2 to Postgres 14.17.
|
warningMessage := `Steampipe database has been upgraded from Postgres 14.17 to Postgres 14.19.
|
||||||
|
|
||||||
Unfortunately the data in your public schema failed migration using the standard pg_dump and pg_restore tools. Your data has been preserved in the ~/.steampipe/db directory.
|
Unfortunately the data in your public schema failed migration using the standard pg_dump and pg_restore tools. Your data has been preserved in the ~/.steampipe/db directory.
|
||||||
|
|
||||||
|
|||||||
@@ -648,11 +648,13 @@ func FindAllSteampipePostgresInstances(ctx context.Context) ([]*psutils.Process,
|
|||||||
var instances []*psutils.Process
|
var instances []*psutils.Process
|
||||||
allProcesses, err := psutils.ProcessesWithContext(ctx)
|
allProcesses, err := psutils.ProcessesWithContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
log.Println("[TRACE] FindAllSteampipePostgresInstances - error retrieving process list: ", err.Error())
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, p := range allProcesses {
|
for _, p := range allProcesses {
|
||||||
cmdLine, err := p.CmdlineSliceWithContext(ctx)
|
cmdLine, err := p.CmdlineSliceWithContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
log.Printf("[TRACE] FindAllSteampipePostgresInstances - error retrieving cmdline for pid %d: %s", p.Pid, err.Error())
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if isSteampipePostgresProcess(ctx, cmdLine) {
|
if isSteampipePostgresProcess(ctx, cmdLine) {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
steampipe query "select 1" --install-dir $target_install_directory
|
steampipe query "select 1" --install-dir $target_install_directory
|
||||||
|
|
||||||
# check postgres binary is present in correct location
|
# check postgres binary is present in correct location
|
||||||
run file $target_install_directory/db/14.17.0/postgres/bin/postgres
|
run file $target_install_directory/db/14.19.0/postgres/bin/postgres
|
||||||
if [[ "$arch" == "x86_64" && "$os" == "Darwin" ]]; then
|
if [[ "$arch" == "x86_64" && "$os" == "Darwin" ]]; then
|
||||||
assert_output --partial 'Mach-O 64-bit executable x86_64'
|
assert_output --partial 'Mach-O 64-bit executable x86_64'
|
||||||
elif [[ "$arch" == "arm64" && "$os" == "Darwin" ]]; then
|
elif [[ "$arch" == "arm64" && "$os" == "Darwin" ]]; then
|
||||||
@@ -21,7 +21,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# check initdb binary is present in the correct location
|
# check initdb binary is present in the correct location
|
||||||
run file $target_install_directory/db/14.17.0/postgres/bin/initdb
|
run file $target_install_directory/db/14.19.0/postgres/bin/initdb
|
||||||
if [[ "$arch" == "arm64" && "$os" == "Darwin" ]]; then
|
if [[ "$arch" == "arm64" && "$os" == "Darwin" ]]; then
|
||||||
assert_output --partial 'Mach-O 64-bit executable arm64'
|
assert_output --partial 'Mach-O 64-bit executable arm64'
|
||||||
elif [[ "$arch" == "x86_64" && "$os" == "Darwin" ]]; then
|
elif [[ "$arch" == "x86_64" && "$os" == "Darwin" ]]; then
|
||||||
@@ -33,7 +33,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# check fdw binary(steampipe_postgres_fdw.so) is present in the correct location
|
# check fdw binary(steampipe_postgres_fdw.so) is present in the correct location
|
||||||
run file $target_install_directory/db/14.17.0/postgres/lib/postgresql/steampipe_postgres_fdw.so
|
run file $target_install_directory/db/14.19.0/postgres/lib/postgresql/steampipe_postgres_fdw.so
|
||||||
if [[ "$arch" == "arm64" && "$os" == "Darwin" ]]; then
|
if [[ "$arch" == "arm64" && "$os" == "Darwin" ]]; then
|
||||||
assert_output --partial 'Mach-O 64-bit bundle arm64'
|
assert_output --partial 'Mach-O 64-bit bundle arm64'
|
||||||
elif [[ "$arch" == "x86_64" && "$os" == "Darwin" ]]; then
|
elif [[ "$arch" == "x86_64" && "$os" == "Darwin" ]]; then
|
||||||
@@ -45,7 +45,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# check fdw extension(steampipe_postgres_fdw.control) is present in the correct location
|
# check fdw extension(steampipe_postgres_fdw.control) is present in the correct location
|
||||||
run file $target_install_directory/db/14.17.0/postgres/share/postgresql/extension/steampipe_postgres_fdw.control
|
run file $target_install_directory/db/14.19.0/postgres/share/postgresql/extension/steampipe_postgres_fdw.control
|
||||||
assert_output --partial 'ASCII text'
|
assert_output --partial 'ASCII text'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -75,6 +75,78 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
rm -f verify*
|
rm -f verify*
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "verify data is properly migrated when upgrading from v2.2.0" {
|
||||||
|
# setup sql statements
|
||||||
|
setup_sql[0]="create table sample(sample_col_1 char(10), sample_col_2 char(10))"
|
||||||
|
setup_sql[1]="insert into sample(sample_col_1,sample_col_2) values ('foo','bar')"
|
||||||
|
setup_sql[2]="insert into sample(sample_col_1,sample_col_2) values ('foo1','bar1')"
|
||||||
|
setup_sql[3]="create function sample_func() returns integer as 'select 1 as result;' language sql;"
|
||||||
|
|
||||||
|
# verify sql statements
|
||||||
|
verify_sql[0]="select * from sample"
|
||||||
|
verify_sql[1]="select * from sample_func()"
|
||||||
|
|
||||||
|
# create a temp directory to install steampipe(2.2.0)
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
mkdir -p "${tmpdir}"
|
||||||
|
tmpdir="${tmpdir%/}"
|
||||||
|
|
||||||
|
# find the name of the zip file as per OS and arch
|
||||||
|
case $(uname -sm) in
|
||||||
|
"Darwin x86_64") target="darwin_amd64.zip" ;;
|
||||||
|
"Darwin arm64") target="darwin_arm64.zip" ;;
|
||||||
|
"Linux x86_64") target="linux_amd64.tar.gz" ;;
|
||||||
|
"Linux aarch64") target="linux_arm64.tar.gz" ;;
|
||||||
|
*) echo "Error: '$(uname -sm)' is not supported yet." 1>&2;exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# download the zip and extract
|
||||||
|
steampipe_uri="https://github.com/turbot/steampipe/releases/download/v2.2.0/steampipe_${target}"
|
||||||
|
case $(uname -s) in
|
||||||
|
"Darwin") zip_location="${tmpdir}/steampipe.zip" ;;
|
||||||
|
"Linux") zip_location="${tmpdir}/steampipe.tar.gz" ;;
|
||||||
|
*) echo "Error: steampipe is not supported on '$(uname -s)' yet." 1>&2;exit 1 ;;
|
||||||
|
esac
|
||||||
|
curl --fail --location --progress-bar --output "$zip_location" "$steampipe_uri"
|
||||||
|
tar -xf "$zip_location" -C "$tmpdir"
|
||||||
|
|
||||||
|
# start the service
|
||||||
|
$tmpdir/steampipe --install-dir $tmpdir service start
|
||||||
|
|
||||||
|
# execute the setup sql statements
|
||||||
|
for ((i = 0; i < ${#setup_sql[@]}; i++)); do
|
||||||
|
$tmpdir/steampipe --install-dir $tmpdir query "${setup_sql[$i]}"
|
||||||
|
done
|
||||||
|
|
||||||
|
# store the result of the verification statements(1.0.3)
|
||||||
|
for ((i = 0; i < ${#verify_sql[@]}; i++)); do
|
||||||
|
$tmpdir/steampipe --install-dir $tmpdir query "${verify_sql[$i]}" > verify$i.txt
|
||||||
|
done
|
||||||
|
|
||||||
|
# stop the service
|
||||||
|
$tmpdir/steampipe --install-dir $tmpdir service stop
|
||||||
|
|
||||||
|
# Now run this version - which should migrate the data
|
||||||
|
steampipe --install-dir $tmpdir service start
|
||||||
|
|
||||||
|
# store the result of the verification statements(0.14.*)
|
||||||
|
for ((i = 0; i < ${#verify_sql[@]}; i++)); do
|
||||||
|
echo "VerifySQL: ${verify_sql[$i]}"
|
||||||
|
steampipe --install-dir $tmpdir query "${verify_sql[$i]}" > verify$i$i.txt
|
||||||
|
done
|
||||||
|
|
||||||
|
# stop the service
|
||||||
|
steampipe --install-dir $tmpdir service stop
|
||||||
|
|
||||||
|
# verify data is migrated correctly
|
||||||
|
for ((i = 0; i < ${#verify_sql[@]}; i++)); do
|
||||||
|
assert_equal "$(cat verify$i.txt)" "$(cat verify$i$i.txt)"
|
||||||
|
done
|
||||||
|
|
||||||
|
rm -rf $tmpdir
|
||||||
|
rm -f verify*
|
||||||
|
}
|
||||||
|
|
||||||
function teardown_file() {
|
function teardown_file() {
|
||||||
# list running processes
|
# list running processes
|
||||||
ps -ef | grep steampipe
|
ps -ef | grep steampipe
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
current_year=$(date +"%Y")
|
current_year=$(date +"%Y")
|
||||||
steampipe service start
|
steampipe service start
|
||||||
|
|
||||||
run openssl x509 -enddate -noout -in $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/root.crt
|
run openssl x509 -enddate -noout -in $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/root.crt
|
||||||
echo $output
|
echo $output
|
||||||
# check enddate
|
# check enddate
|
||||||
assert_output --partial "notAfter=Dec 31 23:59:59 9999 GMT"
|
assert_output --partial "notAfter=Dec 31 23:59:59 9999 GMT"
|
||||||
@@ -13,7 +13,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
server_expiry=$((current_year + 3))
|
server_expiry=$((current_year + 3))
|
||||||
echo $server_expiry
|
echo $server_expiry
|
||||||
|
|
||||||
run openssl x509 -enddate -noout -in $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.crt
|
run openssl x509 -enddate -noout -in $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.crt
|
||||||
echo $output
|
echo $output
|
||||||
# check enddate
|
# check enddate
|
||||||
assert_output --partial "$server_expiry"
|
assert_output --partial "$server_expiry"
|
||||||
@@ -23,25 +23,25 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
steampipe service start
|
steampipe service start
|
||||||
|
|
||||||
# save file hash
|
# save file hash
|
||||||
run cksum $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/root.crt
|
run cksum $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/root.crt
|
||||||
id_root=$(echo $output | awk '{print $1}')
|
id_root=$(echo $output | awk '{print $1}')
|
||||||
echo $id_root
|
echo $id_root
|
||||||
|
|
||||||
# save file hash
|
# save file hash
|
||||||
run cksum $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.crt
|
run cksum $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.crt
|
||||||
id_server=$(echo $output | awk '{print $1}')
|
id_server=$(echo $output | awk '{print $1}')
|
||||||
echo $id_server
|
echo $id_server
|
||||||
|
|
||||||
steampipe service restart
|
steampipe service restart
|
||||||
|
|
||||||
# check file hash after restart
|
# check file hash after restart
|
||||||
run cksum $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/root.crt
|
run cksum $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/root.crt
|
||||||
id_root_new=$(echo $output | awk '{print $1}')
|
id_root_new=$(echo $output | awk '{print $1}')
|
||||||
echo $id_root_new
|
echo $id_root_new
|
||||||
assert_equal $id_root $id_root_new
|
assert_equal $id_root $id_root_new
|
||||||
|
|
||||||
# check file hash after restart
|
# check file hash after restart
|
||||||
run cksum $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.crt
|
run cksum $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.crt
|
||||||
id_server_new=$(echo $output | awk '{print $1}')
|
id_server_new=$(echo $output | awk '{print $1}')
|
||||||
echo $id_server_new
|
echo $id_server_new
|
||||||
|
|
||||||
@@ -52,17 +52,17 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
|
|
||||||
@test "deleting root certificate, service start should regenerate server and root certs" {
|
@test "deleting root certificate, service start should regenerate server and root certs" {
|
||||||
# save file hash
|
# save file hash
|
||||||
run cksum $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.crt
|
run cksum $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.crt
|
||||||
id_server=$(echo $output | awk '{print $1}')
|
id_server=$(echo $output | awk '{print $1}')
|
||||||
echo $id_server
|
echo $id_server
|
||||||
|
|
||||||
# delete root certificate
|
# delete root certificate
|
||||||
rm -f $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/root.crt
|
rm -f $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/root.crt
|
||||||
|
|
||||||
steampipe service start
|
steampipe service start
|
||||||
|
|
||||||
# save new file hash
|
# save new file hash
|
||||||
run cksum $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.crt
|
run cksum $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.crt
|
||||||
id_server_new=$(echo $output | awk '{print $1}')
|
id_server_new=$(echo $output | awk '{print $1}')
|
||||||
echo $id_server_new
|
echo $id_server_new
|
||||||
|
|
||||||
@@ -78,9 +78,9 @@ load "$LIB_BATS_SUPPORT/load.bash"
|
|||||||
|
|
||||||
@test "adding an encrypted private key should work fine and service should start successfully" {
|
@test "adding an encrypted private key should work fine and service should start successfully" {
|
||||||
skip "TODO update test and enable later"
|
skip "TODO update test and enable later"
|
||||||
run openssl genrsa -aes256 -out $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.key -passout pass:steampipe -traditional 2048
|
run openssl genrsa -aes256 -out $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.key -passout pass:steampipe -traditional 2048
|
||||||
|
|
||||||
run openssl req -key $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.key -passin pass:steampipe -new -x509 -out $STEAMPIPE_INSTALL_DIR/db/14.17.0/data/server.crt -subj "/CN=steampipe.io"
|
run openssl req -key $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.key -passin pass:steampipe -new -x509 -out $STEAMPIPE_INSTALL_DIR/db/14.19.0/data/server.crt -subj "/CN=steampipe.io"
|
||||||
|
|
||||||
steampipe service start --database-password steampipe
|
steampipe service start --database-password steampipe
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user