jprdonnelly/butler-sos
1
0
Fork 0
mirror of https://github.com/ptarmiganlabs/butler-sos.git synced 2025-12-19 17:58:18 -05:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1094 from ptarmiganlabs/copilot/fix-088b2c3a-f49e-45c6-90f0-4ae42b3b11b0

Browse Source 
Add InfluxDB v3 support to Butler SOS
This commit is contained in:
Göran Sander
2025-12-15 09:07:03 +01:00
committed by GitHub
parent b810815016 fdff29f5d2
commit d0c36d773c
69 changed files with 6315 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
docs/docker-compose/.env_influxdb_v3 Normal file
View File

@@ -0,0 +1,17 @@
# Adapted from https://github.com/InfluxCommunity/TIG-Stack-using-InfluxDB-3/blob/main/.env
# Butler SOS configuration
BUTLER_SOS_CONFIG_FILE=/production_influxdb_v3.yaml # File placed in ./config directory
# InfluxDB Configuration
INFLUXDB_HTTP_PORT=8181 # for influxdb3 enterprise database, change this to port 8182
INFLUXDB_HOST=influxdb3-core # for influxdb3 enterprise database, change this to "influxdb3-enterprise"
INFLUXDB_TOKEN=
INFLUXDB_DATABASE=local_system # Your Database name
INFLUXDB_ORG=local_org
INFLUXDB_NODE_ID=node0
# Grafana Configuration
GRAFANA_PORT=3000
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin

104
docs/docker-compose/README.md Normal file
View File

@@ -0,0 +1,104 @@
# Docker Compose Files for Butler SOS with InfluxDB
This directory contains Docker Compose configurations for running Butler SOS with different versions of InfluxDB.
## Available Configurations
### InfluxDB v1.x
- **File**: `docker-compose_fullstack_influxdb_v1.yml`
- **InfluxDB Image**: `influxdb:1.8-alpine`
- **Features**: Traditional InfluxDB with SQL-like query language
- **Configuration**: Set `Butler-SOS.influxdbConfig.version: 1` in your config file
- **Environment**: Set `NODE_ENV=production_influxdb_v1`
### InfluxDB v2.x
- **File**: `docker-compose_fullstack_influxdb_v2.yml`
- **InfluxDB Image**: `influxdb:2.7-alpine`
- **Features**: Modern InfluxDB with Flux query language, unified time series platform
- **Configuration**: Set `Butler-SOS.influxdbConfig.version: 2` in your config file
- **Environment**: Set `NODE_ENV=production_influxdb_v2`
- **Default Credentials**:
- Username: `admin`
- Password: `butlersos123`
- Organization: `butler-sos`
- Bucket: `butler-sos`
- Token: `butlersos-token`
### InfluxDB v3.x
- **File**: `docker-compose_fullstack_influxdb_v3.yml`
- **InfluxDB Image**: `influxdb:latest`
- **Features**: Latest InfluxDB architecture with enhanced performance and cloud-native design
- **Configuration**: Set `Butler-SOS.influxdbConfig.version: 3` in your config file
- **Environment**: Set `NODE_ENV=production_influxdb_v3`
- **Default Credentials**: Same as v2.x but with database concept support
## Usage
1. Choose the appropriate docker-compose file for your InfluxDB version
2. Create the corresponding configuration file (e.g., `production_influxdb_v2.yaml`)
3. Configure Butler SOS with the correct InfluxDB version and connection details
4. Run with: `docker-compose -f docker-compose_fullstack_influxdb_v2.yml up -d`
## Configuration Requirements
### For InfluxDB v1.x
```yaml
Butler-SOS:
influxdbConfig:
enable: true
version: 1
host: influxdb-v1
port: 8086
v1Config:
auth:
enable: false
dbName: SenseOps
retentionPolicy:
name: 10d
duration: 10d
```
### For InfluxDB v2.x
```yaml
Butler-SOS:
influxdbConfig:
enable: true
version: 2
host: influxdb-v2
port: 8086
v2Config:
org: butler-sos
bucket: butler-sos
token: butlersos-token
description: Butler SOS metrics
retentionDuration: 10d
```
### For InfluxDB v3.x
```yaml
Butler-SOS:
influxdbConfig:
enable: true
version: 3
host: influxdb-v3
port: 8086
v3Config:
database: butler-sos
token: butlersos-token
description: Butler SOS metrics
retentionDuration: 10d
```
## Migration Notes
- **v1 to v2**: Requires data migration using InfluxDB tools
- **v2 to v3**: Uses similar client libraries but different internal architecture
- **v1 to v3**: Significant migration required, consider using InfluxDB migration tools
For detailed configuration options, refer to the main Butler SOS documentation.

27
docs/docker-compose/docker-compose.yml
View File

@@ -1,27 +0,0 @@
# docker-compose.yml
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
command:
- 'node'
- 'src/butler-sos.js'
- '--configfile'
- '/nodeapp/config/production.yaml'
ports:
- '9997:9997' # UDP user events
- '9996:9996' # UDP log events
- '9842:9842' # Prometheus metrics
- '3100:3100' # Config file visualization
volumes:
# Make config file accessible outside of container
- './config:/nodeapp/config'
- './log:/nodeapp/log'
environment:
- 'NODE_ENV=production' # Means that Butler SOS will read config data from production.yaml
logging:
driver: 'json-file'
options:
max-file: '5'
max-size: '5m'

19
docs/docker-compose/docker-compose_fullstack_influxdb.yml → docs/docker-compose/docker-compose_fullstack_influxdb_v1.yml Executable file → Normal file
View File

@@ -1,16 +1,19 @@
# docker-compose_fullstack_influxdb.yml
version: "3.3"
# docker-compose_fullstack_influxdb_v1.yml
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
restart: unless-stopped
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_influxdb" # Means that Butler SOS will read config data from production_influxdb.yaml
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_influxdb_v1.yaml"]
logging:
driver: "json-file"
options:
@@ -21,8 +24,8 @@ services:
influxdb:
image: influxdb:1.12.2
container_name: influxdb
restart: always
container_name: influxdb-v1
restart: unless-stopped
volumes:
- ./influxdb/data:/var/lib/influxdb # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb/ # Mount for influxdb configuration
@@ -39,7 +42,7 @@ services:
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: always
restart: unless-stopped
ports:
- "3000:3000"
volumes:

60
docs/docker-compose/docker-compose_fullstack_influxdb_v2.yml Normal file
View File

@@ -0,0 +1,60 @@
# docker-compose_fullstack_influxdb_v2.yml
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: unless-stopped
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_influxdb_v2.yaml"]
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "5m"
networks:
- senseops
influxdb:
image: influxdb:2.7-alpine
container_name: influxdb-v2
restart: unless-stopped
volumes:
- ./influxdb/data:/var/lib/influxdb2 # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb2/ # Mount for influxdb configuration
ports:
# The API for InfluxDB is served on port 8086
- "8086:8086"
environment:
# Initial setup parameters
- "DOCKER_INFLUXDB_INIT_MODE=setup"
- "DOCKER_INFLUXDB_INIT_USERNAME=admin"
- "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123"
- "DOCKER_INFLUXDB_INIT_ORG=butler-sos"
- "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos"
- "DOCKER_INFLUXDB_INIT_RETENTION=10d"
- "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token"
networks:
- senseops
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: unless-stopped
ports:
- "3000:3000"
volumes:
- ./grafana/data:/var/lib/grafana
networks:
- senseops
networks:
senseops:
driver: bridge

84
docs/docker-compose/docker-compose_fullstack_influxdb_v3.yml Normal file
View File

@@ -0,0 +1,84 @@
# docker-compose_fullstack_influxdb_v3.yml
# InfluxDB v3.x (Core) - using the InfluxDB 3.x Community Edition
# Inspiration from https://github.com/InfluxCommunity/TIG-Stack-using-InfluxDB-3/blob/main/docker-compose.yml
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: unless-stopped
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/${BUTLER_SOS_CONFIG_FILE}"]
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "5m"
depends_on:
# Or switch to influxdb3-enterprise as needed
- influxdb-v3-core
networks:
- senseops
influxdb-v3-core:
# Note: InfluxDB v3 Core is available as influxdb3 image
# For production use, consider InfluxDB Cloud or Enterprise
image: influxdb:3-core
container_name: influxdb-v3-core
restart: unless-stopped
ports:
- ${INFLUXDB_HTTP_PORT}:8181
command:
- influxdb3
- serve
- --node-id=${INFLUXDB_NODE_ID}
- --object-store=file
- --data-dir=/var/lib/influxdb3
volumes:
- ./influxdb/data:/var/lib/influxdb3 # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb3/ # Mount for influxdb configuration
# environment:
# InfluxDB v3 setup - uses similar setup to v2 but different internal architecture
# - "DOCKER_INFLUXDB_INIT_MODE=setup"
# - "DOCKER_INFLUXDB_INIT_USERNAME=admin"
# - "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123"
# - "DOCKER_INFLUXDB_INIT_ORG=butler-sos"
# - "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos"
# - "DOCKER_INFLUXDB_INIT_DATABASE=butler-sos" # v3 uses database concept
# - "DOCKER_INFLUXDB_INIT_RETENTION=10d"
# - "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token"
healthcheck:
test: ["CMD-SHELL", "curl -f -H 'Authorization: Bearer ${INFLUXDB_TOKEN}' http://localhost:8181/health || exit 1"]
interval: 30s
timeout: 10s
retries: 3
networks:
- senseops
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: unless-stopped
ports:
- "${GRAFANA_PORT}:3000"
environment:
- GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER}
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
volumes:
- ./grafana/data:/var/lib/grafana
depends_on:
# Or switch to influxdb3-enterprise as needed
- influxdb-v3-core
networks:
- senseops
networks:
senseops:
driver: bridge

9
docs/docker-compose/docker-compose_fullstack_prometheus.yml Executable file → Normal file
View File

@@ -1,16 +1,19 @@
# docker-compose_fullstack_prometheus.yml
version: "3.3"
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_prometheus" # Means that Butler SOS will read config data from production_prometheus.yaml
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_prometheus.yaml"]
logging:
driver: "json-file"
options:

378
package-lock.json generated
View File

@@ -15,6 +15,7 @@
"@fastify/static": "^8.3.0",
"@influxdata/influxdb-client": "^1.35.0",
"@influxdata/influxdb-client-apis": "^1.35.0",
"@influxdata/influxdb3-client": "^1.4.0",
"ajv": "^8.17.1",
"ajv-keywords": "^5.1.0",
"async-mutex": "^0.5.0",
@@ -44,7 +45,7 @@
"devDependencies": {
"@babel/eslint-parser": "^7.28.5",
"@babel/plugin-syntax-import-assertions": "^7.27.1",
"@eslint/js": "^9.39.1",
"@eslint/js": "^9.39.2",
"audit-ci": "^7.1.0",
"esbuild": "^0.27.1",
"eslint-config-prettier": "^10.1.8",
@@ -52,7 +53,7 @@
"eslint-plugin-jsdoc": "^61.5.0",
"eslint-plugin-prettier": "^5.5.4",
"globals": "^16.5.0",
"jest": "^30.1.3",
"jest": "^30.2.0",
"jsdoc-to-markdown": "^9.1.3",
"license-checker-rseidelsohn": "^4.4.2",
"lockfile-lint": "^4.14.1",
@@ -681,9 +682,9 @@
}
},
"node_modules/@emnapi/core": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.5.0.tgz",
"integrity": "sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==",
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz",
"integrity": "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==",
"dev": true,
"license": "MIT",
"optional": true,
@@ -693,9 +694,9 @@
}
},
"node_modules/@emnapi/runtime": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz",
"integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==",
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz",
"integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==",
"dev": true,
"license": "MIT",
"optional": true,
@@ -1317,9 +1318,9 @@
"peer": true
},
"node_modules/@eslint/js": {
"version": "9.39.1",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz",
"integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==",
"version": "9.39.2",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz",
"integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==",
"dev": true,
"license": "MIT",
"engines": {
@@ -1575,6 +1576,37 @@
"fastify-plugin": "^5.0.0"
}
},
"node_modules/@grpc/grpc-js": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.0.tgz",
"integrity": "sha512-N8Jx6PaYzcTRNzirReJCtADVoq4z7+1KQ4E70jTg/koQiMoUSN1kbNjPOqpPbhMFhfU1/l7ixspPl8dNY+FoUg==",
"license": "Apache-2.0",
"dependencies": {
"@grpc/proto-loader": "^0.8.0",
"@js-sdsl/ordered-map": "^4.4.2"
},
"engines": {
"node": ">=12.10.0"
}
},
"node_modules/@grpc/proto-loader": {
"version": "0.8.0",
"resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz",
"integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==",
"license": "Apache-2.0",
"dependencies": {
"lodash.camelcase": "^4.3.0",
"long": "^5.0.0",
"protobufjs": "^7.5.3",
"yargs": "^17.7.2"
},
"bin": {
"proto-loader-gen-types": "build/bin/proto-loader-gen-types.js"
},
"engines": {
"node": ">=6"
}
},
"node_modules/@humanfs/core": {
"version": "0.19.1",
"resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
@@ -1660,6 +1692,20 @@
"@influxdata/influxdb-client": "*"
}
},
"node_modules/@influxdata/influxdb3-client": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@influxdata/influxdb3-client/-/influxdb3-client-1.4.0.tgz",
"integrity": "sha512-N07XQxQGyQ8TIscZnjS12ga4Vu2pPtvjzOZSNqeMimyV8VKRM0OEkCH/y2klCeIJkVV+A2/WZ2r4enQa5Z5wjw==",
"license": "MIT",
"dependencies": {
"@grpc/grpc-js": "^1.9.9",
"@protobuf-ts/grpc-transport": "^2.9.1",
"@protobuf-ts/grpcweb-transport": "^2.9.1",
"@protobuf-ts/runtime-rpc": "^2.9.1",
"apache-arrow": "^19.0.0",
"grpc-web": "^1.5.0"
}
},
"node_modules/@isaacs/balanced-match": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
@@ -2333,6 +2379,16 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@js-sdsl/ordered-map": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz",
"integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==",
"license": "MIT",
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/js-sdsl"
}
},
"node_modules/@jsdoc/salty": {
"version": "0.2.9",
"resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.9.tgz",
@@ -2510,6 +2566,108 @@
"cross-spawn": "^7.0.6"
}
},
"node_modules/@protobuf-ts/grpc-transport": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/grpc-transport/-/grpc-transport-2.11.1.tgz",
"integrity": "sha512-l6wrcFffY+tuNnuyrNCkRM8hDIsAZVLA8Mn7PKdVyYxITosYh60qW663p9kL6TWXYuDCL3oxH8ih3vLKTDyhtg==",
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1"
},
"peerDependencies": {
"@grpc/grpc-js": "^1.6.0"
}
},
"node_modules/@protobuf-ts/grpcweb-transport": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/grpcweb-transport/-/grpcweb-transport-2.11.1.tgz",
"integrity": "sha512-1W4utDdvOB+RHMFQ0soL4JdnxjXV+ddeGIUg08DvZrA8Ms6k5NN6GBFU2oHZdTOcJVpPrDJ02RJlqtaoCMNBtw==",
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1"
}
},
"node_modules/@protobuf-ts/runtime": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.11.1.tgz",
"integrity": "sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==",
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@protobuf-ts/runtime-rpc": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/runtime-rpc/-/runtime-rpc-2.11.1.tgz",
"integrity": "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ==",
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1"
}
},
"node_modules/@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/base64": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/codegen": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/eventemitter": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
"integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/fetch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
"integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
"license": "BSD-3-Clause",
"dependencies": {
"@protobufjs/aspromise": "^1.1.1",
"@protobufjs/inquire": "^1.1.0"
}
},
"node_modules/@protobufjs/float": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/inquire": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
"integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/path": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
"license": "BSD-3-Clause"
},
"node_modules/@protobufjs/utf8": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
"integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
"license": "BSD-3-Clause"
},
"node_modules/@sentry-internal/tracing": {
"version": "7.120.3",
"resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.120.3.tgz",
@@ -2645,6 +2803,15 @@
"text-hex": "1.0.x"
}
},
"node_modules/@swc/helpers": {
"version": "0.5.17",
"resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz",
"integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.8.0"
}
},
"node_modules/@tybys/wasm-util": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz",
@@ -2701,6 +2868,18 @@
"@babel/types": "^7.28.2"
}
},
"node_modules/@types/command-line-args": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz",
"integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==",
"license": "MIT"
},
"node_modules/@types/command-line-usage": {
"version": "5.0.4",
"resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz",
"integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==",
"license": "MIT"
},
"node_modules/@types/estree": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
@@ -2808,9 +2987,9 @@
}
},
"node_modules/@types/yargs": {
"version": "17.0.33",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
"integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==",
"version": "17.0.35",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz",
"integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -2825,9 +3004,9 @@
"license": "MIT"
},
"node_modules/@typescript-eslint/types": {
"version": "8.48.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.48.1.tgz",
"integrity": "sha512-+fZ3LZNeiELGmimrujsDCT4CRIbq5oXdHe7chLiW8qzqyPMnn1puNstCrMNVAqwcl2FdIxkuJ4tOs/RFDBVc/Q==",
"version": "8.49.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.49.0.tgz",
"integrity": "sha512-e9k/fneezorUo6WShlQpMxXh8/8wfyc+biu6tnAqA81oWrEic0k21RHzP9uqqpyBBeBKu4T+Bsjy9/b8u7obXQ==",
"dev": true,
"license": "MIT",
"engines": {
@@ -3305,6 +3484,35 @@
"node": ">= 8"
}
},
"node_modules/apache-arrow": {
"version": "19.0.1",
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-19.0.1.tgz",
"integrity": "sha512-APmMLzS4qbTivLrPdQXexGM4JRr+0g62QDaobzEvip/FdQIrv2qLy0mD5Qdmw4buydtVJgbFeKR8f59I6PPGDg==",
"license": "Apache-2.0",
"dependencies": {
"@swc/helpers": "^0.5.11",
"@types/command-line-args": "^5.2.3",
"@types/command-line-usage": "^5.0.4",
"@types/node": "^20.13.0",
"command-line-args": "^6.0.1",
"command-line-usage": "^7.0.1",
"flatbuffers": "^24.3.25",
"json-bignum": "^0.0.3",
"tslib": "^2.6.2"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/apache-arrow/node_modules/@types/node": {
"version": "20.19.17",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.17.tgz",
"integrity": "sha512-gfehUI8N1z92kygssiuWvLiwcbOB3IRktR6hTDgJlXMYh5OvkPSRmgfoBUmfZt+vhwJtX7v1Yw4KvvAf7c5QKQ==",
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/are-docs-informative": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/are-docs-informative/-/are-docs-informative-0.0.2.tgz",
@@ -3324,7 +3532,6 @@
"version": "6.2.2",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz",
"integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=12.17"
@@ -3787,7 +3994,6 @@
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
@@ -3803,7 +4009,6 @@
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz",
"integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==",
"dev": true,
"license": "MIT",
"dependencies": {
"chalk": "^4.1.2"
@@ -3826,9 +4031,9 @@
}
},
"node_modules/ci-info": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.3.0.tgz",
"integrity": "sha512-l+2bNRMiQgcfILUi33labAZYIWlH1kWDp+ecNo5iisRKrbm0xcRyCww71/YU0Fkw0mAFpz9bJayXPjey6vkmaQ==",
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.3.1.tgz",
"integrity": "sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA==",
"dev": true,
"funding": [
{
@@ -3842,9 +4047,9 @@
}
},
"node_modules/cjs-module-lexer": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.1.0.tgz",
"integrity": "sha512-UX0OwmYRYQQetfrLEZeewIFFI+wSTofC+pMBLNuH3RUuu/xzG1oz84UCEDOSoQlN3fZ4+AzmV50ZYvGqkMh9yA==",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.1.1.tgz",
"integrity": "sha512-+CmxIZ/L2vNcEfvNtLdU0ZQ6mbq3FZnwAP2PPTiKP+1QOoKwlKlPgb8UKV0Dds7QVaMnHm+FwSft2VB0s/SLjQ==",
"dev": true,
"license": "MIT"
},
@@ -3852,7 +4057,6 @@
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
"dev": true,
"license": "ISC",
"dependencies": {
"string-width": "^4.2.0",
@@ -3867,7 +4071,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"ansi-styles": "^4.0.0",
@@ -3893,9 +4096,9 @@
}
},
"node_modules/collect-v8-coverage": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz",
"integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==",
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz",
"integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==",
"dev": true,
"license": "MIT"
},
@@ -3985,7 +4188,6 @@
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-6.0.1.tgz",
"integrity": "sha512-Jr3eByUjqyK0qd8W0SGFW1nZwqCaNCtbXjRo2cRJC1OYxWl3MZ5t1US3jq+cO4sPavqgw4l9BMGX0CBe+trepg==",
"dev": true,
"license": "MIT",
"dependencies": {
"array-back": "^6.2.2",
@@ -4009,7 +4211,6 @@
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.3.tgz",
"integrity": "sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"array-back": "^6.2.2",
@@ -4560,7 +4761,6 @@
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
@@ -5323,7 +5523,6 @@
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-5.0.2.tgz",
"integrity": "sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=14"
@@ -5369,6 +5568,12 @@
"node": ">=16"
}
},
"node_modules/flatbuffers": {
"version": "24.12.23",
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.12.23.tgz",
"integrity": "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==",
"license": "Apache-2.0"
},
"node_modules/flatted": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
@@ -5506,7 +5711,6 @@
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"dev": true,
"license": "ISC",
"engines": {
"node": "6.* || 8.* || >= 10.*"
@@ -5712,6 +5916,12 @@
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="
},
"node_modules/grpc-web": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/grpc-web/-/grpc-web-1.5.0.tgz",
"integrity": "sha512-y1tS3BBIoiVSzKTDF3Hm7E8hV2n7YY7pO0Uo7depfWJqKzWE+SKr0jvHNIJsJJYILQlpYShpi/DRJJMbosgDMQ==",
"license": "Apache-2.0"
},
"node_modules/handlebars": {
"version": "4.7.8",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz",
@@ -5737,7 +5947,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true,
"engines": {
"node": ">=8"
}
@@ -6130,9 +6339,9 @@
}
},
"node_modules/istanbul-lib-instrument/node_modules/semver": {
"version": "7.7.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
"integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
"version": "7.7.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
"integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==",
"dev": true,
"license": "ISC",
"bin": {
@@ -6840,9 +7049,9 @@
}
},
"node_modules/jest-snapshot/node_modules/semver": {
"version": "7.7.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
"integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
"version": "7.7.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
"integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==",
"dev": true,
"license": "ISC",
"bin": {
@@ -7155,6 +7364,14 @@
"node": ">=6"
}
},
"node_modules/json-bignum": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
"integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/json-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
@@ -7495,7 +7712,6 @@
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
"dev": true,
"license": "MIT"
},
"node_modules/lodash.clonedeep": {
@@ -7533,6 +7749,12 @@
"node": ">= 12.0.0"
}
},
"node_modules/long": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
"integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
"license": "Apache-2.0"
},
"node_modules/lru-cache": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
@@ -7565,9 +7787,9 @@
}
},
"node_modules/make-dir/node_modules/semver": {
"version": "7.7.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
"integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
"version": "7.7.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
"integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==",
"dev": true,
"license": "ISC",
"bin": {
@@ -7845,9 +8067,9 @@
"license": "MIT"
},
"node_modules/napi-postinstall": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.3.tgz",
"integrity": "sha512-uTp172LLXSxuSYHv/kou+f6KW3SMppU9ivthaVTXian9sOt3XM/zHYHpRZiLgQoxeWfYUnslNWQHF1+G71xcow==",
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz",
"integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==",
"dev": true,
"license": "MIT",
"bin": {
@@ -7942,9 +8164,9 @@
}
},
"node_modules/npm-check-updates": {
"version": "19.1.2",
"resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-19.1.2.tgz",
"integrity": "sha512-FNeFCVgPOj0fz89hOpGtxP2rnnRHR7hD2E8qNU8SMWfkyDZXA/xpgjsL3UMLSo3F/K13QvJDnbxPngulNDDo/g==",
"version": "19.2.0",
"resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-19.2.0.tgz",
"integrity": "sha512-XSIuL0FNgzXPDZa4lje7+OwHjiyEt84qQm6QMsQRbixNY5EHEM9nhgOjxjlK9jIbN+ysvSqOV8DKNS0zydwbdg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
@@ -8531,6 +8753,30 @@
"node": "^16 || ^18 || >=20"
}
},
"node_modules/protobufjs": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz",
"integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==",
"hasInstallScript": true,
"license": "BSD-3-Clause",
"dependencies": {
"@protobufjs/aspromise": "^1.1.2",
"@protobufjs/base64": "^1.1.2",
"@protobufjs/codegen": "^2.0.4",
"@protobufjs/eventemitter": "^1.1.0",
"@protobufjs/fetch": "^1.1.0",
"@protobufjs/float": "^1.0.2",
"@protobufjs/inquire": "^1.1.0",
"@protobufjs/path": "^1.1.2",
"@protobufjs/pool": "^1.1.0",
"@protobufjs/utf8": "^1.1.0",
"@types/node": ">=13.7.0",
"long": "^5.0.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
@@ -8782,7 +9028,6 @@
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@@ -9112,9 +9357,9 @@
}
},
"node_modules/snyk": {
"version": "1.1301.0",
"resolved": "https://registry.npmjs.org/snyk/-/snyk-1.1301.0.tgz",
"integrity": "sha512-kTb8F9L1PlI3nYWlp60wnSGWGmcRs6bBtSBl9s8YYhAiFZNseIZfXolQXBSCaya5QlcxzfH1pb4aqCNMbi0tgg==",
"version": "1.1301.1",
"resolved": "https://registry.npmjs.org/snyk/-/snyk-1.1301.1.tgz",
"integrity": "sha512-EYgBCi0+diYgqiibdwyUowBCcowKDGcfqXkZoBWG3qNdcLVZqjq7ogOEKwOcbNern7doDzm2TSZtbRCu+SpVMQ==",
"dev": true,
"hasInstallScript": true,
"license": "Apache-2.0",
@@ -9483,7 +9728,6 @@
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"dependencies": {
"has-flag": "^4.0.0"
},
@@ -9508,9 +9752,9 @@
}
},
"node_modules/systeminformation": {
"version": "5.27.11",
"resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.27.11.tgz",
"integrity": "sha512-K3Lto/2m3K2twmKHdgx5B+0in9qhXK4YnoT9rIlgwN/4v7OV5c8IjbeAUkuky/6VzCQC7iKCAqi8rZathCdjHg==",
"version": "5.27.13",
"resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.27.13.tgz",
"integrity": "sha512-geeE/7eNDoOhdc9j+qCsLlwbcyh0HnqhOZzmfNK4WBioWGUZbhwYrg+YZsZ3UJh4tmybQsnDuqzr3UoumMifew==",
"license": "MIT",
"os": [
"darwin",
@@ -9553,7 +9797,6 @@
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz",
"integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==",
"dev": true,
"license": "MIT",
"dependencies": {
"array-back": "^6.2.2",
@@ -9819,7 +10062,6 @@
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz",
"integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=12.17"
@@ -9846,9 +10088,9 @@
"license": "MIT"
},
"node_modules/ua-parser-js": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-2.0.6.tgz",
"integrity": "sha512-EmaxXfltJaDW75SokrY4/lXMrVyXomE/0FpIIqP2Ctic93gK7rlme55Cwkz8l3YZ6gqf94fCU7AnIkidd/KXPg==",
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-2.0.7.tgz",
"integrity": "sha512-CFdHVHr+6YfbktNZegH3qbYvYgC7nRNEUm2tk7nSFXSODUu4tDBpaFpP1jdXBUOKKwapVlWRfTtS8bCPzsQ47w==",
"funding": [
{
"type": "opencollective",
@@ -10181,7 +10423,6 @@
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz",
"integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=12.17"
@@ -10384,7 +10625,6 @@
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
"dev": true,
"license": "ISC",
"engines": {
"node": ">=10"
@@ -10400,7 +10640,6 @@
"version": "17.7.2",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
"integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
"dev": true,
"license": "MIT",
"dependencies": {
"cliui": "^8.0.1",
@@ -10419,7 +10658,6 @@
"version": "21.1.1",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
"dev": true,
"license": "ISC",
"engines": {
"node": ">=12"

8
package.json
View File

@@ -1,10 +1,11 @@
{
"name": "butler-sos",
"version": "14.0.0",
"description": "Butler SenseOps Stats (\"Butler SOS\") is a tool that publishes operational Qlik Sense metrics to Influxdb, Prometheus, New Relic and MQTT.",
"description": "Butler SenseOps Stats (\"Butler SOS\") is a tool that publishes operational Qlik Sense metrics to InfluxDB (v1, v2, v3), Prometheus, New Relic and MQTT.",
"main": "butler-sos.js",
"scripts": {
"build": "npx jsdoc-to-markdown 'src/**/*.js' > docs/src-code-overview.md",
"build:docker": "docker build -t butler-sos:latest .",
"butler-sos": "node src/butler-sos.js",
"jest": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js",
"test": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js && snyk test && npm run format",
@@ -52,6 +53,7 @@
"@fastify/static": "^8.3.0",
"@influxdata/influxdb-client": "^1.35.0",
"@influxdata/influxdb-client-apis": "^1.35.0",
"@influxdata/influxdb3-client": "^1.4.0",
"ajv": "^8.17.1",
"ajv-keywords": "^5.1.0",
"async-mutex": "^0.5.0",
@@ -81,7 +83,7 @@
"devDependencies": {
"@babel/eslint-parser": "^7.28.5",
"@babel/plugin-syntax-import-assertions": "^7.27.1",
"@eslint/js": "^9.39.1",
"@eslint/js": "^9.39.2",
"audit-ci": "^7.1.0",
"esbuild": "^0.27.1",
"eslint-config-prettier": "^10.1.8",
@@ -89,7 +91,7 @@
"eslint-plugin-jsdoc": "^61.5.0",
"eslint-plugin-prettier": "^5.5.4",
"globals": "^16.5.0",
"jest": "^30.1.3",
"jest": "^30.2.0",
"jsdoc-to-markdown": "^9.1.3",
"license-checker-rseidelsohn": "^4.4.2",
"lockfile-lint": "^4.14.1",

5
src/butler-sos.js
View File

@@ -24,7 +24,8 @@ import { setupAnonUsageReportTimer } from './lib/telemetry.js';
import { setupPromClient } from './lib/prom-client.js';
import { setupConfigVisServer } from './lib/config-visualise.js';
import { setupUdpEventsStorage } from './lib/udp-event.js';
import { setupUdpQueueMetricsStorage } from './lib/post-to-influxdb.js';
import { setupUdpQueueMetricsStorage } from './lib/influxdb/index.js';
import { logError } from './lib/log-error.js';
// Suppress experimental warnings
// https://stackoverflow.com/questions/55778283/how-to-disable-warnings-when-node-is-launched-via-a-global-shell-script
@@ -204,7 +205,7 @@ async function mainScript() {
);
}
} catch (err) {
globals.logger.error(`CONFIG: Error initiating host info: ${globals.getErrorMessage(err)}`);
logError('CONFIG: Error initiating host info', err);
}
// Set up UDP handler for user activity/events

74
src/config/production_template.yaml
View File

@@ -63,12 +63,12 @@ Butler-SOS:
enable: true # Should Butler SOS' uptime (how long since it was started) be sent to New Relic?
attribute:
static: # Static attributes/dimensions to attach to the data sent to New Relic.
# - name: metricType
# value: butler-sos-uptime
# - name: qs_service
# value: butler-sos
# - name: qs_environment
# value: prod
- name: metricType
value: butler-sos-uptime
- name: qs_service
value: butler-sos
- name: qs_env
value: dev
dynamic:
butlerVersion:
enable: true # Should the Butler SOS version be included in the data sent to New Relic?
@@ -97,10 +97,8 @@ Butler-SOS:
influxdb:
measurementName: event_count # Name of the InfluxDB measurement where event count is stored
tags: # Tags are added to the data before it's stored in InfluxDB
# - name: env
# value: DEV
# - name: foo
# value: bar
- name: qs_env
value: dev
rejectedEventCount: # Rejected events are events that are received from Sense, that are correctly formatted,
# but that are rejected by Butler SOS based on the configuration in this file.
# An example of a rejected event is a performance log event that is filtered out by Butler SOS.
@@ -137,13 +135,11 @@ Butler-SOS:
writeFrequency: 20000 # How often to write metrics, milliseconds (default: 20000)
measurementName: user_events_queue # InfluxDB measurement name (default: user_events_queue)
tags: # Optional tags added to queue metrics
# - name: env
# value: prod
- name: qs_env
value: dev
tags: # Tags are added to the data before it's stored in InfluxDB
# - name: env
# value: DEV
# - name: foo
# value: bar
- name: qs_env
value: dev
sendToMQTT:
enable: false # Set to true if user events should be forwarded as MQTT messages
postTo: # Control when and to which MQTT topics messages are sent
@@ -193,13 +189,11 @@ Butler-SOS:
writeFrequency: 20000 # How often to write metrics, milliseconds (default: 20000)
measurementName: log_events_queue # InfluxDB measurement name (default: log_events_queue)
tags: # Optional tags added to queue metrics
# - name: env
# value: prod
- name: qs_env
value: dev
tags:
# - name: env
# value: DEV
# - name: foo
# value: bar
- name: qs_env
value: dev
source:
engine:
enable: false # Should log events from the engine service be handled?
@@ -283,10 +277,8 @@ Butler-SOS:
trackRejectedEvents:
enable: false # Should events that are rejected by the app performance monitor be tracked?
tags: # Tags are added to the data before it's stored in InfluxDB
# - name: env
# value: DEV
# - name: foo
# value: bar
- name: qs_env
value: dev
monitorFilter: # What objects should be monitored? Entire apps or just specific object(s) within some specific app(s)?
# Two kinds of monitoring can be done:
# 1) Monitor all apps, except those listed for exclusion. This is defined in the allApps section.
@@ -438,10 +430,10 @@ Butler-SOS:
# value: Header value
attribute:
static: # Static attributes/dimensions to attach to the events sent to New Relic.
# - name: service
# value: butler-sos
# - name: environment
# value: prod
- name: qs_env
value: dev
- name: service
value: butler-sos
dynamic:
butlerSosVersion:
enable: true # Should the Butler SOS version be included in the events sent to New Relic?
@@ -492,10 +484,10 @@ Butler-SOS:
enable: true
attribute:
static: # Static attributes/dimensions to attach to the data sent to New Relic.
# - name: service
# value: butler-sos
# - name: environment
# value: prod
- name: qs_env
value: dev
- name: service
value: butler-sos
dynamic:
butlerSosVersion:
enable: true # Should the Butler SOS version be included in the data sent to New Relic?
@@ -510,10 +502,20 @@ Butler-SOS:
# Influx db config parameters
influxdbConfig:
enable: true
# Feature flag to enable refactored InfluxDB code (recommended for better maintainability)
# Set to true to use the new modular implementation, false for legacy code
useRefactoredCode: false
# Items below are mandatory if influxdbConfig.enable=true
host: influxdb.mycompany.com # InfluxDB host, hostname, FQDN or IP address
port: 8086 # Port where InfluxDBdb is listening, usually 8086
version: 1 # Is the InfluxDB instance version 1.x or 2.x? Valid values are 1 or 2
version: 1 # Is the InfluxDB instance version 1.x or 2.x? Valid values are 1, 2, or 3
v3Config: # Settings for InfluxDB v3.x only, i.e. Butler-SOS.influxdbConfig.version=3
database: mydatabase
description: Butler SOS metrics
token: mytoken
retentionDuration: 10d
timeout: 10000 # Optional: Socket timeout in milliseconds (default: 10000)
queryTimeout: 60000 # Optional: Query timeout in milliseconds (default: 60000)
v2Config: # Settings for InfluxDB v2.x only, i.e. Butler-SOS.influxdbConfig.version=2
org: myorg
bucket: mybucket
@@ -525,7 +527,7 @@ Butler-SOS:
enable: false # Does influxdb instance require authentication (true/false)?
username: <username> # Username for Influxdb authentication. Mandatory if auth.enable=true
password: <password> # Password for Influxdb authentication. Mandatory if auth.enable=true
dbName: SenseOps
dbName: senseops
# Default retention policy that should be created in InfluxDB when Butler SOS creates a new database there.
# Any data older than retention policy threshold will be purged from InfluxDB.
retentionPolicy:

168
src/globals.js
View File

@@ -8,16 +8,39 @@ import winston from 'winston';
import 'winston-daily-rotate-file';
import si from 'systeminformation';
import { readFileSync } from 'fs';
import Influx from 'influx';
import { Command, Option } from 'commander';
import { InfluxDB, HttpError, DEFAULT_WriteOptions } from '@influxdata/influxdb-client';
// Note on InfluxDB libraries:
// v1 client library: https://github.com/node-influx/node-influx
// v2 client library: https://influxdata.github.io/influxdb-client-js/
// v3 client library: https://github.com/InfluxCommunity/influxdb3-js
// v1
import Influx from 'influx';
// v2
// Import InfluxDB as const InfluxDB2 to avoid name clash with Influx from 'influx' above
import {
InfluxDB as InfluxDB2,
HttpError,
DEFAULT_WriteOptions,
} from '@influxdata/influxdb-client';
import { OrgsAPI, BucketsAPI } from '@influxdata/influxdb-client-apis';
// v3
import {
InfluxDBClient as InfluxDBClient3,
Point as Point3,
setLogger as setInfluxV3Logger,
} from '@influxdata/influxdb3-client';
import { fileURLToPath } from 'url';
import sea from './lib/sea-wrapper.js';
import { getServerTags } from './lib/servertags.js';
import { UdpEvents } from './lib/udp-event.js';
import { UdpQueueManager } from './lib/udp-queue-manager.js';
import { ErrorTracker, setupErrorCounterReset } from './lib/error-tracker.js';
import { verifyConfigFileSchema, verifyAppConfig } from './lib/config-file-verify.js';
let instance = null;
@@ -135,9 +158,6 @@ class Settings {
this.appVersion = appVersion;
// Make copy of influxdb client
const InfluxDB2 = InfluxDB;
// Command line parameters
const program = new Command();
program
@@ -574,6 +594,14 @@ Configuration File:
this.rejectedEvents = null;
}
// ------------------------------------
// Track API error counts
this.errorTracker = new ErrorTracker(this.logger);
this.logger.info('ERROR TRACKER: Initialized error tracking with daily UTC reset');
// Setup midnight UTC reset timer for error counters
setupErrorCounterReset();
// ------------------------------------
// Get info on what servers to monitor
this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers');
@@ -701,6 +729,13 @@ Configuration File:
this.logger.info(
`CONFIG: Influxdb retention policy duration: ${this.config.get('Butler-SOS.influxdbConfig.v2Config.retentionDuration')}`
);
} else if (this.config.get('Butler-SOS.influxdbConfig.version') === 3) {
this.logger.info(
`CONFIG: Influxdb database name: ${this.config.get('Butler-SOS.influxdbConfig.v3Config.database')}`
);
this.logger.info(
`CONFIG: Influxdb retention policy duration: ${this.config.get('Butler-SOS.influxdbConfig.v3Config.retentionDuration')}`
);
} else {
this.logger.error(
`CONFIG: Influxdb version ${this.config.get('Butler-SOS.influxdbConfig.version')} is not supported!`
@@ -870,6 +905,86 @@ Configuration File:
);
this.logger.error(`INFLUXDB2 INIT: Exiting.`);
}
} else if (this.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Configure InfluxDB v3 client logger to suppress internal error messages
// The retry logic in Butler SOS provides better error handling
setInfluxV3Logger({
error: () => {
// Suppress InfluxDB client library error messages
// Butler SOS retry logic and logging handles errors
},
warn: () => {
// Suppress InfluxDB client library warning messages
},
});
// Set up Influxdb v3 client (uses its own client library, NOT same as v2)
const hostName = this.config.get('Butler-SOS.influxdbConfig.host');
const port = this.config.get('Butler-SOS.influxdbConfig.port');
const host = `http://${hostName}:${port}`;
const token = this.config.get('Butler-SOS.influxdbConfig.v3Config.token');
const database = this.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Get timeout settings with defaults
const timeout = this.config.has('Butler-SOS.influxdbConfig.v3Config.timeout')
? this.config.get('Butler-SOS.influxdbConfig.v3Config.timeout')
: 10000; // Default 10 seconds for socket timeout
const queryTimeout = this.config.has(
'Butler-SOS.influxdbConfig.v3Config.queryTimeout'
)
? this.config.get('Butler-SOS.influxdbConfig.v3Config.queryTimeout')
: 60000; // Default 60 seconds for gRPC query timeout
try {
this.influx = new InfluxDBClient3({
host,
token,
database,
timeout,
queryTimeout,
});
// Test connection by executing a simple query
this.logger.info(`INFLUXDB3 INIT: Testing connection to InfluxDB v3...`);
try {
// Execute a simple query to test the connection
const testQuery = `SELECT 1 as test LIMIT 1`;
const queryResult = this.influx.query(testQuery, database);
// Try to get first result (this will throw if connection fails)
const iterator = queryResult[Symbol.asyncIterator]();
await iterator.next();
// Connection successful - log details
const tokenPreview = token.substring(0, 4) + '***';
this.logger.info(`INFLUXDB3 INIT: Connection successful!`);
this.logger.info(`INFLUXDB3 INIT: Host: ${hostName}`);
this.logger.info(`INFLUXDB3 INIT: Port: ${port}`);
this.logger.info(`INFLUXDB3 INIT: Database: ${database}`);
this.logger.info(`INFLUXDB3 INIT: Token: ${tokenPreview}`);
this.logger.info(`INFLUXDB3 INIT: Socket timeout: ${timeout}ms`);
this.logger.info(`INFLUXDB3 INIT: Query timeout: ${queryTimeout}ms`);
} catch (testErr) {
this.logger.warn(
`INFLUXDB3 INIT: Could not test connection (this may be normal): ${this.getErrorMessage(testErr)}`
);
// Still log the configuration
const tokenPreview = token.substring(0, 4) + '***';
this.logger.info(`INFLUXDB3 INIT: Client created with:`);
this.logger.info(`INFLUXDB3 INIT: Host: ${hostName}`);
this.logger.info(`INFLUXDB3 INIT: Port: ${port}`);
this.logger.info(`INFLUXDB3 INIT: Database: ${database}`);
this.logger.info(`INFLUXDB3 INIT: Token: ${tokenPreview}`);
this.logger.info(`INFLUXDB3 INIT: Socket timeout: ${timeout}ms`);
this.logger.info(`INFLUXDB3 INIT: Query timeout: ${queryTimeout}ms`);
}
} catch (err) {
this.logger.error(
`INFLUXDB3 INIT: Error creating InfluxDB 3 client: ${this.getErrorMessage(err)}`
);
this.logger.error(`INFLUXDB3 INIT: Exiting.`);
}
} else {
this.logger.error(
`CONFIG: Influxdb version ${this.config.get('Butler-SOS.influxdbConfig.version')} is not supported!`
@@ -1090,8 +1205,8 @@ Configuration File:
maxRetries: 2, // do not retry writes
// ... there are more write options that can be customized, see
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html
// https://influxdata.github.io/influxdb-client-js/interfaces/_influxdata_influxdb-client.WriteOptions.html
// https://influxdata.github.io/influxdb-client-js/interfaces/_influxdata_influxdb-client.WriteRetryOptions.html
};
try {
@@ -1114,6 +1229,45 @@ Configuration File:
}
});
}
} else if (this.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Get config
const databaseName = this.config.get('Butler-SOS.influxdbConfig.v3Config.database');
const description = this.config.get('Butler-SOS.influxdbConfig.v3Config.description');
const token = this.config.get('Butler-SOS.influxdbConfig.v3Config.token');
const retentionDuration = this.config.get(
'Butler-SOS.influxdbConfig.v3Config.retentionDuration'
);
if (
this.influx &&
this.config.get('Butler-SOS.influxdbConfig.enable') === true &&
databaseName?.length > 0 &&
token?.length > 0 &&
retentionDuration?.length > 0
) {
enableInfluxdb = true;
}
if (enableInfluxdb) {
// For InfluxDB v3, we use client.write() directly (no getWriteApi method in v3)
this.logger.info(`INFLUXDB3: Using database "${databaseName}"`);
// For v3, we store the client itself and call write() directly
// The influxWriteApi array will contain objects with client and database info
this.serverList.forEach((server) => {
// Get per-server tags
const tags = getServerTags(this.logger, server);
// Store client info and tags for this server
// v3 uses client.write() directly, not getWriteApi()
this.influxWriteApi.push({
serverName: server.serverName,
writeAPI: this.influx, // Store the client itself
database: databaseName,
defaultTags: tags, // Store tags for later use
});
});
}
}
}

7
src/lib/__tests__/appnamesextract.test.js
View File

@@ -129,9 +129,12 @@ describe('appnamesextract', () => {
expect(qrsInteract).toHaveBeenCalledWith(expect.any(Object));
expect(mockGet).toHaveBeenCalledWith('app');
// Verify error logging
// Verify error logging - logError creates TWO log calls: message + stack trace
expect(globals.logger.error).toHaveBeenCalledWith(
'APP NAMES: Error getting app names: Error: QRS API Error'
'APP NAMES: Error getting app names: QRS API Error'
);
expect(globals.logger.error).toHaveBeenCalledWith(
expect.stringContaining('Stack trace: Error: QRS API Error')
);
});
});

5
src/lib/__tests__/file-prep.test.js
View File

@@ -41,9 +41,8 @@ const handlebars = (await import('handlebars')).default;
const globals = (await import('../../globals.js')).default;
// Import the module under test
const { prepareFile, compileTemplate, getFileContent, getMimeType } = await import(
'../file-prep.js'
);
const { prepareFile, compileTemplate, getFileContent, getMimeType } =
await import('../file-prep.js');
describe('file-prep', () => {
beforeEach(() => {

3
src/lib/__tests__/healthmetrics.test.js
View File

@@ -23,6 +23,9 @@ jest.unstable_mockModule('../../globals.js', () => ({
verbose: jest.fn(),
debug: jest.fn(),
},
errorTracker: {
incrementError: jest.fn(),
},
config: {
get: jest.fn(),
has: jest.fn(),

171
src/lib/__tests__/post-to-influxdb.test.js
View File

@@ -1,6 +1,6 @@
import { jest, describe, test, expect, beforeEach, afterEach } from '@jest/globals';
// Mock the InfluxDB client
// Mock the InfluxDB v2 client
jest.unstable_mockModule('@influxdata/influxdb-client', () => ({
Point: jest.fn().mockImplementation(() => ({
tag: jest.fn().mockReturnThis(),
@@ -13,6 +13,19 @@ jest.unstable_mockModule('@influxdata/influxdb-client', () => ({
})),
}));
// Mock the InfluxDB v3 client
jest.unstable_mockModule('@influxdata/influxdb3-client', () => ({
Point: jest.fn().mockImplementation(() => ({
setTag: jest.fn().mockReturnThis(),
setFloatField: jest.fn().mockReturnThis(),
setIntegerField: jest.fn().mockReturnThis(),
setStringField: jest.fn().mockReturnThis(),
setBooleanField: jest.fn().mockReturnThis(),
timestamp: jest.fn().mockReturnThis(),
toLineProtocol: jest.fn().mockReturnValue('mock-line-protocol'),
})),
}));
// Mock globals
jest.unstable_mockModule('../../globals.js', () => ({
default: {
@@ -232,6 +245,108 @@ describe('post-to-influxdb', () => {
);
});
test('should store log events to InfluxDB (InfluxDB v3)', async () => {
// Setup
globals.config.get = jest.fn((key) => {
if (key === 'Butler-SOS.influxdbConfig.version') return 3;
if (key === 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName') {
return 'events_log';
}
if (key === 'Butler-SOS.influxdbConfig.v3Config.database') return 'test-database';
return undefined;
});
globals.config.has = jest.fn().mockReturnValue(false);
const mockLogEvents = [
{
source: 'test-source',
host: 'test-host',
subsystem: 'test-subsystem',
counter: 5,
timestamp: '2023-01-01T00:00:00.000Z',
message: 'test message',
appName: 'test-app',
appId: 'test-app-id',
executionId: 'test-exec',
command: 'test-cmd',
resultCode: '200',
origin: 'test-origin',
context: 'test-context',
sessionId: 'test-session',
rawEvent: 'test-raw',
level: 'INFO',
log_row: '1',
},
];
globals.udpEvents = {
getLogEvents: jest.fn().mockResolvedValue(mockLogEvents),
getUserEvents: jest.fn().mockResolvedValue([]),
};
globals.options = { instanceTag: 'test-instance' };
// Mock v3 client write method
globals.influx.write = jest.fn().mockResolvedValue(undefined);
// Execute
await influxdb.storeEventCountInfluxDB();
// Verify
expect(globals.influx.write).toHaveBeenCalled();
expect(globals.logger.verbose).toHaveBeenCalledWith(
expect.stringContaining(
'EVENT COUNT INFLUXDB: Sent Butler SOS event count data to InfluxDB'
)
);
});
test('should store user events to InfluxDB (InfluxDB v3)', async () => {
// Setup
globals.config.get = jest.fn((key) => {
if (key === 'Butler-SOS.influxdbConfig.version') return 3;
if (key === 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName') {
return 'events_user';
}
if (key === 'Butler-SOS.influxdbConfig.v3Config.database') return 'test-database';
return undefined;
});
globals.config.has = jest.fn().mockReturnValue(false);
const mockUserEvents = [
{
source: 'test-source',
host: 'test-host',
subsystem: 'test-subsystem',
counter: 3,
timestamp: '2023-01-01T00:00:00.000Z',
message: 'test message',
appName: 'test-app',
appId: 'test-app-id',
executionId: 'test-exec',
command: 'test-cmd',
resultCode: '200',
origin: 'test-origin',
context: 'test-context',
sessionId: 'test-session',
rawEvent: 'test-raw',
},
];
globals.udpEvents = {
getLogEvents: jest.fn().mockResolvedValue([]),
getUserEvents: jest.fn().mockResolvedValue(mockUserEvents),
};
globals.options = { instanceTag: 'test-instance' };
// Mock v3 client write method
globals.influx.write = jest.fn().mockResolvedValue(undefined);
// Execute
await influxdb.storeEventCountInfluxDB();
// Verify
expect(globals.influx.write).toHaveBeenCalled();
expect(globals.logger.verbose).toHaveBeenCalledWith(
expect.stringContaining(
'EVENT COUNT INFLUXDB: Sent Butler SOS event count data to InfluxDB'
)
);
});
test('should handle errors gracefully (InfluxDB v1)', async () => {
// Setup
globals.config.get = jest.fn((key) => {
@@ -250,12 +365,15 @@ describe('post-to-influxdb', () => {
// Execute
await influxdb.storeEventCountInfluxDB();
// Verify
// Verify - logError creates TWO log calls: message + stack trace
expect(globals.logger.error).toHaveBeenCalledWith(
expect.stringContaining(
'EVENT COUNT INFLUXDB: Error saving data to InfluxDB v1! Error: Test error'
'EVENT COUNT INFLUXDB: Error saving data to InfluxDB v1!: Test error'
)
);
expect(globals.logger.error).toHaveBeenCalledWith(
expect.stringContaining('Stack trace: Error: Test error')
);
});
test('should handle errors gracefully (InfluxDB v2)', async () => {
@@ -601,6 +719,53 @@ describe('post-to-influxdb', () => {
expect(globals.influxWriteApi[0].writeAPI.writePoints).toHaveBeenCalled();
});
test('should post health metrics to InfluxDB v3', async () => {
globals.config.get = jest.fn((key) => {
if (key === 'Butler-SOS.influxdbConfig.version') return 3;
if (key === 'Butler-SOS.influxdbConfig.includeFields.activeDocs') return false;
if (key === 'Butler-SOS.influxdbConfig.includeFields.loadedDocs') return false;
if (key === 'Butler-SOS.influxdbConfig.includeFields.inMemoryDocs') return false;
if (key === 'Butler-SOS.appNames.enableAppNameExtract') return false;
if (key === 'Butler-SOS.influxdbConfig.v3Config.database') return 'test-database';
return undefined;
});
// Mock v3 client write method
const mockWrite = jest.fn().mockResolvedValue(undefined);
globals.influxWriteApi = [
{
serverName: 'testserver',
writeAPI: mockWrite,
database: 'test-database',
},
];
globals.influx = {
write: mockWrite,
};
const serverName = 'testserver';
const host = 'testhost';
const serverTags = { host: 'testhost', server_name: 'testserver' };
const healthBody = {
version: '1.0.0',
started: '20220801T121212.000Z',
apps: {
active_docs: [],
loaded_docs: [],
in_memory_docs: [],
calls: 100,
selections: 50,
},
cache: { added: 0, hits: 10, lookups: 15, replaced: 2, bytes_added: 1000 },
cpu: { total: 25 },
mem: { committed: 1000, allocated: 800, free: 200 },
session: { active: 5, total: 10 },
users: { active: 3, total: 8 },
};
await influxdb.postHealthMetricsToInfluxdb(serverName, host, healthBody, serverTags);
expect(mockWrite).toHaveBeenCalled();
});
});
describe('postProxySessionsToInfluxdb', () => {

25
src/lib/__tests__/post-to-mqtt.test.js
View File

@@ -8,6 +8,9 @@ jest.unstable_mockModule('../../globals.js', () => ({
debug: jest.fn(),
verbose: jest.fn(),
},
errorTracker: {
incrementError: jest.fn(),
},
mqttClient: {
publish: jest.fn(),
},
@@ -19,13 +22,20 @@ jest.unstable_mockModule('../../globals.js', () => ({
}));
const globals = (await import('../../globals.js')).default;
// Mock log-error module
const mockLogError = jest.fn();
jest.unstable_mockModule('../log-error.js', () => ({
logError: mockLogError,
}));
// Import the module under test
const { postHealthToMQTT, postUserSessionsToMQTT, postUserEventToMQTT } = await import(
'../post-to-mqtt.js'
);
const { postHealthToMQTT, postUserSessionsToMQTT, postUserEventToMQTT } =
await import('../post-to-mqtt.js');
describe('post-to-mqtt', () => {
beforeEach(() => {
// Reset all mocks before each test
jest.clearAllMocks();
// Setup default config values
globals.config.get.mockImplementation((path) => {
if (path === 'Butler-SOS.mqttConfig.baseTopic') {
@@ -497,7 +507,7 @@ describe('post-to-mqtt', () => {
);
});
test('should handle errors during publishing', () => {
test('should handle errors during publishing', async () => {
// Force an error by making the MQTT client throw
globals.mqttClient.publish.mockImplementation(() => {
throw new Error('MQTT publish error');
@@ -516,11 +526,12 @@ describe('post-to-mqtt', () => {
};
// Call the function being tested
postUserEventToMQTT(userEvent);
await postUserEventToMQTT(userEvent);
// Verify error was logged
expect(globals.logger.error).toHaveBeenCalledWith(
expect.stringContaining('USER EVENT MQTT: Failed posting message to MQTT')
expect(mockLogError).toHaveBeenCalledWith(
expect.stringContaining('USER EVENT MQTT: Failed posting message to MQTT'),
expect.any(Error)
);
});
});

3
src/lib/__tests__/post-to-new-relic.test.js
View File

@@ -39,6 +39,9 @@ jest.unstable_mockModule('../../globals.js', () => ({
debug: jest.fn(),
error: jest.fn(),
},
errorTracker: {
incrementError: jest.fn(),
},
config: {
get: jest.fn().mockImplementation((path) => {
if (path === 'Butler-SOS.newRelic.enable') return true;

10
src/lib/__tests__/proxysessionmetrics.test.js
View File

@@ -52,6 +52,9 @@ jest.unstable_mockModule('../../globals.js', () => ({
debug: jest.fn(),
error: jest.fn(),
},
errorTracker: {
incrementError: jest.fn(),
},
config: {
get: jest.fn().mockImplementation((path) => {
if (path === 'Butler-SOS.cert.clientCert') return '/path/to/cert.pem';
@@ -88,7 +91,7 @@ jest.unstable_mockModule('../../globals.js', () => ({
// Mock dependent modules
const mockPostProxySessionsToInfluxdb = jest.fn().mockResolvedValue();
jest.unstable_mockModule('../post-to-influxdb.js', () => ({
jest.unstable_mockModule('../influxdb/index.js', () => ({
postProxySessionsToInfluxdb: mockPostProxySessionsToInfluxdb,
}));
@@ -116,9 +119,8 @@ jest.unstable_mockModule('../prom-client.js', () => ({
}));
// Import the module under test
const { setupUserSessionsTimer, getProxySessionStatsFromSense } = await import(
'../proxysessionmetrics.js'
);
const { setupUserSessionsTimer, getProxySessionStatsFromSense } =
await import('../proxysessionmetrics.js');
describe('proxysessionmetrics', () => {
let proxysessionmetrics;

5
src/lib/__tests__/sea-certificate-loading.test.js
View File

@@ -28,9 +28,8 @@ const fs = (await import('fs')).default;
const globals = (await import('../../globals.js')).default;
// Import modules under test
const { getCertificates: getCertificatesUtil, createCertificateOptions } = await import(
'../cert-utils.js'
);
const { getCertificates: getCertificatesUtil, createCertificateOptions } =
await import('../cert-utils.js');
describe('Certificate loading', () => {
const mockCertificateOptions = {

13
src/lib/appnamesextract.js
View File

@@ -3,6 +3,7 @@ import qrsInteract from 'qrs-interact';
import clonedeep from 'lodash.clonedeep';
import globals from '../globals.js';
import { logError } from './log-error.js';
/**
* Retrieves application names from the Qlik Repository Service (QRS) API.
@@ -56,11 +57,19 @@ export function getAppNames() {
globals.logger.verbose('APP NAMES: Done getting app names from repository db');
})
.catch((err) => {
// Track error count
const hostname = globals.config.get('Butler-SOS.appNames.hostIP');
globals.errorTracker.incrementError('APP_NAMES_EXTRACT', hostname || '');
// Return error msg
globals.logger.error(`APP NAMES: Error getting app names: ${err}`);
logError('APP NAMES: Error getting app names', err);
});
} catch (err) {
globals.globals.logger.error(`APP NAMES: ${err}`);
// Track error count
const hostname = globals.config.get('Butler-SOS.appNames.hostIP');
globals.errorTracker.incrementError('APP_NAMES_EXTRACT', hostname || '');
logError('APP NAMES', err);
}
}

4
src/lib/config-file-verify.js
View File

@@ -169,10 +169,10 @@ export async function verifyAppConfig(cfg) {
// Verify values of specific config entries
// If InfluxDB is enabled, check if the version is valid
// Valid values: 1 and 2
// Valid values: 1, 2, and 3
if (cfg.get('Butler-SOS.influxdbConfig.enable') === true) {
const influxdbVersion = cfg.get('Butler-SOS.influxdbConfig.version');
if (influxdbVersion !== 1 && influxdbVersion !== 2) {
if (influxdbVersion !== 1 && influxdbVersion !== 2 && influxdbVersion !== 3) {
console.error(
`VERIFY CONFIG FILE ERROR: Butler-SOS.influxdbConfig.enable (=InfluxDB version) ${influxdbVersion} is invalid. Exiting.`
);

24
src/lib/config-schemas/destinations.js
View File

@@ -310,12 +310,36 @@ export const destinationsSchema = {
type: 'object',
properties: {
enable: { type: 'boolean' },
useRefactoredCode: { type: 'boolean' },
host: {
type: 'string',
format: 'hostname',
},
port: { type: 'number' },
version: { type: 'number' },
v3Config: {
type: 'object',
properties: {
database: { type: 'string' },
description: { type: 'string' },
token: { type: 'string' },
retentionDuration: { type: 'string' },
timeout: {
type: 'number',
description: 'Socket timeout for write operations in milliseconds',
default: 10000,
minimum: 1000,
},
queryTimeout: {
type: 'number',
description: 'gRPC timeout for query operations in milliseconds',
default: 60000,
minimum: 1000,
},
},
required: ['database', 'description', 'token', 'retentionDuration'],
additionalProperties: false,
},
v2Config: {
type: 'object',
properties: {

9
src/lib/config-visualise.js
View File

@@ -8,6 +8,7 @@ import * as yaml from 'js-yaml';
import globals from '../globals.js';
import configObfuscate from './config-obfuscate.js';
import { prepareFile, compileTemplate } from './file-prep.js';
import { logError } from './log-error.js';
/**
* Serves the custom 404 error page
@@ -46,7 +47,7 @@ async function serve404Page(request, reply) {
// Send 404 response with custom page
reply.code(404).header('Content-Type', 'text/html; charset=utf-8').send(renderedHtml);
} catch (err) {
globals.logger.error(`CONFIG VIS: Error serving 404 page: ${err.message}`);
logError('CONFIG VIS: Error serving 404 page', err);
reply.code(404).send({ error: 'Page not found' });
}
}
@@ -184,7 +185,7 @@ export async function setupConfigVisServer(logger, config) {
`CONFIG VIS: Directory contents of "${STATIC_PATH}": ${dirContents}`
);
} catch (err) {
globals.logger.error(`CONFIG VIS: Error reading static directory: ${err.message}`);
logError('CONFIG VIS: Error reading static directory', err);
}
const htmlDir = path.resolve(STATIC_PATH, 'configvis');
@@ -253,7 +254,7 @@ export async function setupConfigVisServer(logger, config) {
.header('Content-Type', 'text/html; charset=utf-8')
.send(renderedText);
} catch (err) {
globals.logger.error(`CONFIG VIS: Error serving home page: ${err.message}`);
logError('CONFIG VIS: Error serving home page', err);
reply.code(500).send({ error: 'Internal server error' });
}
});
@@ -268,7 +269,7 @@ export async function setupConfigVisServer(logger, config) {
globals.logger.error(
`CONFIG VIS: Could not set up config visualisation server on ${address}`
);
globals.logger.error(`CONFIG VIS: ${globals.getErrorMessage(err)}`);
logError('CONFIG VIS', err);
configVisServer.log.error(err);
process.exit(1);
}

238
src/lib/error-tracker.js Normal file
View File

@@ -0,0 +1,238 @@
import { Mutex } from 'async-mutex';
import globals from '../globals.js';
import { postErrorMetricsToInfluxdb } from './influxdb/error-metrics.js';
/**
* Class for tracking counts of API errors in Butler SOS.
*
* This class provides thread-safe methods to track different types of API errors:
* - Qlik Sense API errors (Health API, Proxy Sessions API)
* - Data destination errors (InfluxDB, New Relic, MQTT)
*
* Counters reset daily at midnight UTC.
*/
export class ErrorTracker {
/**
* Creates a new ErrorTracker instance.
*
* @param {object} logger - Logger instance with error, debug, info, and verbose methods
*/
constructor(logger) {
this.logger = logger;
// Array of objects with error counts
// Each object has properties:
// - apiType: string (e.g., 'HEALTH_API', 'INFLUXDB_V3_WRITE')
// - serverName: string (name of the server, or empty string if not applicable)
// - count: integer
this.errorCounts = [];
// Mutex for synchronizing access to the array
this.errorMutex = new Mutex();
// Track when counters were last reset
this.lastResetDate = new Date().toISOString().split('T')[0]; // YYYY-MM-DD in UTC
}
/**
* Increments the error count for a specific API type and server.
*
* @param {string} apiType - The type of API that encountered an error (e.g., 'HEALTH_API', 'PROXY_API')
* @param {string} serverName - The name of the server where the error occurred (empty string if not applicable)
* @returns {Promise<void>}
*/
async incrementError(apiType, serverName) {
// Ensure the passed parameters are strings
if (typeof apiType !== 'string') {
this.logger.error(
`ERROR TRACKER: apiType must be a string: ${JSON.stringify(apiType)}`
);
return;
}
if (typeof serverName !== 'string') {
this.logger.error(
`ERROR TRACKER: serverName must be a string: ${JSON.stringify(serverName)}`
);
return;
}
const release = await this.errorMutex.acquire();
try {
// Check if we need to reset counters (new day in UTC)
const currentDate = new Date().toISOString().split('T')[0]; // YYYY-MM-DD in UTC
if (currentDate !== this.lastResetDate) {
this.logger.debug(
`ERROR TRACKER: Date changed from ${this.lastResetDate} to ${currentDate}, resetting counters`
);
await this.resetCounters();
this.lastResetDate = currentDate;
}
const found = this.errorCounts.find((element) => {
return element.apiType === apiType && element.serverName === serverName;
});
if (found) {
found.count += 1;
this.logger.debug(
`ERROR TRACKER: Incremented error count for ${apiType}/${serverName}, new count: ${found.count}`
);
} else {
this.logger.debug(
`ERROR TRACKER: Adding first error count for ${apiType}/${serverName}`
);
this.errorCounts.push({
apiType,
serverName,
count: 1,
});
}
// Log current error statistics
await this.logErrorSummary();
// Call placeholder function to store to InfluxDB (non-blocking)
// This will be implemented later
setImmediate(() => {
postErrorMetricsToInfluxdb(this.getErrorStats()).catch((err) => {
this.logger.debug(
`ERROR TRACKER: Error calling placeholder InfluxDB function: ${err.message}`
);
});
});
} finally {
release();
}
}
/**
* Resets all error counters.
* Should be called at midnight UTC or when starting fresh.
*
* @returns {Promise<void>}
*/
async resetCounters() {
// Note: Caller must hold the mutex before calling this method
this.errorCounts = [];
this.logger.info('ERROR TRACKER: Reset all error counters');
}
/**
* Gets current error statistics grouped by API type.
*
* @returns {object} Object with API types as keys, each containing total count and server breakdown
*/
getErrorStats() {
const stats = {};
for (const error of this.errorCounts) {
if (!stats[error.apiType]) {
stats[error.apiType] = {
total: 0,
servers: {},
};
}
stats[error.apiType].total += error.count;
if (error.serverName) {
stats[error.apiType].servers[error.serverName] = error.count;
} else {
// For errors without server context, use a placeholder
if (!stats[error.apiType].servers['_no_server_context']) {
stats[error.apiType].servers['_no_server_context'] = 0;
}
stats[error.apiType].servers['_no_server_context'] += error.count;
}
}
return stats;
}
/**
* Logs a summary of current error counts at INFO level.
*
* @returns {Promise<void>}
*/
async logErrorSummary() {
const stats = this.getErrorStats();
if (Object.keys(stats).length === 0) {
return; // No errors to log
}
// Calculate grand total
let grandTotal = 0;
for (const apiType in stats) {
grandTotal += stats[apiType].total;
}
this.logger.info(
`ERROR TRACKER: Error counts today (UTC): Total=${grandTotal}, Details=${JSON.stringify(stats)}`
);
}
/**
* Gets all error counts (for testing purposes).
*
* @returns {Promise<Array>} Array of error count objects
*/
async getErrorCounts() {
const release = await this.errorMutex.acquire();
try {
return this.errorCounts;
} finally {
release();
}
}
}
/**
* Sets up a timer that resets error counters at midnight UTC.
*
* This function calculates the time until next midnight UTC and schedules
* a reset, then reschedules itself for the following midnight.
*
* @returns {void}
*/
export function setupErrorCounterReset() {
/**
* Schedules the next reset at midnight UTC.
*/
const scheduleNextReset = () => {
// Calculate milliseconds until next midnight UTC
const now = new Date();
const nextMidnight = new Date(now);
nextMidnight.setUTCHours(24, 0, 0, 0);
const msUntilMidnight = nextMidnight - now;
globals.logger.info(
`ERROR TRACKER: Scheduled next error counter reset at ${nextMidnight.toISOString()} (in ${Math.round(msUntilMidnight / 1000 / 60)} minutes)`
);
setTimeout(async () => {
globals.logger.info('ERROR TRACKER: Midnight UTC reached, resetting error counters');
// Log final daily summary before reset
const release = await globals.errorTracker.errorMutex.acquire();
try {
await globals.errorTracker.logErrorSummary();
await globals.errorTracker.resetCounters();
globals.errorTracker.lastResetDate = new Date().toISOString().split('T')[0];
} finally {
release();
}
// Schedule next reset
scheduleNextReset();
}, msUntilMidnight);
};
// Start the reset cycle
scheduleNextReset();
}

5
src/lib/file-prep.js
View File

@@ -5,6 +5,7 @@ import sea from './sea-wrapper.js';
import handlebars from 'handlebars';
import globals from '../globals.js';
import { logError } from './log-error.js';
// Define MIME types for different file extensions
const MIME_TYPES = {
@@ -90,7 +91,7 @@ export async function prepareFile(filePath, encoding) {
stream = Readable.from([content]);
}
} catch (err) {
globals.logger.error(`FILE PREP: Error preparing file: ${err.message}`);
logError('FILE PREP: Error preparing file', err);
exists = false;
}
@@ -116,7 +117,7 @@ export function compileTemplate(templateContent, data) {
const template = handlebars.compile(templateContent);
return template(data);
} catch (err) {
globals.logger.error(`FILE PREP: Error compiling handlebars template: ${err.message}`);
logError('FILE PREP: Error compiling handlebars template', err);
throw err;
}
}

15
src/lib/healthmetrics.js
View File

@@ -7,13 +7,14 @@ import https from 'https';
import axios from 'axios';
import globals from '../globals.js';
import { postHealthMetricsToInfluxdb } from './post-to-influxdb.js';
import { postHealthMetricsToInfluxdb } from './influxdb/index.js';
import { postHealthMetricsToNewRelic } from './post-to-new-relic.js';
import { postHealthToMQTT } from './post-to-mqtt.js';
import { getServerHeaders } from './serverheaders.js';
import { getServerTags } from './servertags.js';
import { saveHealthMetricsToPrometheus } from './prom-client.js';
import { getCertificates, createCertificateOptions } from './cert-utils.js';
import { logError } from './log-error.js';
/**
* Retrieves health statistics from Qlik Sense server via the engine healthcheck API.
@@ -101,11 +102,19 @@ export function getHealthStatsFromSense(serverName, host, tags, headers) {
globals.logger.debug('HEALTH: Calling HEALTH metrics Prometheus method');
saveHealthMetricsToPrometheus(host, response.data, tags);
}
} else {
globals.logger.error(
`HEALTH: Received non-200 response code (${response.status}) from server '${serverName}' (${host})`
);
}
})
.catch((err) => {
globals.logger.error(
`HEALTH: Error when calling health check API for server '${serverName}' (${host}): ${globals.getErrorMessage(err)}`
// Track error count
globals.errorTracker.incrementError('HEALTH_API', serverName);
logError(
`HEALTH: Error when calling health check API for server '${serverName}' (${host})`,
err
);
});
}

88
src/lib/influxdb/README.md Normal file
View File

@@ -0,0 +1,88 @@
# InfluxDB Module Refactoring
This directory contains the refactored InfluxDB integration code, organized by version for better maintainability and testability.
## Structure
```text
influxdb/
├── shared/ # Shared utilities and helpers
│ └── utils.js # Common functions used across all versions
├── v1/ # InfluxDB 1.x implementations
├── v2/ # InfluxDB 2.x implementations
├── v3/ # InfluxDB 3.x implementations
│ └── health-metrics.js # Health metrics for v3
├── factory.js # Version router that delegates to appropriate implementation
└── index.js # Main facade providing backward compatibility
```
## Feature Flag
The refactored code is controlled by the `Butler-SOS.influxdbConfig.useRefactoredCode` configuration flag:
```yaml
Butler-SOS:
influxdbConfig:
enable: true
useRefactoredCode: false # Set to true to use refactored code
version: 3
# ... other config
```
**Default:** `false` (uses original code for backward compatibility)
## Migration Status
### Completed
- ✅ Directory structure
- ✅ Shared utilities (`getFormattedTime`, `processAppDocuments`, etc.)
- ✅ V3 health metrics implementation
- ✅ Factory router with feature flag
- ✅ Backward-compatible facade
- ✅ Configuration schema updated
### In Progress
- 🚧 V3 remaining modules (sessions, log events, user events, queue metrics)
- 🚧 V2 implementations
- 🚧 V1 implementations
### Pending
- ⏳ Complete test coverage for all modules
- ⏳ Integration tests
- ⏳ Performance benchmarking
## Usage
### For Developers
When the feature flag is enabled, the facade in `index.js` will route calls to the refactored implementations. If a version-specific implementation is not yet complete, it automatically falls back to the original code.
```javascript
// Imports work the same way
import { postHealthMetricsToInfluxdb } from './lib/influxdb/index.js';
// Function automatically routes based on feature flag
await postHealthMetricsToInfluxdb(serverName, host, body, serverTags);
```
### Adding New Implementations
1. Create the version-specific module (e.g., `v3/sessions.js`)
2. Import and export it in `factory.js`
3. Update the facade in `index.js` to use the factory
4. Add tests in the appropriate `__tests__` directory
## Benefits
1. **Maintainability**: Smaller, focused files instead of one 3000+ line file
2. **Testability**: Each module can be tested in isolation
3. **Code Reuse**: Shared utilities reduce duplication
4. **Version Management**: Easy to deprecate old versions when needed
5. **Safe Migration**: Feature flag allows gradual rollout
## Original Implementation
The original implementation remains in `/src/lib/post-to-influxdb.js` and continues to work as before. This ensures no breaking changes during migration.

48
src/lib/influxdb/error-metrics.js Normal file
View File

@@ -0,0 +1,48 @@
/**
* Placeholder function for storing error metrics to InfluxDB.
*
* This function will be implemented in the future to store API error counts
* to InfluxDB for historical tracking and visualization.
*
* @param {object} errorStats - Error statistics object grouped by API type
* @param {object} errorStats.apiType - Object containing total count and server breakdown
* @param {number} errorStats.apiType.total - Total error count for this API type
* @param {object} errorStats.apiType.servers - Object with server names as keys and error counts as values
* @returns {Promise<void>}
*
* @example
* const stats = {
* HEALTH_API: {
* total: 5,
* servers: {
* 'sense1': 3,
* 'sense2': 2
* }
* },
* INFLUXDB_V3_WRITE: {
* total: 2,
* servers: {
* '_no_server_context': 2
* }
* }
* };
* await postErrorMetricsToInfluxdb(stats);
*/
export async function postErrorMetricsToInfluxdb(errorStats) {
// TODO: Implement InfluxDB storage for error metrics
// This function should:
// 1. Check if InfluxDB is enabled in config
// 2. Route to appropriate version-specific implementation (v1/v2/v3)
// 3. Create data points with:
// - Measurement: 'api_error_counts' or similar
// - Tags: apiType, serverName
// - Fields: errorCount, timestamp
// 4. Write to InfluxDB with appropriate error handling
//
// For now, this is a no-op placeholder
// Uncomment for debugging during development:
// console.log('ERROR METRICS: Would store to InfluxDB:', JSON.stringify(errorStats, null, 2));
return Promise.resolve();
}

255
src/lib/influxdb/factory.js Normal file
View File

@@ -0,0 +1,255 @@
import globals from '../../globals.js';
import { getInfluxDbVersion, useRefactoredInfluxDb } from './shared/utils.js';
// Import version-specific implementations
import { storeHealthMetricsV1 } from './v1/health-metrics.js';
import { storeSessionsV1 } from './v1/sessions.js';
import { storeButlerMemoryV1 } from './v1/butler-memory.js';
import { storeUserEventV1 } from './v1/user-events.js';
import { storeEventCountV1, storeRejectedEventCountV1 } from './v1/event-counts.js';
import { storeUserEventQueueMetricsV1, storeLogEventQueueMetricsV1 } from './v1/queue-metrics.js';
import { storeLogEventV1 } from './v1/log-events.js';
import { storeHealthMetricsV2 } from './v2/health-metrics.js';
import { storeSessionsV2 } from './v2/sessions.js';
import { storeButlerMemoryV2 } from './v2/butler-memory.js';
import { storeUserEventV2 } from './v2/user-events.js';
import { storeEventCountV2, storeRejectedEventCountV2 } from './v2/event-counts.js';
import { storeUserEventQueueMetricsV2, storeLogEventQueueMetricsV2 } from './v2/queue-metrics.js';
import { storeLogEventV2 } from './v2/log-events.js';
import { postHealthMetricsToInfluxdbV3 } from './v3/health-metrics.js';
import { postProxySessionsToInfluxdbV3 } from './v3/sessions.js';
import { postButlerSOSMemoryUsageToInfluxdbV3 } from './v3/butler-memory.js';
import { postUserEventToInfluxdbV3 } from './v3/user-events.js';
import { storeEventCountInfluxDBV3, storeRejectedEventCountInfluxDBV3 } from './v3/event-counts.js';
import {
postUserEventQueueMetricsToInfluxdbV3,
postLogEventQueueMetricsToInfluxdbV3,
} from './v3/queue-metrics.js';
import { postLogEventToInfluxdbV3 } from './v3/log-events.js';
/**
* Factory function that routes health metrics to the appropriate InfluxDB version implementation.
*
* @param {string} serverName - The name of the Qlik Sense server
* @param {string} host - The hostname or IP of the Qlik Sense server
* @param {object} body - The health metrics data from Sense engine healthcheck API
* @param {object} serverTags - Tags to associate with the metrics
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postHealthMetricsToInfluxdb(serverName, host, body, serverTags) {
const version = getInfluxDbVersion();
if (version === 1) {
return storeHealthMetricsV1(serverTags, body);
}
if (version === 2) {
return storeHealthMetricsV2(serverName, host, body);
}
if (version === 3) {
return postHealthMetricsToInfluxdbV3(serverName, host, body, serverTags);
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
/**
* Factory function that routes proxy sessions to the appropriate InfluxDB version implementation.
*
* @param {object} userSessions - User session data
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postProxySessionsToInfluxdb(userSessions) {
const version = getInfluxDbVersion();
if (version === 1) {
return storeSessionsV1(userSessions);
}
if (version === 2) {
return storeSessionsV2(userSessions);
}
if (version === 3) {
return postProxySessionsToInfluxdbV3(userSessions);
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
/**
* Factory function that routes Butler SOS memory usage to the appropriate InfluxDB version implementation.
*
* @param {object} memory - Memory usage data object
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postButlerSOSMemoryUsageToInfluxdb(memory) {
const version = getInfluxDbVersion();
if (version === 1) {
return storeButlerMemoryV1(memory);
}
if (version === 2) {
return storeButlerMemoryV2(memory);
}
if (version === 3) {
return postButlerSOSMemoryUsageToInfluxdbV3(memory);
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
/**
* Factory function that routes user events to the appropriate InfluxDB version implementation.
*
* @param {object} msg - The user event message
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postUserEventToInfluxdb(msg) {
const version = getInfluxDbVersion();
if (version === 1) {
return storeUserEventV1(msg);
}
if (version === 2) {
return storeUserEventV2(msg);
}
if (version === 3) {
return postUserEventToInfluxdbV3(msg);
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
/**
* Factory function that routes event count storage to the appropriate InfluxDB version implementation.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function storeEventCountInfluxDB() {
const version = getInfluxDbVersion();
if (version === 1) {
return storeEventCountV1();
}
if (version === 2) {
return storeEventCountV2();
}
if (version === 3) {
return storeEventCountInfluxDBV3();
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
/**
* Factory function that routes rejected event count storage to the appropriate InfluxDB version implementation.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function storeRejectedEventCountInfluxDB() {
const version = getInfluxDbVersion();
if (version === 1) {
return storeRejectedEventCountV1();
}
if (version === 2) {
return storeRejectedEventCountV2();
}
if (version === 3) {
return storeRejectedEventCountInfluxDBV3();
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
/**
* Factory function that routes user event queue metrics to the appropriate InfluxDB version implementation.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postUserEventQueueMetricsToInfluxdb() {
try {
const version = getInfluxDbVersion();
if (version === 1) {
return storeUserEventQueueMetricsV1();
}
if (version === 2) {
return storeUserEventQueueMetricsV2();
}
if (version === 3) {
return postUserEventQueueMetricsToInfluxdbV3();
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
} catch (err) {
globals.logger.error(
`INFLUXDB FACTORY: Error in postUserEventQueueMetricsToInfluxdb: ${err.message}`
);
globals.logger.debug(`INFLUXDB FACTORY: Error stack: ${err.stack}`);
throw err;
}
}
/**
* Factory function that routes log event queue metrics to the appropriate InfluxDB version implementation.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postLogEventQueueMetricsToInfluxdb() {
try {
const version = getInfluxDbVersion();
if (version === 1) {
return storeLogEventQueueMetricsV1();
}
if (version === 2) {
return storeLogEventQueueMetricsV2();
}
if (version === 3) {
return postLogEventQueueMetricsToInfluxdbV3();
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
} catch (err) {
globals.logger.error(
`INFLUXDB FACTORY: Error in postLogEventQueueMetricsToInfluxdb: ${err.message}`
);
globals.logger.debug(`INFLUXDB FACTORY: Error stack: ${err.stack}`);
throw err;
}
}
/**
* Factory function that routes log events to the appropriate InfluxDB version implementation.
*
* @param {object} msg - The log event message
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postLogEventToInfluxdb(msg) {
const version = getInfluxDbVersion();
if (version === 1) {
return storeLogEventV1(msg);
}
if (version === 2) {
return storeLogEventV2(msg);
}
if (version === 3) {
return postLogEventToInfluxdbV3(msg);
}
globals.logger.debug(`INFLUXDB FACTORY: Unknown InfluxDB version: v${version}`);
throw new Error(`InfluxDB v${version} not supported`);
}
// TODO: Add other factory functions as they're implemented
// etc...

315
src/lib/influxdb/index.js Normal file
View File

@@ -0,0 +1,315 @@
import { useRefactoredInfluxDb, getFormattedTime } from './shared/utils.js';
import * as factory from './factory.js';
import globals from '../../globals.js';
// Import original implementation for fallback
import * as original from '../post-to-influxdb.js';
/**
* Main facade that routes to either refactored or original implementation based on feature flag.
*
* This allows for safe migration by testing refactored code alongside original implementation.
*/
/**
* Calculates and formats the uptime of a Qlik Sense engine.
* This function is version-agnostic and always uses the shared implementation.
*
* @param {string} serverStarted - The server start time in format "YYYYMMDDThhmmss"
* @returns {string} A formatted string representing uptime (e.g. "5 days, 3h 45m 12s")
*/
export { getFormattedTime };
/**
* Posts health metrics data from Qlik Sense to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {string} serverName - The name of the Qlik Sense server
* @param {string} host - The hostname or IP of the Qlik Sense server
* @param {object} body - The health metrics data from Sense engine healthcheck API
* @param {object} serverTags - Tags to associate with the metrics
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postHealthMetricsToInfluxdb(serverName, host, body, serverTags) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postHealthMetricsToInfluxdb(serverName, host, body, serverTags);
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
return await original.postHealthMetricsToInfluxdb(serverName, host, body, serverTags);
}
}
return await original.postHealthMetricsToInfluxdb(serverName, host, body, serverTags);
}
/**
* Posts proxy sessions data to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} userSessions - User session data
* @returns {Promise<void>}
*/
export async function postProxySessionsToInfluxdb(userSessions) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postProxySessionsToInfluxdb(userSessions);
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
return await original.postProxySessionsToInfluxdb(userSessions);
}
}
return await original.postProxySessionsToInfluxdb(userSessions);
}
/**
* Posts Butler SOS's own memory usage to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} memory - Memory usage data object
* @returns {Promise<void>}
*/
export async function postButlerSOSMemoryUsageToInfluxdb(memory) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postButlerSOSMemoryUsageToInfluxdb(memory);
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
return await original.postButlerSOSMemoryUsageToInfluxdb(memory);
}
}
return await original.postButlerSOSMemoryUsageToInfluxdb(memory);
}
/**
* Posts user events to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} msg - The user event message
* @returns {Promise<void>}
*/
export async function postUserEventToInfluxdb(msg) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postUserEventToInfluxdb(msg);
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original globals.logger.error(`INFLUXDB ROUTING: User event - falling back to legacy code due to error: ${err.message}`);
globals.logger.debug(`INFLUXDB ROUTING: User event - error stack: ${err.stack}`);
return await original.postUserEventToInfluxdb(msg);
}
}
return await original.postUserEventToInfluxdb(msg);
}
/**
* Posts log events to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} msg - The log event message
* @returns {Promise<void>}
*/
export async function postLogEventToInfluxdb(msg) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postLogEventToInfluxdb(msg);
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original globals.logger.error(`INFLUXDB ROUTING: Log event - falling back to legacy code due to error: ${err.message}`);
globals.logger.debug(`INFLUXDB ROUTING: Log event - error stack: ${err.stack}`);
return await original.postLogEventToInfluxdb(msg);
}
}
return await original.postLogEventToInfluxdb(msg);
}
/**
* Stores event counts to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {string} eventsSinceMidnight - Events since midnight data
* @param {string} eventsLastHour - Events last hour data
* @returns {Promise<void>}
*/
export async function storeEventCountInfluxDB(eventsSinceMidnight, eventsLastHour) {
if (useRefactoredInfluxDb()) {
try {
return await factory.storeEventCountInfluxDB();
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
return await original.storeEventCountInfluxDB(eventsSinceMidnight, eventsLastHour);
}
}
return await original.storeEventCountInfluxDB(eventsSinceMidnight, eventsLastHour);
}
/**
* Stores rejected event counts to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} rejectedSinceMidnight - Rejected events since midnight
* @param {object} rejectedLastHour - Rejected events last hour
* @returns {Promise<void>}
*/
export async function storeRejectedEventCountInfluxDB(rejectedSinceMidnight, rejectedLastHour) {
if (useRefactoredInfluxDb()) {
try {
return await factory.storeRejectedEventCountInfluxDB();
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
return await original.storeRejectedEventCountInfluxDB(
rejectedSinceMidnight,
rejectedLastHour
);
}
}
return await original.storeRejectedEventCountInfluxDB(rejectedSinceMidnight, rejectedLastHour);
}
/**
* Stores user event queue metrics to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} queueMetrics - Queue metrics data
* @returns {Promise<void>}
*/
export async function postUserEventQueueMetricsToInfluxdb(queueMetrics) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postUserEventQueueMetricsToInfluxdb();
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
globals.logger.error(
`INFLUXDB ROUTING: User event queue metrics - falling back to legacy code due to error: ${err.message}`
);
globals.logger.debug(
`INFLUXDB ROUTING: User event queue metrics - error stack: ${err.stack}`
);
return await original.postUserEventQueueMetricsToInfluxdb(queueMetrics);
}
}
globals.logger.verbose(
'INFLUXDB ROUTING: User event queue metrics - using original implementation'
);
return await original.postUserEventQueueMetricsToInfluxdb(queueMetrics);
}
/**
* Stores log event queue metrics to InfluxDB.
*
* Routes to refactored or original implementation based on feature flag.
*
* @param {object} queueMetrics - Queue metrics data
* @returns {Promise<void>}
*/
export async function postLogEventQueueMetricsToInfluxdb(queueMetrics) {
if (useRefactoredInfluxDb()) {
try {
return await factory.postLogEventQueueMetricsToInfluxdb();
} catch (err) {
// If refactored code not yet implemented for this version, fall back to original
globals.logger.error(
`INFLUXDB ROUTING: Log event queue metrics - falling back to legacy code due to error: ${err.message}`
);
globals.logger.debug(
`INFLUXDB ROUTING: Log event queue metrics - error stack: ${err.stack}`
);
return await original.postLogEventQueueMetricsToInfluxdb(queueMetrics);
}
}
return await original.postLogEventQueueMetricsToInfluxdb(queueMetrics);
}
/**
* Sets up timers for queue metrics storage.
*
* @returns {object} Object containing interval IDs for cleanup
*/
export function setupUdpQueueMetricsStorage() {
const intervalIds = {
userEvents: null,
logEvents: null,
};
// Check if InfluxDB is enabled
if (globals.config.get('Butler-SOS.influxdbConfig.enable') !== true) {
globals.logger.info(
'UDP QUEUE METRICS: InfluxDB is disabled. Skipping setup of queue metrics storage'
);
return intervalIds;
}
// Set up user events queue metrics storage
if (
globals.config.get('Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.enable') ===
true
) {
const writeFrequency = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.writeFrequency'
);
intervalIds.userEvents = setInterval(async () => {
try {
globals.logger.verbose(
'UDP QUEUE METRICS: Timer for storing user event queue metrics to InfluxDB triggered'
);
await postUserEventQueueMetricsToInfluxdb();
} catch (err) {
globals.logger.error(
`UDP QUEUE METRICS: Error storing user event queue metrics to InfluxDB: ${
err && err.stack ? err.stack : err
}`
);
}
}, writeFrequency);
globals.logger.info(
`UDP QUEUE METRICS: Set up timer for storing user event queue metrics to InfluxDB (interval: ${writeFrequency}ms)`
);
} else {
globals.logger.info(
'UDP QUEUE METRICS: User event queue metrics storage to InfluxDB is disabled'
);
}
// Set up log events queue metrics storage
if (
globals.config.get('Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.enable') ===
true
) {
const writeFrequency = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.writeFrequency'
);
intervalIds.logEvents = setInterval(async () => {
try {
globals.logger.verbose(
'UDP QUEUE METRICS: Timer for storing log event queue metrics to InfluxDB triggered'
);
await postLogEventQueueMetricsToInfluxdb();
} catch (err) {
globals.logger.error(
`UDP QUEUE METRICS: Error storing log event queue metrics to InfluxDB: ${
err && err.stack ? err.stack : err
}`
);
}
}, writeFrequency);
globals.logger.info(
`UDP QUEUE METRICS: Set up timer for storing log event queue metrics to InfluxDB (interval: ${writeFrequency}ms)`
);
} else {
globals.logger.info(
'UDP QUEUE METRICS: Log event queue metrics storage to InfluxDB is disabled'
);
}
return intervalIds;
}

289
src/lib/influxdb/shared/utils.js Normal file
View File

@@ -0,0 +1,289 @@
import globals from '../../../globals.js';
const sessionAppPrefix = 'SessionApp';
const MIN_TIMESTAMP_LENGTH = 15;
/**
* Calculates and formats the uptime of a Qlik Sense engine.
*
* This function takes the server start time from the engine healthcheck API
* and calculates how long the server has been running, returning a formatted string.
*
* @param {string} serverStarted - The server start time in format "YYYYMMDDThhmmss"
* @returns {string} A formatted string representing uptime (e.g. "5 days, 3h 45m 12s")
*/
export function getFormattedTime(serverStarted) {
// Handle invalid or empty input
if (
!serverStarted ||
typeof serverStarted !== 'string' ||
serverStarted.length < MIN_TIMESTAMP_LENGTH
) {
return '';
}
const dateTime = Date.now();
const timestamp = Math.floor(dateTime);
const str = serverStarted;
const year = str.substring(0, 4);
const month = str.substring(4, 6);
const day = str.substring(6, 8);
const hour = str.substring(9, 11);
const minute = str.substring(11, 13);
const second = str.substring(13, 15);
// Validate date components
if (
isNaN(year) ||
isNaN(month) ||
isNaN(day) ||
isNaN(hour) ||
isNaN(minute) ||
isNaN(second)
) {
return '';
}
const dateTimeStarted = new Date(year, month - 1, day, hour, minute, second);
// Check if the date is valid
if (isNaN(dateTimeStarted.getTime())) {
return '';
}
const timestampStarted = Math.floor(dateTimeStarted);
const diff = timestamp - timestampStarted;
// Create a new JavaScript Date object based on the timestamp
// multiplied by 1000 so that the argument is in milliseconds, not seconds.
const date = new Date(diff);
const days = Math.trunc(diff / (1000 * 60 * 60 * 24));
// Hours part from the timestamp
const hours = date.getHours();
// Minutes part from the timestamp
const minutes = `0${date.getMinutes()}`;
// Seconds part from the timestamp
const seconds = `0${date.getSeconds()}`;
// Will display time in 10:30:23 format
return `${days} days, ${hours}h ${minutes.substr(-2)}m ${seconds.substr(-2)}s`;
}
/**
* Processes app documents and categorizes them as session apps or regular apps.
* Returns arrays of app names for both categories.
*
* @param {string[]} docIDs - Array of document IDs to process
* @param {string} logPrefix - Prefix for log messages
* @param {string} appState - Description of app state (e.g., 'active', 'loaded', 'in memory')
* @returns {Promise<{appNames: string[], sessionAppNames: string[]}>} Object containing sorted arrays of app names
*/
export async function processAppDocuments(docIDs, logPrefix, appState) {
const appNames = [];
const sessionAppNames = [];
/**
* Stores a document ID in the appropriate array based on its type.
*
* @param {string} docID - The document ID to store
* @returns {Promise<void>} Promise that resolves when the document ID has been processed
*/
const storeDoc = (docID) => {
return new Promise((resolve, _reject) => {
if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) {
// Session app
globals.logger.debug(`${logPrefix}: Session app is ${appState}: ${docID}`);
sessionAppNames.push(docID);
} else {
// Not session app
const app = globals.appNames.find((element) => element.id === docID);
if (app) {
globals.logger.debug(`${logPrefix}: App is ${appState}: ${app.name}`);
appNames.push(app.name);
} else {
appNames.push(docID);
}
}
resolve();
});
};
const promises = docIDs.map(
(docID) =>
new Promise(async (resolve, _reject) => {
await storeDoc(docID);
resolve();
})
);
await Promise.all(promises);
appNames.sort();
sessionAppNames.sort();
return { appNames, sessionAppNames };
}
/**
* Checks if InfluxDB is enabled and initialized.
*
* @returns {boolean} True if InfluxDB is enabled and initialized
*/
export function isInfluxDbEnabled() {
if (!globals.influx) {
globals.logger.warn(
'INFLUXDB: Influxdb object not initialized. Data will not be sent to InfluxDB'
);
return false;
}
return true;
}
/**
* Gets the InfluxDB version from configuration.
*
* @returns {number} The InfluxDB version (1, 2, or 3)
*/
export function getInfluxDbVersion() {
return globals.config.get('Butler-SOS.influxdbConfig.version');
}
/**
* Checks if the refactored InfluxDB code path should be used.
*
* @returns {boolean} True if refactored code should be used
*/
export function useRefactoredInfluxDb() {
// Feature flag to enable/disable refactored code path
// Default to false for backward compatibility
return globals.config.get('Butler-SOS.influxdbConfig.useRefactoredCode') === true;
}
/**
* Applies tags from a tags object to an InfluxDB Point3 object.
* This is needed for v3 as it doesn't have automatic default tags like v2.
*
* @param {object} point - The Point3 object to apply tags to
* @param {object} tags - Object containing tag key-value pairs
* @returns {object} The Point3 object with tags applied (for chaining)
*/
export function applyTagsToPoint3(point, tags) {
if (!tags || typeof tags !== 'object') {
return point;
}
// Apply each tag to the point
Object.entries(tags).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
point.setTag(key, String(value));
}
});
return point;
}
/**
* Writes data to InfluxDB v3 with retry logic and exponential backoff.
*
* This function attempts to write data to InfluxDB v3 with configurable retry logic.
* If a write fails due to timeout or network issues, it will retry up to maxRetries times
* with exponential backoff between attempts.
*
* @param {Function} writeFn - Async function that performs the write operation
* @param {string} context - Description of what's being written (for logging)
* @param {object} options - Retry options
* @param {number} options.maxRetries - Maximum number of retry attempts (default: 3)
* @param {number} options.initialDelayMs - Initial delay before first retry in ms (default: 1000)
* @param {number} options.maxDelayMs - Maximum delay between retries in ms (default: 10000)
* @param {number} options.backoffMultiplier - Multiplier for exponential backoff (default: 2)
*
* @returns {Promise<void>} Promise that resolves when write succeeds or rejects after all retries fail
*
* @throws {Error} The last error encountered after all retries are exhausted
*/
export async function writeToInfluxV3WithRetry(writeFn, context, options = {}) {
const {
maxRetries = 3,
initialDelayMs = 1000,
maxDelayMs = 10000,
backoffMultiplier = 2,
} = options;
let lastError;
let attempt = 0;
while (attempt <= maxRetries) {
try {
await writeFn();
// Log success if this was a retry
if (attempt > 0) {
globals.logger.info(
`INFLUXDB V3 RETRY: ${context} - Write succeeded on attempt ${attempt + 1}/${maxRetries + 1}`
);
}
return; // Success!
} catch (err) {
lastError = err;
attempt++;
// Check if this is a timeout error - check constructor name and message
const errorName = err.constructor?.name || err.name || '';
const errorMessage = err.message || '';
const isTimeoutError =
errorName === 'RequestTimedOutError' ||
errorMessage.includes('timeout') ||
errorMessage.includes('timed out') ||
errorMessage.includes('Request timed out');
// Log the error type for debugging
globals.logger.debug(
`INFLUXDB V3 RETRY: ${context} - Error caught: ${errorName}, message: ${errorMessage}, isTimeout: ${isTimeoutError}`
);
// Don't retry on non-timeout errors - fail immediately
if (!isTimeoutError) {
globals.logger.warn(
`INFLUXDB V3 WRITE: ${context} - Non-timeout error (${errorName}), not retrying: ${globals.getErrorMessage(err)}`
);
throw err;
}
// This is a timeout error - check if we have retries left
if (attempt <= maxRetries) {
// Calculate delay with exponential backoff
const delayMs = Math.min(
initialDelayMs * Math.pow(backoffMultiplier, attempt - 1),
maxDelayMs
);
globals.logger.warn(
`INFLUXDB V3 RETRY: ${context} - Timeout (${errorName}) on attempt ${attempt}/${maxRetries + 1}, retrying in ${delayMs}ms...`
);
// Wait before retrying
await new Promise((resolve) => setTimeout(resolve, delayMs));
} else {
// All retries exhausted
globals.logger.error(
`INFLUXDB V3 RETRY: ${context} - All ${maxRetries + 1} attempts failed. Last error: ${globals.getErrorMessage(err)}`
);
// Track error count (final failure after all retries)
await globals.errorTracker.incrementError('INFLUXDB_V3_WRITE', '');
}
}
}
// All retries failed, throw the last error
throw lastError;
}

46
src/lib/influxdb/v1/butler-memory.js Normal file
View File

@@ -0,0 +1,46 @@
import globals from '../../../globals.js';
/**
* Store Butler SOS memory usage to InfluxDB v1
*
* @param {object} memory - Memory usage data
* @returns {Promise<void>}
*/
export async function storeButlerMemoryV1(memory) {
try {
const butlerVersion = globals.appVersion;
const datapoint = [
{
measurement: 'butlersos_memory_usage',
tags: {
butler_sos_instance: memory.instanceTag,
version: butlerVersion,
},
fields: {
heap_used: memory.heapUsedMByte,
heap_total: memory.heapTotalMByte,
external: memory.externalMemoryMByte,
process_memory: memory.processMemoryMByte,
},
},
];
globals.logger.silly(
`MEMORY USAGE V1: Influxdb datapoint for Butler SOS memory usage: ${JSON.stringify(
datapoint,
null,
2
)}`
);
await globals.influx.writePoints(datapoint);
globals.logger.verbose('MEMORY USAGE V1: Sent Butler SOS memory usage data to InfluxDB');
} catch (err) {
globals.logger.error(
`MEMORY USAGE V1: Error saving Butler SOS memory data: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

216
src/lib/influxdb/v1/event-counts.js Normal file
View File

@@ -0,0 +1,216 @@
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
/**
* Store event counts to InfluxDB v1
* Aggregates and stores counts for log and user events
*
* @returns {Promise<void>}
*/
export async function storeEventCountV1() {
try {
// Get array of log events
const logEvents = await globals.udpEvents.getLogEvents();
const userEvents = await globals.udpEvents.getUserEvents();
globals.logger.debug(`EVENT COUNT V1: Log events: ${JSON.stringify(logEvents, null, 2)}`);
globals.logger.debug(`EVENT COUNT V1: User events: ${JSON.stringify(userEvents, null, 2)}`);
// Are there any events to store?
if (logEvents.length === 0 && userEvents.length === 0) {
globals.logger.verbose('EVENT COUNT V1: No events to store in InfluxDB');
return;
}
const points = [];
// Get measurement name to use for event counts
const measurementName = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName'
);
// Loop through data in log events and create datapoints
for (const event of logEvents) {
const point = {
measurement: measurementName,
tags: {
event_type: 'log',
source: event.source,
host: event.host,
subsystem: event.subsystem,
},
fields: {
counter: event.counter,
},
};
// Add static tags from config file
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !==
null &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags').length > 0
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
for (const item of configTags) {
point.tags[item.name] = item.value;
}
}
points.push(point);
}
// Loop through data in user events and create datapoints
for (const event of userEvents) {
const point = {
measurement: measurementName,
tags: {
event_type: 'user',
source: event.source,
host: event.host,
subsystem: event.subsystem,
},
fields: {
counter: event.counter,
},
};
// Add static tags from config file
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !==
null &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags').length > 0
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
for (const item of configTags) {
point.tags[item.name] = item.value;
}
}
points.push(point);
}
await globals.influx.writePoints(points);
globals.logger.verbose('EVENT COUNT V1: Sent event count data to InfluxDB');
} catch (err) {
logError('EVENT COUNT V1: Error saving data', err);
throw err;
}
}
/**
* Store rejected event counts to InfluxDB v1
* Tracks events that were rejected due to validation failures or rate limiting
*
* @returns {Promise<void>}
*/
export async function storeRejectedEventCountV1() {
try {
// Get array of rejected log events
const rejectedLogEvents = await globals.rejectedEvents.getRejectedLogEvents();
globals.logger.debug(
`REJECTED EVENT COUNT V1: Rejected log events: ${JSON.stringify(
rejectedLogEvents,
null,
2
)}`
);
// Are there any events to store?
if (rejectedLogEvents.length === 0) {
globals.logger.verbose('REJECTED EVENT COUNT V1: No events to store in InfluxDB');
return;
}
const points = [];
// Get measurement name to use for rejected events
const measurementName = globals.config.get(
'Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName'
);
// Loop through data in rejected log events and create datapoints
// Use counter and process_time as fields
for (const event of rejectedLogEvents) {
if (event.source === 'qseow-qix-perf') {
// For each unique combination of source, appId, appName, method and objectType,
// write the counter and processTime properties to InfluxDB
const tags = {
source: event.source,
app_id: event.appId,
method: event.method,
object_type: event.objectType,
};
// Tags that are empty in some cases. Only add if they are non-empty
if (event?.appName?.length > 0) {
tags.app_name = event.appName;
tags.app_name_set = 'true';
} else {
tags.app_name_set = 'false';
}
// Add static tags from config file
if (
globals.config.has(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
) &&
globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
) !== null &&
globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
).length > 0
) {
const configTags = globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
);
for (const item of configTags) {
tags[item.name] = item.value;
}
}
const fields = {
counter: event.counter,
process_time: event.processTime,
};
const point = {
measurement: measurementName,
tags,
fields,
};
points.push(point);
} else {
const point = {
measurement: measurementName,
tags: {
source: event.source,
},
fields: {
counter: event.counter,
},
};
points.push(point);
}
}
await globals.influx.writePoints(points);
globals.logger.verbose(
'REJECTED EVENT COUNT V1: Sent rejected event count data to InfluxDB'
);
} catch (err) {
logError('REJECTED EVENT COUNT V1: Error saving data', err);
throw err;
}
}

160
src/lib/influxdb/v1/health-metrics.js Normal file
View File

@@ -0,0 +1,160 @@
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
import { getFormattedTime, processAppDocuments } from '../shared/utils.js';
/**
* Store health metrics from multiple Sense engines to InfluxDB v1
*
* @param {object} serverTags - Server tags for all measurements
* @param {object} body - Health metrics data from Sense engine
* @returns {Promise<void>}
*/
export async function storeHealthMetricsV1(serverTags, body) {
try {
// Process app names for different document types
const [appNamesActive, sessionAppNamesActive] = await processAppDocuments(
body.apps.active_docs
);
const [appNamesLoaded, sessionAppNamesLoaded] = await processAppDocuments(
body.apps.loaded_docs
);
const [appNamesInMemory, sessionAppNamesInMemory] = await processAppDocuments(
body.apps.in_memory_docs
);
// Create datapoint array for v1 - plain objects with measurement, tags, fields
const datapoint = [
{
measurement: 'sense_server',
tags: serverTags,
fields: {
version: body.version,
started: body.started,
uptime: getFormattedTime(body.started),
},
},
{
measurement: 'mem',
tags: serverTags,
fields: {
comitted: body.mem.committed,
allocated: body.mem.allocated,
free: body.mem.free,
},
},
{
measurement: 'apps',
tags: serverTags,
fields: {
active_docs_count: body.apps.active_docs.length,
loaded_docs_count: body.apps.loaded_docs.length,
in_memory_docs_count: body.apps.in_memory_docs.length,
active_docs: globals.config.get(
'Butler-SOS.influxdbConfig.includeFields.activeDocs'
)
? body.apps.active_docs
: '',
active_docs_names:
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? appNamesActive.map((name) => `"${name}"`).join(',')
: '',
active_session_docs_names:
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? sessionAppNamesActive.map((name) => `"${name}"`).join(',')
: '',
loaded_docs: globals.config.get(
'Butler-SOS.influxdbConfig.includeFields.loadedDocs'
)
? body.apps.loaded_docs
: '',
loaded_docs_names:
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? appNamesLoaded.map((name) => `"${name}"`).join(',')
: '',
loaded_session_docs_names:
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? sessionAppNamesLoaded.map((name) => `"${name}"`).join(',')
: '',
in_memory_docs: globals.config.get(
'Butler-SOS.influxdbConfig.includeFields.inMemoryDocs'
)
? body.apps.in_memory_docs
: '',
in_memory_docs_names:
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? appNamesInMemory.map((name) => `"${name}"`).join(',')
: '',
in_memory_session_docs_names:
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? sessionAppNamesInMemory.map((name) => `"${name}"`).join(',')
: '',
calls: body.apps.calls,
selections: body.apps.selections,
},
},
{
measurement: 'cpu',
tags: serverTags,
fields: {
total: body.cpu.total,
},
},
{
measurement: 'session',
tags: serverTags,
fields: {
active: body.session.active,
total: body.session.total,
},
},
{
measurement: 'users',
tags: serverTags,
fields: {
active: body.users.active,
total: body.users.total,
},
},
{
measurement: 'cache',
tags: serverTags,
fields: {
hits: body.cache.hits,
lookups: body.cache.lookups,
added: body.cache.added,
replaced: body.cache.replaced,
bytes_added: body.cache.bytes_added,
},
},
{
measurement: 'saturated',
tags: serverTags,
fields: {
saturated: body.saturated,
},
},
];
// Write to InfluxDB v1 using node-influx library
await globals.influx.writePoints(datapoint);
globals.logger.verbose(
`INFLUXDB V1 HEALTH METRICS: Stored health data from server: ${serverTags.server_name}`
);
} catch (err) {
// Track error count
await globals.errorTracker.incrementError('INFLUXDB_V1_WRITE', serverTags.server_name);
logError('INFLUXDB V1 HEALTH METRICS: Error saving health data', err);
throw err;
}
}

211
src/lib/influxdb/v1/log-events.js Normal file
View File

@@ -0,0 +1,211 @@
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
/**
* Store log event to InfluxDB v1
* Handles log events from different Sense sources
*
* @param {object} msg - Log event message
* @returns {Promise<void>}
*/
export async function storeLogEventV1(msg) {
try {
globals.logger.debug(`LOG EVENT V1: ${JSON.stringify(msg)}`);
// Check if this is a supported source
if (
msg.source !== 'qseow-engine' &&
msg.source !== 'qseow-proxy' &&
msg.source !== 'qseow-scheduler' &&
msg.source !== 'qseow-repository' &&
msg.source !== 'qseow-qix-perf'
) {
globals.logger.warn(`LOG EVENT V1: Unsupported log event source: ${msg.source}`);
return;
}
let tags;
let fields;
// Process each source type
if (msg.source === 'qseow-engine') {
tags = {
host: msg.host,
level: msg.level,
source: msg.source,
log_row: msg.log_row,
subsystem: msg.subsystem,
};
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) tags.user_full = msg.user_full;
if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory;
if (msg?.user_id?.length > 0) tags.user_id = msg.user_id;
if (msg?.result_code?.length > 0) tags.result_code = msg.result_code;
if (msg?.windows_user?.length > 0) tags.windows_user = msg.windows_user;
if (msg?.task_id?.length > 0) tags.task_id = msg.task_id;
if (msg?.task_name?.length > 0) tags.task_name = msg.task_name;
if (msg?.app_id?.length > 0) tags.app_id = msg.app_id;
if (msg?.app_name?.length > 0) tags.app_name = msg.app_name;
if (msg?.engine_exe_version?.length > 0)
tags.engine_exe_version = msg.engine_exe_version;
fields = {
message: msg.message,
exception_message: msg.exception_message,
command: msg.command,
result_code: msg.result_code,
origin: msg.origin,
context: msg.context,
session_id: msg.session_id,
raw_event: JSON.stringify(msg),
};
} else if (msg.source === 'qseow-proxy') {
tags = {
host: msg.host,
level: msg.level,
source: msg.source,
log_row: msg.log_row,
subsystem: msg.subsystem,
};
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) tags.user_full = msg.user_full;
if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory;
if (msg?.user_id?.length > 0) tags.user_id = msg.user_id;
if (msg?.result_code?.length > 0) tags.result_code = msg.result_code;
fields = {
message: msg.message,
exception_message: msg.exception_message,
command: msg.command,
result_code: msg.result_code,
origin: msg.origin,
context: msg.context,
raw_event: JSON.stringify(msg),
};
} else if (msg.source === 'qseow-scheduler') {
tags = {
host: msg.host,
level: msg.level,
source: msg.source,
log_row: msg.log_row,
subsystem: msg.subsystem,
};
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) tags.user_full = msg.user_full;
if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory;
if (msg?.user_id?.length > 0) tags.user_id = msg.user_id;
if (msg?.task_id?.length > 0) tags.task_id = msg.task_id;
if (msg?.task_name?.length > 0) tags.task_name = msg.task_name;
fields = {
message: msg.message,
exception_message: msg.exception_message,
app_name: msg.app_name,
app_id: msg.app_id,
execution_id: msg.execution_id,
raw_event: JSON.stringify(msg),
};
} else if (msg.source === 'qseow-repository') {
tags = {
host: msg.host,
level: msg.level,
source: msg.source,
log_row: msg.log_row,
subsystem: msg.subsystem,
};
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) tags.user_full = msg.user_full;
if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory;
if (msg?.user_id?.length > 0) tags.user_id = msg.user_id;
if (msg?.result_code?.length > 0) tags.result_code = msg.result_code;
fields = {
message: msg.message,
exception_message: msg.exception_message,
command: msg.command,
result_code: msg.result_code,
origin: msg.origin,
context: msg.context,
raw_event: JSON.stringify(msg),
};
} else if (msg.source === 'qseow-qix-perf') {
tags = {
host: msg.host?.length > 0 ? msg.host : '<Unknown>',
level: msg.level?.length > 0 ? msg.level : '<Unknown>',
source: msg.source?.length > 0 ? msg.source : '<Unknown>',
log_row: msg.log_row?.length > 0 ? msg.log_row : '-1',
subsystem: msg.subsystem?.length > 0 ? msg.subsystem : '<Unknown>',
method: msg.method?.length > 0 ? msg.method : '<Unknown>',
object_type: msg.object_type?.length > 0 ? msg.object_type : '<Unknown>',
proxy_session_id: msg.proxy_session_id?.length > 0 ? msg.proxy_session_id : '-1',
session_id: msg.session_id?.length > 0 ? msg.session_id : '-1',
event_activity_source:
msg.event_activity_source?.length > 0 ? msg.event_activity_source : '<Unknown>',
};
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) tags.user_full = msg.user_full;
if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory;
if (msg?.user_id?.length > 0) tags.user_id = msg.user_id;
if (msg?.app_id?.length > 0) tags.app_id = msg.app_id;
if (msg?.app_name?.length > 0) tags.app_name = msg.app_name;
if (msg?.object_id?.length > 0) tags.object_id = msg.object_id;
fields = {
app_id: msg.app_id,
process_time: msg.process_time,
work_time: msg.work_time,
lock_time: msg.lock_time,
validate_time: msg.validate_time,
traverse_time: msg.traverse_time,
handle: msg.handle,
net_ram: msg.net_ram,
peak_ram: msg.peak_ram,
raw_event: JSON.stringify(msg),
};
}
// Add log event categories to tags if available
// The msg.category array contains objects with properties 'name' and 'value'
if (msg?.category?.length > 0) {
msg.category.forEach((category) => {
tags[category.name] = category.value;
});
}
// Add custom tags from config file to payload
if (
globals.config.has('Butler-SOS.logEvents.tags') &&
globals.config.get('Butler-SOS.logEvents.tags') !== null &&
globals.config.get('Butler-SOS.logEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.logEvents.tags');
for (const item of configTags) {
tags[item.name] = item.value;
}
}
const datapoint = [
{
measurement: 'log_event',
tags,
fields,
},
];
globals.logger.silly(
`LOG EVENT V1: Influxdb datapoint: ${JSON.stringify(datapoint, null, 2)}`
);
await globals.influx.writePoints(datapoint);
globals.logger.verbose('LOG EVENT V1: Sent log event data to InfluxDB');
} catch (err) {
logError('LOG EVENT V1: Error saving log event', err);
throw err;
}
}

152
src/lib/influxdb/v1/queue-metrics.js Normal file
View File

@@ -0,0 +1,152 @@
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
/**
* Store user event queue metrics to InfluxDB v1
*
* @returns {Promise<void>}
*/
export async function storeUserEventQueueMetricsV1() {
try {
// Check if queue metrics are enabled
if (
!globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.enable'
)
) {
return;
}
// Get metrics from queue manager
const queueManager = globals.udpQueueManagerUserActivity;
if (!queueManager) {
globals.logger.warn('USER EVENT QUEUE METRICS V1: Queue manager not initialized');
return;
}
const metrics = await queueManager.getMetrics();
// Get configuration
const measurementName = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.measurementName'
);
const configTags = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.tags'
);
const point = {
measurement: measurementName,
tags: {
queue_type: 'user_events',
host: globals.hostInfo.hostname,
},
fields: {
queue_size: metrics.queueSize,
queue_max_size: metrics.queueMaxSize,
queue_utilization_pct: metrics.queueUtilizationPct,
queue_pending: metrics.queuePending,
messages_received: metrics.messagesReceived,
messages_queued: metrics.messagesQueued,
messages_processed: metrics.messagesProcessed,
messages_failed: metrics.messagesFailed,
messages_dropped_total: metrics.messagesDroppedTotal,
messages_dropped_rate_limit: metrics.messagesDroppedRateLimit,
messages_dropped_queue_full: metrics.messagesDroppedQueueFull,
messages_dropped_size: metrics.messagesDroppedSize,
processing_time_avg_ms: metrics.processingTimeAvgMs,
processing_time_p95_ms: metrics.processingTimeP95Ms,
processing_time_max_ms: metrics.processingTimeMaxMs,
rate_limit_current: metrics.rateLimitCurrent,
backpressure_active: metrics.backpressureActive,
},
};
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.tags[item.name] = item.value;
}
}
await globals.influx.writePoints([point]);
globals.logger.verbose('USER EVENT QUEUE METRICS V1: Sent queue metrics data to InfluxDB');
} catch (err) {
logError('USER EVENT QUEUE METRICS V1: Error saving data', err);
throw err;
}
}
/**
* Store log event queue metrics to InfluxDB v1
*
* @returns {Promise<void>}
*/
export async function storeLogEventQueueMetricsV1() {
try {
// Check if queue metrics are enabled
if (
!globals.config.get('Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.enable')
) {
return;
}
// Get metrics from queue manager
const queueManager = globals.udpQueueManagerLogEvents;
if (!queueManager) {
globals.logger.warn('LOG EVENT QUEUE METRICS V1: Queue manager not initialized');
return;
}
const metrics = await queueManager.getMetrics();
// Get configuration
const measurementName = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.measurementName'
);
const configTags = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.tags'
);
const point = {
measurement: measurementName,
tags: {
queue_type: 'log_events',
host: globals.hostInfo.hostname,
},
fields: {
queue_size: metrics.queueSize,
queue_max_size: metrics.queueMaxSize,
queue_utilization_pct: metrics.queueUtilizationPct,
queue_pending: metrics.queuePending,
messages_received: metrics.messagesReceived,
messages_queued: metrics.messagesQueued,
messages_processed: metrics.messagesProcessed,
messages_failed: metrics.messagesFailed,
messages_dropped_total: metrics.messagesDroppedTotal,
messages_dropped_rate_limit: metrics.messagesDroppedRateLimit,
messages_dropped_queue_full: metrics.messagesDroppedQueueFull,
messages_dropped_size: metrics.messagesDroppedSize,
processing_time_avg_ms: metrics.processingTimeAvgMs,
processing_time_p95_ms: metrics.processingTimeP95Ms,
processing_time_max_ms: metrics.processingTimeMaxMs,
rate_limit_current: metrics.rateLimitCurrent,
backpressure_active: metrics.backpressureActive,
},
};
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.tags[item.name] = item.value;
}
}
await globals.influx.writePoints([point]);
globals.logger.verbose('LOG EVENT QUEUE METRICS V1: Sent queue metrics data to InfluxDB');
} catch (err) {
logError('LOG EVENT QUEUE METRICS V1: Error saving data', err);
throw err;
}
}

39
src/lib/influxdb/v1/sessions.js Normal file
View File

@@ -0,0 +1,39 @@
import globals from '../../../globals.js';
/**
* Store proxy session data to InfluxDB v1
*
* @param {object} userSessions - User session data including datapointInfluxdb array
* @returns {Promise<void>}
*/
export async function storeSessionsV1(userSessions) {
try {
globals.logger.silly(
`PROXY SESSIONS V1: Influxdb datapoint for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${JSON.stringify(
userSessions.datapointInfluxdb,
null,
2
)}`
);
// Data points are already in InfluxDB v1 format (plain objects)
// Write array of measurements: user_session_summary, user_session_list, user_session_details
await globals.influx.writePoints(userSessions.datapointInfluxdb);
globals.logger.debug(
`PROXY SESSIONS V1: Session count for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${userSessions.sessionCount}`
);
globals.logger.debug(
`PROXY SESSIONS V1: User list for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${userSessions.uniqueUserList}`
);
globals.logger.verbose(
`PROXY SESSIONS V1: Sent user session data to InfluxDB for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"`
);
} catch (err) {
globals.logger.error(
`PROXY SESSIONS V1: Error saving user session data: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

73
src/lib/influxdb/v1/user-events.js Normal file
View File

@@ -0,0 +1,73 @@
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
/**
* Store user event to InfluxDB v1
*
* @param {object} msg - User event message
* @returns {Promise<void>}
*/
export async function storeUserEventV1(msg) {
try {
globals.logger.debug(`USER EVENT V1: ${JSON.stringify(msg)}`);
// First prepare tags relating to the actual user event, then add tags defined in the config file
// The config file tags can for example be used to separate data from DEV/TEST/PROD environments
const tags = {
host: msg.host,
event_action: msg.command,
userFull: `${msg.user_directory}\\${msg.user_id}`,
userDirectory: msg.user_directory,
userId: msg.user_id,
origin: msg.origin,
};
// Add app id and name to tags if available
if (msg?.appId) tags.appId = msg.appId;
if (msg?.appName) tags.appName = msg.appName;
// Add user agent info to tags if available
if (msg?.ua?.browser?.name) tags.uaBrowserName = msg?.ua?.browser?.name;
if (msg?.ua?.browser?.major) tags.uaBrowserMajorVersion = msg?.ua?.browser?.major;
if (msg?.ua?.os?.name) tags.uaOsName = msg?.ua?.os?.name;
if (msg?.ua?.os?.version) tags.uaOsVersion = msg?.ua?.os?.version;
// Add custom tags from config file to payload
if (
globals.config.has('Butler-SOS.userEvents.tags') &&
globals.config.get('Butler-SOS.userEvents.tags') !== null &&
globals.config.get('Butler-SOS.userEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.userEvents.tags');
for (const item of configTags) {
tags[item.name] = item.value;
}
}
const datapoint = [
{
measurement: 'user_events',
tags,
fields: {
userFull: tags.userFull,
userId: tags.userId,
},
},
];
// Add app id and name to fields if available
if (msg?.appId) datapoint[0].fields.appId = msg.appId;
if (msg?.appName) datapoint[0].fields.appName = msg.appName;
globals.logger.silly(
`USER EVENT V1: Influxdb datapoint: ${JSON.stringify(datapoint, null, 2)}`
);
await globals.influx.writePoints(datapoint);
globals.logger.verbose('USER EVENT V1: Sent user event data to InfluxDB');
} catch (err) {
logError('USER EVENT V1: Error saving user event', err);
throw err;
}
}

56
src/lib/influxdb/v2/butler-memory.js Normal file
View File

@@ -0,0 +1,56 @@
import { Point } from '@influxdata/influxdb-client';
import globals from '../../../globals.js';
/**
* Store Butler SOS memory usage to InfluxDB v2
*
* @param {object} memory - Memory usage data
* @returns {Promise<void>}
*/
export async function storeButlerMemoryV2(memory) {
try {
const butlerVersion = globals.appVersion;
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('MEMORY USAGE V2: Influxdb write API object not found');
return;
}
// Create point using v2 Point class
const point = new Point('butlersos_memory_usage')
.tag('butler_sos_instance', memory.instanceTag)
.tag('version', butlerVersion)
.floatField('heap_used', memory.heapUsedMByte)
.floatField('heap_total', memory.heapTotalMByte)
.floatField('external', memory.externalMemoryMByte)
.floatField('process_memory', memory.processMemoryMByte);
globals.logger.silly(
`MEMORY USAGE V2: Influxdb datapoint for Butler SOS memory usage: ${JSON.stringify(
point,
null,
2
)}`
);
await writeApi.writePoint(point);
globals.logger.verbose('MEMORY USAGE V2: Sent Butler SOS memory usage data to InfluxDB');
} catch (err) {
globals.logger.error(
`MEMORY USAGE V2: Error saving Butler SOS memory data: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

217
src/lib/influxdb/v2/event-counts.js Normal file
View File

@@ -0,0 +1,217 @@
import { Point } from '@influxdata/influxdb-client';
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
/**
* Store event counts to InfluxDB v2
* Aggregates and stores counts for log and user events
*
* @returns {Promise<void>}
*/
export async function storeEventCountV2() {
try {
// Get array of log events
const logEvents = await globals.udpEvents.getLogEvents();
const userEvents = await globals.udpEvents.getUserEvents();
globals.logger.debug(`EVENT COUNT V2: Log events: ${JSON.stringify(logEvents, null, 2)}`);
globals.logger.debug(`EVENT COUNT V2: User events: ${JSON.stringify(userEvents, null, 2)}`);
// Are there any events to store?
if (logEvents.length === 0 && userEvents.length === 0) {
globals.logger.verbose('EVENT COUNT V2: No events to store in InfluxDB');
return;
}
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('EVENT COUNT V2: Influxdb write API object not found');
return;
}
const points = [];
// Get measurement name to use for event counts
const measurementName = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName'
);
// Loop through data in log events and create datapoints
for (const event of logEvents) {
const point = new Point(measurementName)
.tag('event_type', 'log')
.tag('source', event.source)
.tag('host', event.host)
.tag('subsystem', event.subsystem)
.intField('counter', event.counter);
// Add static tags from config file
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !==
null &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags').length > 0
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
points.push(point);
}
// Loop through data in user events and create datapoints
for (const event of userEvents) {
const point = new Point(measurementName)
.tag('event_type', 'user')
.tag('source', event.source)
.tag('host', event.host)
.tag('subsystem', event.subsystem)
.intField('counter', event.counter);
// Add static tags from config file
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !==
null &&
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags').length > 0
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
points.push(point);
}
await writeApi.writePoints(points);
globals.logger.verbose('EVENT COUNT V2: Sent event count data to InfluxDB');
} catch (err) {
logError('EVENT COUNT V2: Error saving data', err);
throw err;
}
}
/**
* Store rejected event counts to InfluxDB v2
* Tracks events that were rejected due to validation failures or rate limiting
*
* @returns {Promise<void>}
*/
export async function storeRejectedEventCountV2() {
try {
// Get array of rejected log events
const rejectedLogEvents = await globals.rejectedEvents.getRejectedLogEvents();
globals.logger.debug(
`REJECTED EVENT COUNT V2: Rejected log events: ${JSON.stringify(
rejectedLogEvents,
null,
2
)}`
);
// Are there any events to store?
if (rejectedLogEvents.length === 0) {
globals.logger.verbose('REJECTED EVENT COUNT V2: No events to store in InfluxDB');
return;
}
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('REJECTED EVENT COUNT V2: Influxdb write API object not found');
return;
}
const points = [];
// Get measurement name to use for rejected events
const measurementName = globals.config.get(
'Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName'
);
// Loop through data in rejected log events and create datapoints
for (const event of rejectedLogEvents) {
if (event.source === 'qseow-qix-perf') {
// For qix-perf events, include app info and performance metrics
let point = new Point(measurementName)
.tag('source', event.source)
.tag('app_id', event.appId)
.tag('method', event.method)
.tag('object_type', event.objectType)
.intField('counter', event.counter)
.floatField('process_time', event.processTime);
if (event?.appName?.length > 0) {
point.tag('app_name', event.appName).tag('app_name_set', 'true');
} else {
point.tag('app_name_set', 'false');
}
// Add static tags from config file
if (
globals.config.has(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
) &&
globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
) !== null &&
globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
).length > 0
) {
const configTags = globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
);
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
points.push(point);
} else {
const point = new Point(measurementName)
.tag('source', event.source)
.intField('counter', event.counter);
points.push(point);
}
}
await writeApi.writePoints(points);
globals.logger.verbose(
'REJECTED EVENT COUNT V2: Sent rejected event count data to InfluxDB'
);
} catch (err) {
logError('REJECTED EVENT COUNT V2: Error saving data', err);
throw err;
}
}

151
src/lib/influxdb/v2/health-metrics.js Normal file
View File

@@ -0,0 +1,151 @@
import { Point } from '@influxdata/influxdb-client';
import globals from '../../../globals.js';
import { getFormattedTime, processAppDocuments } from '../shared/utils.js';
/**
* Store health metrics from multiple Sense engines to InfluxDB v2
*
* @param {string} serverName - The name of the Qlik Sense server
* @param {string} host - The hostname or IP of the Qlik Sense server
* @param {object} body - Health metrics data from Sense engine
* @returns {Promise<void>}
*/
export async function storeHealthMetricsV2(serverName, host, body) {
try {
// Find writeApi for the server specified by serverName
const writeApi = globals.influxWriteApi.find(
(element) => element.serverName === serverName
);
if (!writeApi) {
globals.logger.warn(
`HEALTH METRICS V2: Influxdb write API object not found for host ${host}`
);
return;
}
// Process app names for different document types
const [appNamesActive, sessionAppNamesActive] = await processAppDocuments(
body.apps.active_docs
);
const [appNamesLoaded, sessionAppNamesLoaded] = await processAppDocuments(
body.apps.loaded_docs
);
const [appNamesInMemory, sessionAppNamesInMemory] = await processAppDocuments(
body.apps.in_memory_docs
);
const formattedTime = getFormattedTime(body.started);
// Create points using v2 Point class
const points = [
new Point('sense_server')
.stringField('version', body.version)
.stringField('started', body.started)
.stringField('uptime', formattedTime),
new Point('mem')
.floatField('comitted', body.mem.committed)
.floatField('allocated', body.mem.allocated)
.floatField('free', body.mem.free),
new Point('apps')
.intField('active_docs_count', body.apps.active_docs.length)
.intField('loaded_docs_count', body.apps.loaded_docs.length)
.intField('in_memory_docs_count', body.apps.in_memory_docs.length)
.stringField(
'active_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? body.apps.active_docs
: ''
)
.stringField(
'active_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? appNamesActive.toString()
: ''
)
.stringField(
'active_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? sessionAppNamesActive.toString()
: ''
)
.stringField(
'loaded_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? body.apps.loaded_docs
: ''
)
.stringField(
'loaded_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? appNamesLoaded.toString()
: ''
)
.stringField(
'loaded_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? sessionAppNamesLoaded.toString()
: ''
)
.stringField(
'in_memory_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? body.apps.in_memory_docs
: ''
)
.stringField(
'in_memory_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? appNamesInMemory.toString()
: ''
)
.stringField(
'in_memory_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? sessionAppNamesInMemory.toString()
: ''
)
.uintField('calls', body.apps.calls)
.uintField('selections', body.apps.selections),
new Point('cpu').floatField('total', body.cpu.total),
new Point('session')
.uintField('active', body.session.active)
.uintField('total', body.session.total),
new Point('users')
.uintField('active', body.users.active)
.uintField('total', body.users.total),
new Point('cache')
.uintField('hits', body.cache.hits)
.uintField('lookups', body.cache.lookups)
.intField('added', body.cache.added)
.intField('replaced', body.cache.replaced)
.intField('bytes_added', body.cache.bytes_added),
new Point('saturated').booleanField('saturated', body.saturated),
];
await writeApi.writeAPI.writePoints(points);
globals.logger.verbose(`HEALTH METRICS V2: Stored health data from server: ${serverName}`);
} catch (err) {
// Track error count
await globals.errorTracker.incrementError('INFLUXDB_V2_WRITE', serverName);
globals.logger.error(
`HEALTH METRICS V2: Error saving health data: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

197
src/lib/influxdb/v2/log-events.js Normal file
View File

@@ -0,0 +1,197 @@
import { Point } from '@influxdata/influxdb-client';
import globals from '../../../globals.js';
/**
* Store log event to InfluxDB v2
* Handles log events from different Sense sources
*
* @param {object} msg - Log event message
* @returns {Promise<void>}
*/
export async function storeLogEventV2(msg) {
try {
globals.logger.debug(`LOG EVENT V2: ${JSON.stringify(msg)}`);
// Check if this is a supported source
if (
msg.source !== 'qseow-engine' &&
msg.source !== 'qseow-proxy' &&
msg.source !== 'qseow-scheduler' &&
msg.source !== 'qseow-repository' &&
msg.source !== 'qseow-qix-perf'
) {
globals.logger.warn(`LOG EVENT V2: Unsupported log event source: ${msg.source}`);
return;
}
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('LOG EVENT V2: Influxdb write API object not found');
return;
}
let point;
// Process each source type
if (msg.source === 'qseow-engine') {
point = new Point('log_event')
.tag('host', msg.host)
.tag('level', msg.level)
.tag('source', msg.source)
.tag('log_row', msg.log_row)
.tag('subsystem', msg.subsystem)
.stringField('message', msg.message)
.stringField('exception_message', msg.exception_message)
.stringField('command', msg.command)
.stringField('result_code', msg.result_code)
.stringField('origin', msg.origin)
.stringField('context', msg.context)
.stringField('session_id', msg.session_id)
.stringField('raw_event', JSON.stringify(msg));
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.tag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.tag('result_code', msg.result_code);
if (msg?.windows_user?.length > 0) point.tag('windows_user', msg.windows_user);
if (msg?.task_id?.length > 0) point.tag('task_id', msg.task_id);
if (msg?.task_name?.length > 0) point.tag('task_name', msg.task_name);
if (msg?.app_id?.length > 0) point.tag('app_id', msg.app_id);
if (msg?.app_name?.length > 0) point.tag('app_name', msg.app_name);
if (msg?.engine_exe_version?.length > 0)
point.tag('engine_exe_version', msg.engine_exe_version);
} else if (msg.source === 'qseow-proxy') {
point = new Point('log_event')
.tag('host', msg.host)
.tag('level', msg.level)
.tag('source', msg.source)
.tag('log_row', msg.log_row)
.tag('subsystem', msg.subsystem)
.stringField('message', msg.message)
.stringField('exception_message', msg.exception_message)
.stringField('command', msg.command)
.stringField('result_code', msg.result_code)
.stringField('origin', msg.origin)
.stringField('context', msg.context)
.stringField('raw_event', JSON.stringify(msg));
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.tag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.tag('result_code', msg.result_code);
} else if (msg.source === 'qseow-scheduler') {
point = new Point('log_event')
.tag('host', msg.host)
.tag('level', msg.level)
.tag('source', msg.source)
.tag('log_row', msg.log_row)
.tag('subsystem', msg.subsystem)
.stringField('message', msg.message)
.stringField('exception_message', msg.exception_message)
.stringField('app_name', msg.app_name)
.stringField('app_id', msg.app_id)
.stringField('execution_id', msg.execution_id)
.stringField('raw_event', JSON.stringify(msg));
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.tag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id);
if (msg?.task_id?.length > 0) point.tag('task_id', msg.task_id);
if (msg?.task_name?.length > 0) point.tag('task_name', msg.task_name);
} else if (msg.source === 'qseow-repository') {
point = new Point('log_event')
.tag('host', msg.host)
.tag('level', msg.level)
.tag('source', msg.source)
.tag('log_row', msg.log_row)
.tag('subsystem', msg.subsystem)
.stringField('message', msg.message)
.stringField('exception_message', msg.exception_message)
.stringField('command', msg.command)
.stringField('result_code', msg.result_code)
.stringField('origin', msg.origin)
.stringField('context', msg.context)
.stringField('raw_event', JSON.stringify(msg));
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.tag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.tag('result_code', msg.result_code);
} else if (msg.source === 'qseow-qix-perf') {
point = new Point('log_event')
.tag('host', msg.host)
.tag('level', msg.level)
.tag('source', msg.source)
.tag('log_row', msg.log_row)
.tag('subsystem', msg.subsystem)
.tag('method', msg.method)
.tag('object_type', msg.object_type)
.tag('proxy_session_id', msg.proxy_session_id)
.tag('session_id', msg.session_id)
.tag('event_activity_source', msg.event_activity_source)
.stringField('app_id', msg.app_id)
.floatField('process_time', parseFloat(msg.process_time))
.floatField('work_time', parseFloat(msg.work_time))
.floatField('lock_time', parseFloat(msg.lock_time))
.floatField('validate_time', parseFloat(msg.validate_time))
.floatField('traverse_time', parseFloat(msg.traverse_time))
.stringField('handle', msg.handle)
.intField('net_ram', parseInt(msg.net_ram))
.intField('peak_ram', parseInt(msg.peak_ram))
.stringField('raw_event', JSON.stringify(msg));
// Tags that are empty in some cases. Only add if they are non-empty
if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.tag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id);
if (msg?.app_id?.length > 0) point.tag('app_id', msg.app_id);
if (msg?.app_name?.length > 0) point.tag('app_name', msg.app_name);
if (msg?.object_id?.length > 0) point.tag('object_id', msg.object_id);
}
// Add log event categories to tags if available
// The msg.category array contains objects with properties 'name' and 'value'
if (msg?.category?.length > 0) {
msg.category.forEach((category) => {
point.tag(category.name, category.value);
});
}
// Add custom tags from config file to payload
if (
globals.config.has('Butler-SOS.logEvents.tags') &&
globals.config.get('Butler-SOS.logEvents.tags') !== null &&
globals.config.get('Butler-SOS.logEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.logEvents.tags');
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
globals.logger.silly(`LOG EVENT V2: Influxdb datapoint: ${JSON.stringify(point, null, 2)}`);
await writeApi.writePoint(point);
globals.logger.verbose('LOG EVENT V2: Sent log event data to InfluxDB');
} catch (err) {
globals.logger.error(
`LOG EVENT V2: Error saving log event: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

175
src/lib/influxdb/v2/queue-metrics.js Normal file
View File

@@ -0,0 +1,175 @@
import { Point } from '@influxdata/influxdb-client';
import globals from '../../../globals.js';
import { logError } from '../../log-error.js';
/**
* Store user event queue metrics to InfluxDB v2
*
* @returns {Promise<void>}
*/
export async function storeUserEventQueueMetricsV2() {
try {
// Check if queue metrics are enabled
if (
!globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.enable'
)
) {
return;
}
// Get metrics from queue manager
const queueManager = globals.udpQueueManagerUserActivity;
if (!queueManager) {
globals.logger.warn('USER EVENT QUEUE METRICS V2: Queue manager not initialized');
return;
}
const metrics = await queueManager.getMetrics();
// Get configuration
const measurementName = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.measurementName'
);
const configTags = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.tags'
);
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('USER EVENT QUEUE METRICS V2: Influxdb write API object not found');
return;
}
const point = new Point(measurementName)
.tag('queue_type', 'user_events')
.tag('host', globals.hostInfo.hostname)
.intField('queue_size', metrics.queueSize)
.intField('queue_max_size', metrics.queueMaxSize)
.floatField('queue_utilization_pct', metrics.queueUtilizationPct)
.intField('queue_pending', metrics.queuePending)
.intField('messages_received', metrics.messagesReceived)
.intField('messages_queued', metrics.messagesQueued)
.intField('messages_processed', metrics.messagesProcessed)
.intField('messages_failed', metrics.messagesFailed)
.intField('messages_dropped_total', metrics.messagesDroppedTotal)
.intField('messages_dropped_rate_limit', metrics.messagesDroppedRateLimit)
.intField('messages_dropped_queue_full', metrics.messagesDroppedQueueFull)
.intField('messages_dropped_size', metrics.messagesDroppedSize)
.floatField('processing_time_avg_ms', metrics.processingTimeAvgMs)
.floatField('processing_time_p95_ms', metrics.processingTimeP95Ms)
.floatField('processing_time_max_ms', metrics.processingTimeMaxMs)
.intField('rate_limit_current', metrics.rateLimitCurrent)
.intField('backpressure_active', metrics.backpressureActive);
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
writeApi.writePoint(point);
await writeApi.close();
globals.logger.verbose('USER EVENT QUEUE METRICS V2: Sent queue metrics data to InfluxDB');
} catch (err) {
logError('USER EVENT QUEUE METRICS V2: Error saving data', err);
throw err;
}
}
/**
* Store log event queue metrics to InfluxDB v2
*
* @returns {Promise<void>}
*/
export async function storeLogEventQueueMetricsV2() {
try {
// Check if queue metrics are enabled
if (
!globals.config.get('Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.enable')
) {
return;
}
// Get metrics from queue manager
const queueManager = globals.udpQueueManagerLogEvents;
if (!queueManager) {
globals.logger.warn('LOG EVENT QUEUE METRICS V2: Queue manager not initialized');
return;
}
const metrics = await queueManager.getMetrics();
// Get configuration
const measurementName = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.measurementName'
);
const configTags = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.tags'
);
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('LOG EVENT QUEUE METRICS V2: Influxdb write API object not found');
return;
}
const point = new Point(measurementName)
.tag('queue_type', 'log_events')
.tag('host', globals.hostInfo.hostname)
.intField('queue_size', metrics.queueSize)
.intField('queue_max_size', metrics.queueMaxSize)
.floatField('queue_utilization_pct', metrics.queueUtilizationPct)
.intField('queue_pending', metrics.queuePending)
.intField('messages_received', metrics.messagesReceived)
.intField('messages_queued', metrics.messagesQueued)
.intField('messages_processed', metrics.messagesProcessed)
.intField('messages_failed', metrics.messagesFailed)
.intField('messages_dropped_total', metrics.messagesDroppedTotal)
.intField('messages_dropped_rate_limit', metrics.messagesDroppedRateLimit)
.intField('messages_dropped_queue_full', metrics.messagesDroppedQueueFull)
.intField('messages_dropped_size', metrics.messagesDroppedSize)
.floatField('processing_time_avg_ms', metrics.processingTimeAvgMs)
.floatField('processing_time_p95_ms', metrics.processingTimeP95Ms)
.floatField('processing_time_max_ms', metrics.processingTimeMaxMs)
.intField('rate_limit_current', metrics.rateLimitCurrent)
.intField('backpressure_active', metrics.backpressureActive);
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
writeApi.writePoint(point);
await writeApi.close();
globals.logger.verbose('LOG EVENT QUEUE METRICS V2: Sent queue metrics data to InfluxDB');
} catch (err) {
logError('LOG EVENT QUEUE METRICS V2: Error saving data', err);
throw err;
}
}

47
src/lib/influxdb/v2/sessions.js Normal file
View File

@@ -0,0 +1,47 @@
import globals from '../../../globals.js';
/**
* Store proxy session data to InfluxDB v2
*
* @param {object} userSessions - User session data including datapointInfluxdb array
* @returns {Promise<void>}
*/
export async function storeSessionsV2(userSessions) {
try {
// Find writeApi for the server specified by serverName
const writeApi = globals.influxWriteApi.find(
(element) => element.serverName === userSessions.serverName
);
if (!writeApi) {
globals.logger.warn(
`PROXY SESSIONS V2: Influxdb write API object not found for host ${userSessions.host}`
);
return;
}
globals.logger.silly(
`PROXY SESSIONS V2: Influxdb datapoint for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${JSON.stringify(
userSessions.datapointInfluxdb,
null,
2
)}`
);
// Data points are already in InfluxDB v2 format (Point objects)
// Write array of measurements: user_session_summary, user_session_list, user_session_details
await writeApi.writeAPI.writePoints(userSessions.datapointInfluxdb);
globals.logger.verbose(
`PROXY SESSIONS V2: Sent user session data to InfluxDB for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"`
);
} catch (err) {
// Track error count
await globals.errorTracker.incrementError('INFLUXDB_V2_WRITE', userSessions.serverName);
globals.logger.error(
`PROXY SESSIONS V2: Error saving user session data: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

80
src/lib/influxdb/v2/user-events.js Normal file
View File

@@ -0,0 +1,80 @@
import { Point } from '@influxdata/influxdb-client';
import globals from '../../../globals.js';
/**
* Store user event to InfluxDB v2
*
* @param {object} msg - User event message
* @returns {Promise<void>}
*/
export async function storeUserEventV2(msg) {
try {
globals.logger.debug(`USER EVENT V2: ${JSON.stringify(msg)}`);
// Create write API with options
const writeOptions = {
flushInterval: 5000,
maxRetries: 2,
};
const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org');
const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket');
const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions);
if (!writeApi) {
globals.logger.warn('USER EVENT V2: Influxdb write API object not found');
return;
}
// Create point using v2 Point class
const point = new Point('user_events')
.tag('host', msg.host)
.tag('event_action', msg.command)
.tag('userFull', `${msg.user_directory}\\${msg.user_id}`)
.tag('userDirectory', msg.user_directory)
.tag('userId', msg.user_id)
.tag('origin', msg.origin)
.stringField('userFull', `${msg.user_directory}\\${msg.user_id}`)
.stringField('userId', msg.user_id);
// Add app id and name to tags if available
if (msg?.appId) point.tag('appId', msg.appId);
if (msg?.appName) point.tag('appName', msg.appName);
// Add user agent info to tags if available
if (msg?.ua?.browser?.name) point.tag('uaBrowserName', msg?.ua?.browser?.name);
if (msg?.ua?.browser?.major) point.tag('uaBrowserMajorVersion', msg?.ua?.browser?.major);
if (msg?.ua?.os?.name) point.tag('uaOsName', msg?.ua?.os?.name);
if (msg?.ua?.os?.version) point.tag('uaOsVersion', msg?.ua?.os?.version);
// Add custom tags from config file to payload
if (
globals.config.has('Butler-SOS.userEvents.tags') &&
globals.config.get('Butler-SOS.userEvents.tags') !== null &&
globals.config.get('Butler-SOS.userEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.userEvents.tags');
for (const item of configTags) {
point.tag(item.name, item.value);
}
}
// Add app id and name to fields if available
if (msg?.appId) point.stringField('appId', msg.appId);
if (msg?.appName) point.stringField('appName', msg.appName);
globals.logger.silly(
`USER EVENT V2: Influxdb datapoint: ${JSON.stringify(point, null, 2)}`
);
await writeApi.writePoint(point);
globals.logger.verbose('USER EVENT V2: Sent user event data to InfluxDB');
} catch (err) {
globals.logger.error(
`USER EVENT V2: Error saving user event: ${globals.getErrorMessage(err)}`
);
throw err;
}
}

23
src/lib/influxdb/v3/__tests__/health-metrics.test.js Normal file
View File

@@ -0,0 +1,23 @@
/**
* Tests for v3 health metrics module
*
* Note: These tests are skipped due to complex ES module mocking requirements.
* Full integration tests with actual InfluxDB connections are performed separately.
* The refactored code is functionally tested through the main post-to-influxdb tests.
*/
import { jest } from '@jest/globals';
describe.skip('v3/health-metrics', () => {
test('module exports postHealthMetricsToInfluxdbV3 function', async () => {
const healthMetrics = await import('../health-metrics.js');
expect(healthMetrics.postHealthMetricsToInfluxdbV3).toBeDefined();
expect(typeof healthMetrics.postHealthMetricsToInfluxdbV3).toBe('function');
});
test('module can be imported without errors', async () => {
expect(async () => {
await import('../health-metrics.js');
}).not.toThrow();
});
});

55
src/lib/influxdb/v3/butler-memory.js Normal file
View File

@@ -0,0 +1,55 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import { isInfluxDbEnabled, writeToInfluxV3WithRetry } from '../shared/utils.js';
/**
* Posts Butler SOS memory usage metrics to InfluxDB v3.
*
* This function captures memory usage metrics from the Butler SOS process itself
* and stores them in InfluxDB v3.
*
* @param {object} memory - Memory usage data object
* @param {string} memory.instanceTag - Instance identifier tag
* @param {number} memory.heapUsedMByte - Heap used in MB
* @param {number} memory.heapTotalMByte - Total heap size in MB
* @param {number} memory.externalMemoryMByte - External memory usage in MB
* @param {number} memory.processMemoryMByte - Process memory usage in MB
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postButlerSOSMemoryUsageToInfluxdbV3(memory) {
globals.logger.debug(`MEMORY USAGE V3: Memory usage ${JSON.stringify(memory, null, 2)})`);
// Get Butler version
const butlerVersion = globals.appVersion;
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Create point for v3
const point = new Point3('butlersos_memory_usage')
.setTag('butler_sos_instance', memory.instanceTag)
.setTag('version', butlerVersion)
.setFloatField('heap_used', memory.heapUsedMByte)
.setFloatField('heap_total', memory.heapTotalMByte)
.setFloatField('external', memory.externalMemoryMByte)
.setFloatField('process_memory', memory.processMemoryMByte);
try {
// Convert point to line protocol and write directly with retry logic
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
'Memory usage metrics'
);
globals.logger.debug(`MEMORY USAGE V3: Wrote data to InfluxDB v3`);
} catch (err) {
globals.logger.error(
`MEMORY USAGE V3: Error saving memory usage data to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
}
globals.logger.verbose('MEMORY USAGE V3: Sent Butler SOS memory usage data to InfluxDB');
}

258
src/lib/influxdb/v3/event-counts.js Normal file
View File

@@ -0,0 +1,258 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import { isInfluxDbEnabled, writeToInfluxV3WithRetry } from '../shared/utils.js';
/**
* Store event count in InfluxDB v3
*
* @description
* This function reads arrays of log and user events from the `udpEvents` object,
* and stores the data in InfluxDB v3. The data is written to a measurement named after
* the `Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName` config setting.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
* @throws {Error} Error if unable to write data to InfluxDB
*/
export async function storeEventCountInfluxDBV3() {
// Get array of log events
const logEvents = await globals.udpEvents.getLogEvents();
const userEvents = await globals.udpEvents.getUserEvents();
// Debug
globals.logger.debug(
`EVENT COUNT INFLUXDB V3: Log events: ${JSON.stringify(logEvents, null, 2)}`
);
globals.logger.debug(
`EVENT COUNT INFLUXDB V3: User events: ${JSON.stringify(userEvents, null, 2)}`
);
// Are there any events to store?
if (logEvents.length === 0 && userEvents.length === 0) {
globals.logger.verbose('EVENT COUNT INFLUXDB V3: No events to store in InfluxDB');
return;
}
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
try {
// Store data for each log event
for (const logEvent of logEvents) {
const tags = {
butler_sos_instance: globals.options.instanceTag,
event_type: 'log',
source: logEvent.source,
host: logEvent.host,
subsystem: logEvent.subsystem,
};
// Add static tags defined in config file, if any
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
Array.isArray(
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags')
)
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
configTags.forEach((tag) => {
tags[tag.name] = tag.value;
});
}
const point = new Point3(
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName')
)
.setTag('event_type', 'log')
.setTag('source', logEvent.source)
.setTag('host', logEvent.host)
.setTag('subsystem', logEvent.subsystem)
.setIntegerField('counter', logEvent.counter);
// Add additional tags to point
Object.keys(tags).forEach((key) => {
point.setTag(key, tags[key]);
});
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
'Log event counts'
);
globals.logger.debug(`EVENT COUNT INFLUXDB V3: Wrote log event data to InfluxDB v3`);
}
// Loop through data in user events and create datapoints
for (const event of userEvents) {
const tags = {
butler_sos_instance: globals.options.instanceTag,
event_type: 'user',
source: event.source,
host: event.host,
subsystem: event.subsystem,
};
// Add static tags defined in config file, if any
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
Array.isArray(
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags')
)
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
configTags.forEach((tag) => {
tags[tag.name] = tag.value;
});
}
const point = new Point3(
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName')
)
.setTag('event_type', 'user')
.setTag('source', event.source)
.setTag('host', event.host)
.setTag('subsystem', event.subsystem)
.setIntegerField('counter', event.counter);
// Add additional tags to point
Object.keys(tags).forEach((key) => {
point.setTag(key, tags[key]);
});
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
'User event counts'
);
globals.logger.debug(`EVENT COUNT INFLUXDB V3: Wrote user event data to InfluxDB v3`);
}
globals.logger.verbose(
'EVENT COUNT INFLUXDB V3: Sent Butler SOS event count data to InfluxDB'
);
} catch (err) {
globals.logger.error(
`EVENT COUNT INFLUXDB V3: Error writing data to InfluxDB: ${globals.getErrorMessage(err)}`
);
}
}
/**
* Store rejected event count in InfluxDB v3
*
* @description
* This function reads an array of rejected log events from the `rejectedEvents` object,
* and stores the data in InfluxDB v3. The data is written to a measurement named after
* the `Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName` config setting.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
* @throws {Error} Error if unable to write data to InfluxDB
*/
export async function storeRejectedEventCountInfluxDBV3() {
// Get array of rejected log events
const rejectedLogEvents = await globals.rejectedEvents.getRejectedLogEvents();
// Debug
globals.logger.debug(
`REJECTED EVENT COUNT INFLUXDB V3: Rejected log events: ${JSON.stringify(
rejectedLogEvents,
null,
2
)}`
);
// Are there any events to store?
if (rejectedLogEvents.length === 0) {
globals.logger.verbose('REJECTED EVENT COUNT INFLUXDB V3: No events to store in InfluxDB');
return;
}
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
try {
const points = [];
const measurementName = globals.config.get(
'Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName'
);
rejectedLogEvents.forEach((event) => {
globals.logger.debug(`REJECTED LOG EVENT INFLUXDB V3: ${JSON.stringify(event)}`);
if (event.source === 'qseow-qix-perf') {
let point = new Point3(measurementName)
.setTag('source', event.source)
.setTag('object_type', event.objectType)
.setTag('method', event.method)
.setIntegerField('counter', event.counter)
.setFloatField('process_time', event.processTime);
// Add app_id and app_name if available
if (event?.appId) {
point.setTag('app_id', event.appId);
}
if (event?.appName?.length > 0) {
point.setTag('app_name', event.appName);
point.setTag('app_name_set', 'true');
} else {
point.setTag('app_name_set', 'false');
}
// Add static tags defined in config file, if any
if (
globals.config.has(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
) &&
Array.isArray(
globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
)
)
) {
const configTags = globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
);
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
points.push(point);
} else {
let point = new Point3(measurementName)
.setTag('source', event.source)
.setIntegerField('counter', event.counter);
points.push(point);
}
});
// Write to InfluxDB
for (const point of points) {
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
'Rejected event counts'
);
}
globals.logger.debug(`REJECT LOG EVENT INFLUXDB V3: Wrote data to InfluxDB v3`);
globals.logger.verbose(
'REJECT LOG EVENT INFLUXDB V3: Sent Butler SOS rejected event count data to InfluxDB'
);
} catch (err) {
globals.logger.error(
`REJECTED LOG EVENT INFLUXDB V3: Error writing data to InfluxDB: ${globals.getErrorMessage(err)}`
);
}
}

211
src/lib/influxdb/v3/health-metrics.js Normal file
View File

@@ -0,0 +1,211 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import {
getFormattedTime,
processAppDocuments,
isInfluxDbEnabled,
applyTagsToPoint3,
writeToInfluxV3WithRetry,
} from '../shared/utils.js';
/**
* Posts health metrics data from Qlik Sense to InfluxDB v3.
*
* This function processes health data from the Sense engine's healthcheck API and
* formats it for storage in InfluxDB v3. It handles various metrics including:
* - CPU usage
* - Memory usage
* - Cache metrics
* - Active/loaded/in-memory apps
* - Session counts
* - User counts
*
* @param {string} serverName - The name of the Qlik Sense server
* @param {string} host - The hostname or IP of the Qlik Sense server
* @param {object} body - The health metrics data from Sense engine healthcheck API
* @param {object} serverTags - Tags to associate with the metrics
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postHealthMetricsToInfluxdbV3(serverName, host, body, serverTags) {
// Calculate server uptime
const formattedTime = getFormattedTime(body.started);
// Build tags structure that will be passed to InfluxDB
globals.logger.debug(
`HEALTH METRICS TO INFLUXDB V3: Health data: Tags sent to InfluxDB: ${JSON.stringify(
serverTags
)}`
);
globals.logger.debug(
`HEALTH METRICS TO INFLUXDB V3: Number of apps active: ${body.apps.active_docs.length}`
);
globals.logger.debug(
`HEALTH METRICS TO INFLUXDB V3: Number of apps loaded: ${body.apps.loaded_docs.length}`
);
globals.logger.debug(
`HEALTH METRICS TO INFLUXDB V3: Number of apps in memory: ${body.apps.in_memory_docs.length}`
);
// Get active app names
const { appNames: appNamesActive, sessionAppNames: sessionAppNamesActive } =
await processAppDocuments(body.apps.active_docs, 'HEALTH METRICS TO INFLUXDB V3', 'active');
// Get loaded app names
const { appNames: appNamesLoaded, sessionAppNames: sessionAppNamesLoaded } =
await processAppDocuments(body.apps.loaded_docs, 'HEALTH METRICS TO INFLUXDB V3', 'loaded');
// Get in memory app names
const { appNames: appNamesInMemory, sessionAppNames: sessionAppNamesInMemory } =
await processAppDocuments(
body.apps.in_memory_docs,
'HEALTH METRICS TO INFLUXDB V3',
'in memory'
);
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
// Only write to InfluxDB if the global influxWriteApi object has been initialized
if (!globals.influxWriteApi) {
globals.logger.warn(
'HEALTH METRICS V3: Influxdb write API object not initialized. Data will not be sent to InfluxDB'
);
return;
}
// Find writeApi for the server specified by serverName
const writeApi = globals.influxWriteApi.find((element) => element.serverName === serverName);
// Ensure that the writeApi object was found
if (!writeApi) {
globals.logger.warn(
`HEALTH METRICS V3: Influxdb write API object not found for host ${host}. Data will not be sent to InfluxDB`
);
return;
}
// Get database from config
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Create a new point with the data to be written to InfluxDB v3
const points = [
new Point3('sense_server')
.setStringField('version', body.version)
.setStringField('started', body.started)
.setStringField('uptime', formattedTime),
new Point3('mem')
.setFloatField('comitted', body.mem.committed)
.setFloatField('allocated', body.mem.allocated)
.setFloatField('free', body.mem.free),
new Point3('apps')
.setIntegerField('active_docs_count', body.apps.active_docs.length)
.setIntegerField('loaded_docs_count', body.apps.loaded_docs.length)
.setIntegerField('in_memory_docs_count', body.apps.in_memory_docs.length)
.setStringField(
'active_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? body.apps.active_docs
: ''
)
.setStringField(
'active_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? appNamesActive.toString()
: ''
)
.setStringField(
'active_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? sessionAppNamesActive.toString()
: ''
)
.setStringField(
'loaded_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? body.apps.loaded_docs
: ''
)
.setStringField(
'loaded_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? appNamesLoaded.toString()
: ''
)
.setStringField(
'loaded_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? sessionAppNamesLoaded.toString()
: ''
)
.setStringField(
'in_memory_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? body.apps.in_memory_docs
: ''
)
.setStringField(
'in_memory_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? appNamesInMemory.toString()
: ''
)
.setStringField(
'in_memory_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? sessionAppNamesInMemory.toString()
: ''
)
.setIntegerField('calls', body.apps.calls)
.setIntegerField('selections', body.apps.selections),
new Point3('cpu').setIntegerField('total', body.cpu.total),
new Point3('session')
.setIntegerField('active', body.session.active)
.setIntegerField('total', body.session.total),
new Point3('users')
.setIntegerField('active', body.users.active)
.setIntegerField('total', body.users.total),
new Point3('cache')
.setIntegerField('hits', body.cache.hits)
.setIntegerField('lookups', body.cache.lookups)
.setIntegerField('added', body.cache.added)
.setIntegerField('replaced', body.cache.replaced)
.setIntegerField('bytes_added', body.cache.bytes_added),
new Point3('saturated').setBooleanField('saturated', body.saturated),
];
// Write to InfluxDB
try {
for (const point of points) {
// Apply server tags to each point
applyTagsToPoint3(point, serverTags);
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
`Health metrics for ${host}`
);
}
globals.logger.debug(`HEALTH METRICS V3: Wrote data to InfluxDB v3`);
} catch (err) {
// Track error count
await globals.errorTracker.incrementError('INFLUXDB_V3_WRITE', serverName);
globals.logger.error(
`HEALTH METRICS V3: Error saving health data to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
}
}

211
src/lib/influxdb/v3/log-events.js Normal file
View File

@@ -0,0 +1,211 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import { isInfluxDbEnabled, writeToInfluxV3WithRetry } from '../shared/utils.js';
/**
* Post log event to InfluxDB v3
*
* @description
* Handles log events from 5 different Qlik Sense sources:
* - qseow-engine: Engine log events
* - qseow-proxy: Proxy log events
* - qseow-scheduler: Scheduler log events
* - qseow-repository: Repository log events
* - qseow-qix-perf: QIX performance metrics
*
* Each source has specific fields and tags that are written to InfluxDB.
*
* @param {object} msg - The log event message
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
* @throws {Error} Error if unable to write data to InfluxDB
*/
export async function postLogEventToInfluxdbV3(msg) {
globals.logger.debug(`LOG EVENT INFLUXDB V3: ${msg})`);
try {
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
// Verify the message source is valid
if (
msg.source !== 'qseow-engine' &&
msg.source !== 'qseow-proxy' &&
msg.source !== 'qseow-scheduler' &&
msg.source !== 'qseow-repository' &&
msg.source !== 'qseow-qix-perf'
) {
globals.logger.warn(
`LOG EVENT INFLUXDB V3: Unknown log event source: ${msg.source}. Skipping.`
);
return;
}
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
let point;
// Handle each message type with its specific fields
if (msg.source === 'qseow-engine') {
// Engine fields: message, exception_message, command, result_code_field, origin, context, session_id, raw_event
// NOTE: result_code uses _field suffix to avoid conflict with result_code tag
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem || 'n/a')
.setStringField('message', msg.message)
.setStringField('exception_message', msg.exception_message || '')
.setStringField('command', msg.command || '')
.setStringField('result_code_field', msg.result_code || '')
.setStringField('origin', msg.origin || '')
.setStringField('context', msg.context || '')
.setStringField('session_id', msg.session_id || '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.setTag('result_code', msg.result_code);
if (msg?.windows_user?.length > 0) point.setTag('windows_user', msg.windows_user);
if (msg?.task_id?.length > 0) point.setTag('task_id', msg.task_id);
if (msg?.task_name?.length > 0) point.setTag('task_name', msg.task_name);
if (msg?.app_id?.length > 0) point.setTag('app_id', msg.app_id);
if (msg?.app_name?.length > 0) point.setTag('app_name', msg.app_name);
if (msg?.engine_exe_version?.length > 0)
point.setTag('engine_exe_version', msg.engine_exe_version);
} else if (msg.source === 'qseow-proxy') {
// Proxy fields: message, exception_message, command, result_code_field, origin, context, raw_event
// NOTE: result_code uses _field suffix to avoid conflict with result_code tag
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem || 'n/a')
.setStringField('message', msg.message)
.setStringField('exception_message', msg.exception_message || '')
.setStringField('command', msg.command || '')
.setStringField('result_code_field', msg.result_code || '')
.setStringField('origin', msg.origin || '')
.setStringField('context', msg.context || '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.setTag('result_code', msg.result_code);
} else if (msg.source === 'qseow-scheduler') {
// Scheduler fields: message, exception_message, app_name_field, app_id_field, execution_id, raw_event
// NOTE: app_name and app_id use _field suffix to avoid conflict with conditional tags
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem || 'n/a')
.setStringField('message', msg.message)
.setStringField('exception_message', msg.exception_message || '')
.setStringField('app_name_field', msg.app_name || '')
.setStringField('app_id_field', msg.app_id || '')
.setStringField('execution_id', msg.execution_id || '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.task_id?.length > 0) point.setTag('task_id', msg.task_id);
if (msg?.task_name?.length > 0) point.setTag('task_name', msg.task_name);
} else if (msg.source === 'qseow-repository') {
// Repository fields: message, exception_message, command, result_code_field, origin, context, raw_event
// NOTE: result_code uses _field suffix to avoid conflict with result_code tag
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem || 'n/a')
.setStringField('message', msg.message)
.setStringField('exception_message', msg.exception_message || '')
.setStringField('command', msg.command || '')
.setStringField('result_code_field', msg.result_code || '')
.setStringField('origin', msg.origin || '')
.setStringField('context', msg.context || '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.setTag('result_code', msg.result_code);
} else if (msg.source === 'qseow-qix-perf') {
// QIX Performance fields: app_id, process_time, work_time, lock_time, validate_time, traverse_time, handle, net_ram, peak_ram, raw_event
point = new Point3('log_event')
.setTag('host', msg.host || '<Unknown>')
.setTag('level', msg.level || '<Unknown>')
.setTag('source', msg.source || '<Unknown>')
.setTag('log_row', msg.log_row || '-1')
.setTag('subsystem', msg.subsystem || '<Unknown>')
.setTag('method', msg.method || '<Unknown>')
.setTag('object_type', msg.object_type || '<Unknown>')
.setTag('proxy_session_id', msg.proxy_session_id || '-1')
.setTag('session_id', msg.session_id || '-1')
.setTag('event_activity_source', msg.event_activity_source || '<Unknown>')
.setStringField('app_id_field', msg.app_id || '')
.setFloatField('process_time', msg.process_time)
.setFloatField('work_time', msg.work_time)
.setFloatField('lock_time', msg.lock_time)
.setFloatField('validate_time', msg.validate_time)
.setFloatField('traverse_time', msg.traverse_time)
.setIntegerField('handle', msg.handle)
.setIntegerField('net_ram', msg.net_ram)
.setIntegerField('peak_ram', msg.peak_ram)
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0) point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.app_id?.length > 0) point.setTag('app_id', msg.app_id);
if (msg?.app_name?.length > 0) point.setTag('app_name', msg.app_name);
if (msg?.object_id?.length > 0) point.setTag('object_id', msg.object_id);
}
// Add log event categories to tags if available
// The msg.category array contains objects with properties 'name' and 'value'
if (msg?.category?.length > 0) {
msg.category.forEach((category) => {
point.setTag(category.name, category.value);
});
}
// Add custom tags from config file
if (
globals.config.has('Butler-SOS.logEvents.tags') &&
globals.config.get('Butler-SOS.logEvents.tags') !== null &&
globals.config.get('Butler-SOS.logEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.logEvents.tags');
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
`Log event for ${msg.host}`
);
globals.logger.debug(`LOG EVENT INFLUXDB V3: Wrote data to InfluxDB v3`);
globals.logger.verbose('LOG EVENT INFLUXDB V3: Sent Butler SOS log event data to InfluxDB');
} catch (err) {
globals.logger.error(
`LOG EVENT INFLUXDB V3: Error saving log event to InfluxDB! ${globals.getErrorMessage(err)}`
);
}
}

187
src/lib/influxdb/v3/queue-metrics.js Normal file
View File

@@ -0,0 +1,187 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import { isInfluxDbEnabled, writeToInfluxV3WithRetry } from '../shared/utils.js';
/**
* Store user event queue metrics to InfluxDB v3
*
* @description
* Retrieves metrics from the user event queue manager and stores them in InfluxDB v3
* for monitoring queue health, backpressure, dropped messages, and processing performance.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
* @throws {Error} Error if unable to write data to InfluxDB
*/
export async function postUserEventQueueMetricsToInfluxdbV3() {
try {
// Check if queue metrics are enabled
if (
!globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.enable'
)
) {
return;
}
// Get metrics from queue manager
const queueManager = globals.udpQueueManagerUserActivity;
if (!queueManager) {
globals.logger.warn(
'USER EVENT QUEUE METRICS INFLUXDB V3: Queue manager not initialized'
);
return;
}
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
const metrics = await queueManager.getMetrics();
// Get configuration
const measurementName = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.measurementName'
);
const configTags = globals.config.get(
'Butler-SOS.userEvents.udpServerConfig.queueMetrics.influxdb.tags'
);
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
const point = new Point3(measurementName)
.setTag('queue_type', 'user_events')
.setTag('host', globals.hostInfo.hostname)
.setIntegerField('queue_size', metrics.queueSize)
.setIntegerField('queue_max_size', metrics.queueMaxSize)
.setFloatField('queue_utilization_pct', metrics.queueUtilizationPct)
.setIntegerField('queue_pending', metrics.queuePending)
.setIntegerField('messages_received', metrics.messagesReceived)
.setIntegerField('messages_queued', metrics.messagesQueued)
.setIntegerField('messages_processed', metrics.messagesProcessed)
.setIntegerField('messages_failed', metrics.messagesFailed)
.setIntegerField('messages_dropped_total', metrics.messagesDroppedTotal)
.setIntegerField('messages_dropped_rate_limit', metrics.messagesDroppedRateLimit)
.setIntegerField('messages_dropped_queue_full', metrics.messagesDroppedQueueFull)
.setIntegerField('messages_dropped_size', metrics.messagesDroppedSize)
.setFloatField('processing_time_avg_ms', metrics.processingTimeAvgMs)
.setFloatField('processing_time_p95_ms', metrics.processingTimeP95Ms)
.setFloatField('processing_time_max_ms', metrics.processingTimeMaxMs)
.setIntegerField('rate_limit_current', metrics.rateLimitCurrent)
.setIntegerField('backpressure_active', metrics.backpressureActive);
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
'User event queue metrics'
);
globals.logger.verbose(
'USER EVENT QUEUE METRICS INFLUXDB V3: Sent queue metrics data to InfluxDB v3'
);
// Clear metrics after writing
await queueManager.clearMetrics();
} catch (err) {
globals.logger.error(
`USER EVENT QUEUE METRICS INFLUXDB V3: Error posting queue metrics: ${globals.getErrorMessage(err)}`
);
}
}
/**
* Store log event queue metrics to InfluxDB v3
*
* @description
* Retrieves metrics from the log event queue manager and stores them in InfluxDB v3
* for monitoring queue health, backpressure, dropped messages, and processing performance.
*
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
* @throws {Error} Error if unable to write data to InfluxDB
*/
export async function postLogEventQueueMetricsToInfluxdbV3() {
try {
// Check if queue metrics are enabled
if (
!globals.config.get('Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.enable')
) {
return;
}
// Get metrics from queue manager
const queueManager = globals.udpQueueManagerLogEvents;
if (!queueManager) {
globals.logger.warn(
'LOG EVENT QUEUE METRICS INFLUXDB V3: Queue manager not initialized'
);
return;
}
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
const metrics = await queueManager.getMetrics();
// Get configuration
const measurementName = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.measurementName'
);
const configTags = globals.config.get(
'Butler-SOS.logEvents.udpServerConfig.queueMetrics.influxdb.tags'
);
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
const point = new Point3(measurementName)
.setTag('queue_type', 'log_events')
.setTag('host', globals.hostInfo.hostname)
.setIntegerField('queue_size', metrics.queueSize)
.setIntegerField('queue_max_size', metrics.queueMaxSize)
.setFloatField('queue_utilization_pct', metrics.queueUtilizationPct)
.setIntegerField('queue_pending', metrics.queuePending)
.setIntegerField('messages_received', metrics.messagesReceived)
.setIntegerField('messages_queued', metrics.messagesQueued)
.setIntegerField('messages_processed', metrics.messagesProcessed)
.setIntegerField('messages_failed', metrics.messagesFailed)
.setIntegerField('messages_dropped_total', metrics.messagesDroppedTotal)
.setIntegerField('messages_dropped_rate_limit', metrics.messagesDroppedRateLimit)
.setIntegerField('messages_dropped_queue_full', metrics.messagesDroppedQueueFull)
.setIntegerField('messages_dropped_size', metrics.messagesDroppedSize)
.setFloatField('processing_time_avg_ms', metrics.processingTimeAvgMs)
.setFloatField('processing_time_p95_ms', metrics.processingTimeP95Ms)
.setFloatField('processing_time_max_ms', metrics.processingTimeMaxMs)
.setIntegerField('rate_limit_current', metrics.rateLimitCurrent)
.setIntegerField('backpressure_active', metrics.backpressureActive);
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
'Log event queue metrics'
);
globals.logger.verbose(
'LOG EVENT QUEUE METRICS INFLUXDB V3: Sent queue metrics data to InfluxDB v3'
);
// Clear metrics after writing
await queueManager.clearMetrics();
} catch (err) {
globals.logger.error(
`LOG EVENT QUEUE METRICS INFLUXDB V3: Error posting queue metrics: ${globals.getErrorMessage(err)}`
);
}
}

73
src/lib/influxdb/v3/sessions.js Normal file
View File

@@ -0,0 +1,73 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import { isInfluxDbEnabled, writeToInfluxV3WithRetry } from '../shared/utils.js';
/**
* Posts proxy sessions data to InfluxDB v3.
*
* This function takes user session data from Qlik Sense proxy and formats it for storage
* in InfluxDB v3. It creates three measurements:
* - user_session_summary: Summary with count and user list
* - user_session_list: List of users (for compatibility)
* - user_session_details: Individual session details for each active session
*
* @param {object} userSessions - User session data containing information about active sessions
* @param {string} userSessions.host - The hostname of the server
* @param {string} userSessions.virtualProxy - The virtual proxy name
* @param {string} userSessions.serverName - Server name
* @param {number} userSessions.sessionCount - Number of sessions
* @param {string} userSessions.uniqueUserList - Comma-separated list of unique users
* @param {Array} userSessions.datapointInfluxdb - Array of datapoints including individual sessions
* @returns {Promise<void>} Promise that resolves when data has been posted to InfluxDB
*/
export async function postProxySessionsToInfluxdbV3(userSessions) {
globals.logger.debug(`PROXY SESSIONS V3: User sessions: ${JSON.stringify(userSessions)}`);
globals.logger.silly(
`PROXY SESSIONS V3: Data for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"`
);
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
// Get database from config
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Write all datapoints to InfluxDB
// The datapointInfluxdb array contains summary points and individual session details
try {
if (userSessions.datapointInfluxdb && userSessions.datapointInfluxdb.length > 0) {
for (const point of userSessions.datapointInfluxdb) {
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
`Proxy sessions for ${userSessions.host}/${userSessions.virtualProxy}`
);
}
globals.logger.debug(
`PROXY SESSIONS V3: Wrote ${userSessions.datapointInfluxdb.length} datapoints to InfluxDB v3`
);
} else {
globals.logger.warn('PROXY SESSIONS V3: No datapoints to write to InfluxDB v3');
}
} catch (err) {
// Track error count
await globals.errorTracker.incrementError('INFLUXDB_V3_WRITE', userSessions.serverName);
globals.logger.error(
`PROXY SESSIONS V3: Error saving user session data to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
}
globals.logger.debug(
`PROXY SESSIONS V3: Session count for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${userSessions.sessionCount}`
);
globals.logger.debug(
`PROXY SESSIONS V3: User list for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${userSessions.uniqueUserList}`
);
globals.logger.verbose(
`PROXY SESSIONS V3: Sent user session data to InfluxDB for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"`
);
}

125
src/lib/influxdb/v3/user-events.js Normal file
View File

@@ -0,0 +1,125 @@
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../../../globals.js';
import { isInfluxDbEnabled, writeToInfluxV3WithRetry } from '../shared/utils.js';
/**
* Sanitize tag values for InfluxDB line protocol.
* Remove or replace characters that cause parsing issues.
*
* @param {string} value - The value to sanitize
* @returns {string} - The sanitized value
*/
function sanitizeTagValue(value) {
if (!value) return value;
return String(value)
.replace(/[<>\\]/g, '')
.replace(/\s+/g, '-');
}
/**
* Posts a user event to InfluxDB v3.
*
* @param {object} msg - The event to be posted to InfluxDB. The object should contain the following properties:
* - host: The hostname of the Qlik Sense server that the user event originated from.
* - command: The command (e.g. OpenApp, CreateApp, etc.) that the user event corresponds to.
* - user_directory: The user directory of the user who triggered the event.
* - user_id: The user ID of the user who triggered the event.
* - origin: The origin of the event (e.g. Qlik Sense, QlikView, etc.).
* - appId: The ID of the app that the event corresponds to (if applicable).
* - appName: The name of the app that the event corresponds to (if applicable).
* - ua: An object containing user agent information (if available).
* @returns {Promise<void>} - A promise that resolves when the event has been posted to InfluxDB.
*/
export async function postUserEventToInfluxdbV3(msg) {
globals.logger.debug(`USER EVENT INFLUXDB V3: ${msg})`);
// Only write to InfluxDB if the global influx object has been initialized
if (!isInfluxDbEnabled()) {
return;
}
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Validate required fields
if (!msg.host || !msg.command || !msg.user_directory || !msg.user_id || !msg.origin) {
globals.logger.warn(
`USER EVENT INFLUXDB V3: Missing required fields in user event message: ${JSON.stringify(msg)}`
);
return;
}
// Create a new point with the data to be written to InfluxDB v3
// NOTE: InfluxDB v3 does not allow the same name for both tags and fields,
// unlike v1/v2. Fields use different names with _field suffix where needed.
const point = new Point3('user_events')
.setTag('host', msg.host)
.setTag('event_action', msg.command)
.setTag('userFull', `${msg.user_directory}\\${msg.user_id}`)
.setTag('userDirectory', msg.user_directory)
.setTag('userId', msg.user_id)
.setTag('origin', msg.origin)
.setStringField('userFull_field', `${msg.user_directory}\\${msg.user_id}`)
.setStringField('userId_field', msg.user_id);
// Add app id and name to tags and fields if available
if (msg?.appId) {
point.setTag('appId', msg.appId);
point.setStringField('appId_field', msg.appId);
}
if (msg?.appName) {
point.setTag('appName', msg.appName);
point.setStringField('appName_field', msg.appName);
}
// Add user agent info to tags if available
if (msg?.ua?.browser?.name) point.setTag('uaBrowserName', msg?.ua?.browser?.name);
if (msg?.ua?.browser?.major) point.setTag('uaBrowserMajorVersion', msg?.ua?.browser?.major);
if (msg?.ua?.os?.name) point.setTag('uaOsName', msg?.ua?.os?.name);
if (msg?.ua?.os?.version) point.setTag('uaOsVersion', msg?.ua?.os?.version);
// Add custom tags from config file to payload
if (
globals.config.has('Butler-SOS.userEvents.tags') &&
globals.config.get('Butler-SOS.userEvents.tags') !== null &&
globals.config.get('Butler-SOS.userEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.userEvents.tags');
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
globals.logger.silly(
`USER EVENT INFLUXDB V3: Influxdb datapoint for Butler SOS user event: ${JSON.stringify(
point,
null,
2
)}`
);
// Write to InfluxDB
try {
// Convert point to line protocol and write directly with retry logic
await writeToInfluxV3WithRetry(
async () => await globals.influx.write(point.toLineProtocol(), database),
`User event for ${msg.host}`
);
globals.logger.debug(`USER EVENT INFLUXDB V3: Wrote data to InfluxDB v3`);
} catch (err) {
// Track error count
await globals.errorTracker.incrementError('INFLUXDB_V3_WRITE', '');
globals.logger.error(
`USER EVENT INFLUXDB V3: Error saving user event to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
// Log the line protocol for debugging
try {
const lineProtocol = point.toLineProtocol();
globals.logger.debug(`USER EVENT INFLUXDB V3: Failed line protocol: ${lineProtocol}`);
} catch (e) {
// Ignore errors in debug logging
}
}
globals.logger.verbose('USER EVENT INFLUXDB V3: Sent Butler SOS user event data to InfluxDB');
}

135
src/lib/log-error.js Normal file
View File

@@ -0,0 +1,135 @@
/**
* Enhanced error logging utility for Butler SOS
*
* Provides consistent error logging across the application with different
* behavior for SEA (Single Executable Application) vs non-SEA environments.
*
* In SEA mode: Only the error message is logged (cleaner output for end users)
* In non-SEA mode: Both error message and stack trace are logged as separate
* entries (better debugging for developers)
*/
import globals from '../globals.js';
import sea from './sea-wrapper.js';
/**
* Log an error with appropriate formatting based on execution environment
*
* This function wraps the global logger and provides enhanced error logging:
* - In SEA apps: logs only the error message (cleaner for production)
* - In non-SEA apps: logs error message and stack trace separately (better for debugging)
*
* The function accepts the same parameters as winston logger methods.
*
* @param {string} level - The log level ('error', 'warn', 'info', 'verbose', 'debug')
* @param {string} message - The log message (prefix/context for the error)
* @param {Error} error - The error object to log
* @param {...unknown} args - Additional arguments to pass to the logger
*
* @example
* // Basic error logging
* try {
* // some code
* } catch (err) {
* logError('HEALTH: Error when calling health check API', err);
* }
*
* @example
* // With contextual information
* try {
* // some code
* } catch (err) {
* logError(`PROXY SESSIONS: Error for server '${serverName}' (${host})`, err);
* }
*/
function logErrorWithLevel(level, message, error, ...args) {
// Check if running as SEA app
const isSeaApp = globals.isSea !== undefined ? globals.isSea : sea.isSea();
if (!error) {
// If no error object provided, just log the message normally
globals.logger[level](message, ...args);
return;
}
// Get error message - prefer error.message, fallback to toString()
const errorMessage = error.message || error.toString();
if (isSeaApp) {
// SEA mode: Only log the error message (cleaner output)
globals.logger[level](`${message}: ${errorMessage}`, ...args);
} else {
// Non-SEA mode: Log error message first, then stack trace separately
// This provides better readability and debugging information
// Log 1: The error message with context
globals.logger[level](`${message}: ${errorMessage}`, ...args);
// Log 2: The stack trace (if available)
if (error.stack) {
globals.logger[level](`Stack trace: ${error.stack}`, ...args);
}
}
}
/**
* Convenience function for logging errors at 'error' level
*
* @param {string} message - The log message (prefix/context for the error)
* @param {Error} error - The error object to log
* @param {...unknown} args - Additional arguments to pass to the logger
*
* @example
* try {
* // some code
* } catch (err) {
* logError('HEALTH: Error when calling health check API', err);
* }
*/
export function logError(message, error, ...args) {
logErrorWithLevel('error', message, error, ...args);
}
/**
* Convenience function for logging errors at 'warn' level
*
* @param {string} message - The log message (prefix/context for the error)
* @param {Error} error - The error object to log
* @param {...unknown} args - Additional arguments to pass to the logger
*/
export function logWarn(message, error, ...args) {
logErrorWithLevel('warn', message, error, ...args);
}
/**
* Convenience function for logging errors at 'info' level
*
* @param {string} message - The log message (prefix/context for the error)
* @param {Error} error - The error object to log
* @param {...unknown} args - Additional arguments to pass to the logger
*/
export function logInfo(message, error, ...args) {
logErrorWithLevel('info', message, error, ...args);
}
/**
* Convenience function for logging errors at 'verbose' level
*
* @param {string} message - The log message (prefix/context for the error)
* @param {Error} error - The error object to log
* @param {...unknown} args - Additional arguments to pass to the logger
*/
export function logVerbose(message, error, ...args) {
logErrorWithLevel('verbose', message, error, ...args);
}
/**
* Convenience function for logging errors at 'debug' level
*
* @param {string} message - The log message (prefix/context for the error)
* @param {Error} error - The error object to log
* @param {...unknown} args - Additional arguments to pass to the logger
*/
export function logDebug(message, error, ...args) {
logErrorWithLevel('debug', message, error, ...args);
}

3
src/lib/log-event-categorise.js
View File

@@ -1,4 +1,5 @@
import globals from '../globals.js';
import { logError } from './log-error.js';
/**
* Categorizes log events based on configured rules.
@@ -118,7 +119,7 @@ export function categoriseLogEvent(logLevel, logMessage) {
// Return the log event category and the action taken
return { category: uniqueCategories, actionTaken: 'categorised' };
} catch (err) {
globals.logger.error(`LOG EVENT CATEGORISATION: Error processing log event: ${err}`);
logError('LOG EVENT CATEGORISATION: Error processing log event', err);
return null;
}
}

740
src/lib/post-to-influxdb.js
View File

@@ -1,6 +1,8 @@
import { Point } from '@influxdata/influxdb-client';
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../globals.js';
import { logError } from './log-error.js';
const sessionAppPrefix = 'SessionApp';
const MIN_TIMESTAMP_LENGTH = 15;
@@ -546,6 +548,141 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server
`HEALTH METRICS: Error saving health data to InfluxDB v2! ${globals.getErrorMessage(err)}`
);
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Only write to InfluxDB if the global influxWriteApi object has been initialized
if (!globals.influxWriteApi) {
globals.logger.warn(
'HEALTH METRICS: Influxdb write API object not initialized. Data will not be sent to InfluxDB'
);
return;
}
// Find writeApi for the server specified by serverName
const writeApi = globals.influxWriteApi.find(
(element) => element.serverName === serverName
);
// Ensure that the writeApi object was found
if (!writeApi) {
globals.logger.warn(
`HEALTH METRICS: Influxdb write API object not found for host ${host}. Data will not be sent to InfluxDB`
);
return;
}
// Get database from config
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Create a new point with the data to be written to InfluxDB v3
const points = [
new Point3('sense_server')
.setStringField('version', body.version)
.setStringField('started', body.started)
.setStringField('uptime', formattedTime),
new Point3('mem')
.setFloatField('comitted', body.mem.committed)
.setFloatField('allocated', body.mem.allocated)
.setFloatField('free', body.mem.free),
new Point3('apps')
.setIntegerField('active_docs_count', body.apps.active_docs.length)
.setIntegerField('loaded_docs_count', body.apps.loaded_docs.length)
.setIntegerField('in_memory_docs_count', body.apps.in_memory_docs.length)
.setStringField(
'active_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? body.apps.active_docs
: ''
)
.setStringField(
'active_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? appNamesActive.toString()
: ''
)
.setStringField(
'active_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.activeDocs')
? sessionAppNamesActive.toString()
: ''
)
.setStringField(
'loaded_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? body.apps.loaded_docs
: ''
)
.setStringField(
'loaded_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? appNamesLoaded.toString()
: ''
)
.setStringField(
'loaded_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.loadedDocs')
? sessionAppNamesLoaded.toString()
: ''
)
.setStringField(
'in_memory_docs',
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? body.apps.in_memory_docs
: ''
)
.setStringField(
'in_memory_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? appNamesInMemory.toString()
: ''
)
.setStringField(
'in_memory_session_docs_names',
globals.config.get('Butler-SOS.appNames.enableAppNameExtract') &&
globals.config.get('Butler-SOS.influxdbConfig.includeFields.inMemoryDocs')
? sessionAppNamesInMemory.toString()
: ''
)
.setIntegerField('calls', body.apps.calls)
.setIntegerField('selections', body.apps.selections),
new Point3('cpu').setIntegerField('total', body.cpu.total),
new Point3('session')
.setIntegerField('active', body.session.active)
.setIntegerField('total', body.session.total),
new Point3('users')
.setIntegerField('active', body.users.active)
.setIntegerField('total', body.users.total),
new Point3('cache')
.setIntegerField('hits', body.cache.hits)
.setIntegerField('lookups', body.cache.lookups)
.setIntegerField('added', body.cache.added)
.setIntegerField('replaced', body.cache.replaced)
.setIntegerField('bytes_added', body.cache.bytes_added),
new Point3('saturated').setBooleanField('saturated', body.saturated),
];
// Write to InfluxDB
try {
for (const point of points) {
await globals.influx.write(point.toLineProtocol(), database);
}
globals.logger.debug(`HEALTH METRICS: Wrote data to InfluxDB v3`);
} catch (err) {
globals.logger.error(
`HEALTH METRICS: Error saving health data to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
}
}
}
@@ -640,6 +777,56 @@ export async function postProxySessionsToInfluxdb(userSessions) {
);
}
globals.logger.verbose(
`PROXY SESSIONS: Sent user session data to InfluxDB for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"`
);
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Only write to InfluxDB if the global influxWriteApi object has been initialized
if (!globals.influxWriteApi) {
globals.logger.warn(
'PROXY SESSIONS: Influxdb write API object not initialized. Data will not be sent to InfluxDB'
);
return;
}
// Find writeApi for the specified server
const writeApi = globals.influxWriteApi.find(
(element) => element.serverName === userSessions.serverName
);
// Ensure that the writeApi object was found
if (!writeApi) {
globals.logger.warn(
`PROXY SESSIONS: Influxdb v3 write API object not found for host ${userSessions.host}. Data will not be sent to InfluxDB`
);
return;
}
// Get database from config
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// Create data points
const point = new Point3('user_session_summary')
.setIntegerField('session_count', userSessions.sessionCount)
.setStringField('session_user_id_list', userSessions.uniqueUserList);
// Write to InfluxDB
try {
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.debug(`PROXY SESSIONS: Wrote data to InfluxDB v3`);
} catch (err) {
globals.logger.error(
`PROXY SESSIONS: Error saving user session data to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
}
globals.logger.debug(
`PROXY SESSIONS: Session count for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"": ${userSessions.sessionCount}`
);
globals.logger.debug(
`PROXY SESSIONS: User list for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"": ${userSessions.uniqueUserList}`
);
globals.logger.verbose(
`PROXY SESSIONS: Sent user session data to InfluxDB for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}"`
);
@@ -776,6 +963,45 @@ export async function postButlerSOSMemoryUsageToInfluxdb(memory) {
);
}
globals.logger.verbose(
'MEMORY USAGE INFLUXDB: Sent Butler SOS memory usage data to InfluxDB'
);
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Create new write API object
// advanced write options
const writeOptions = {
/* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */
flushInterval: 5000,
/* the count of internally-scheduled retries upon write failure, the delays between write attempts follow an exponential backoff strategy if there is no Retry-After HTTP header */
maxRetries: 2, // do not retry writes
// ... there are more write options that can be customized, see
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html
};
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
// v3 uses client.write() directly, not getWriteApi()
const point = new Point3('butlersos_memory_usage')
.setTag('butler_sos_instance', memory.instanceTag)
.setTag('version', butlerVersion)
.setFloatField('heap_used', memory.heapUsedMByte)
.setFloatField('heap_total', memory.heapTotalMByte)
.setFloatField('external', memory.externalMemoryMByte)
.setFloatField('process_memory', memory.processMemoryMByte);
try {
// Convert point to line protocol and write directly
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.debug(`MEMORY USAGE INFLUXDB: Wrote data to InfluxDB v3`);
} catch (err) {
globals.logger.error(
`MEMORY USAGE INFLUXDB: Error saving user session data to InfluxDB v3! ${globals.getErrorMessage(err)}`
);
}
globals.logger.verbose(
'MEMORY USAGE INFLUXDB: Sent Butler SOS memory usage data to InfluxDB'
);
@@ -986,6 +1212,39 @@ export async function postUserEventToInfluxdb(msg) {
);
}
globals.logger.verbose(
'USER EVENT INFLUXDB: Sent Butler SOS user event data to InfluxDB'
);
} catch (err) {
globals.logger.error(
`USER EVENT INFLUXDB: Error getting write API: ${globals.getErrorMessage(err)}`
);
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
const point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem ? msg.subsystem : 'n/a')
.setStringField('message', msg.message)
.setStringField('exception_message', msg.exception_message ? msg.exception_message : '')
.setStringField('app_name', msg.appName ? msg.appName : '')
.setStringField('app_id', msg.appId ? msg.appId : '')
.setStringField('execution_id', msg.executionId ? msg.executionId : '')
.setStringField('command', msg.command ? msg.command : '')
.setStringField('result_code', msg.resultCode ? msg.resultCode : '')
.setStringField('origin', msg.origin ? msg.origin : '')
.setStringField('context', msg.context ? msg.context : '')
.setStringField('session_id', msg.sessionId ? msg.sessionId : '')
.setStringField('raw_event', msg.rawEvent ? msg.rawEvent : '');
try {
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.debug(`USER EVENT INFLUXDB: Wrote data to InfluxDB v3`);
globals.logger.verbose(
'USER EVENT INFLUXDB: Sent Butler SOS user event data to InfluxDB'
);
@@ -1449,6 +1708,200 @@ export async function postLogEventToInfluxdb(msg) {
);
}
globals.logger.verbose(
'LOG EVENT INFLUXDB: Sent Butler SOS log event data to InfluxDB'
);
} catch (err) {
globals.logger.error(
`LOG EVENT INFLUXDB: Error getting write API: ${globals.getErrorMessage(err)}`
);
}
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
if (
msg.source === 'qseow-engine' ||
msg.source === 'qseow-proxy' ||
msg.source === 'qseow-scheduler' ||
msg.source === 'qseow-repository' ||
msg.source === 'qseow-qix-perf'
) {
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
let point;
// Handle each message type with its specific fields
if (msg.source === 'qseow-engine') {
// Engine fields: message, exception_message, command, result_code, origin, context, session_id, raw_event
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem ? msg.subsystem : 'n/a')
.setStringField('message', msg.message)
.setStringField(
'exception_message',
msg.exception_message ? msg.exception_message : ''
)
.setStringField('command', msg.command ? msg.command : '')
.setStringField('result_code', msg.result_code ? msg.result_code : '')
.setStringField('origin', msg.origin ? msg.origin : '')
.setStringField('context', msg.context ? msg.context : '')
.setStringField('session_id', msg.session_id ? msg.session_id : '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0)
point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.setTag('result_code', msg.result_code);
if (msg?.windows_user?.length > 0)
point.setTag('windows_user', msg.windows_user);
if (msg?.task_id?.length > 0) point.setTag('task_id', msg.task_id);
if (msg?.task_name?.length > 0) point.setTag('task_name', msg.task_name);
if (msg?.app_id?.length > 0) point.setTag('app_id', msg.app_id);
if (msg?.app_name?.length > 0) point.setTag('app_name', msg.app_name);
if (msg?.engine_exe_version?.length > 0)
point.setTag('engine_exe_version', msg.engine_exe_version);
} else if (msg.source === 'qseow-proxy') {
// Proxy fields: message, exception_message, command, result_code, origin, context, raw_event (NO session_id)
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem ? msg.subsystem : 'n/a')
.setStringField('message', msg.message)
.setStringField(
'exception_message',
msg.exception_message ? msg.exception_message : ''
)
.setStringField('command', msg.command ? msg.command : '')
.setStringField('result_code', msg.result_code ? msg.result_code : '')
.setStringField('origin', msg.origin ? msg.origin : '')
.setStringField('context', msg.context ? msg.context : '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0)
point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.setTag('result_code', msg.result_code);
} else if (msg.source === 'qseow-scheduler') {
// Scheduler fields: message, exception_message, app_name, app_id, execution_id, raw_event
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem ? msg.subsystem : 'n/a')
.setStringField('message', msg.message)
.setStringField(
'exception_message',
msg.exception_message ? msg.exception_message : ''
)
.setStringField('app_name', msg.app_name ? msg.app_name : '')
.setStringField('app_id', msg.app_id ? msg.app_id : '')
.setStringField('execution_id', msg.execution_id ? msg.execution_id : '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0)
point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.task_id?.length > 0) point.setTag('task_id', msg.task_id);
if (msg?.task_name?.length > 0) point.setTag('task_name', msg.task_name);
} else if (msg.source === 'qseow-repository') {
// Repository fields: message, exception_message, command, result_code, origin, context, raw_event
point = new Point3('log_event')
.setTag('host', msg.host)
.setTag('level', msg.level)
.setTag('source', msg.source)
.setTag('log_row', msg.log_row)
.setTag('subsystem', msg.subsystem ? msg.subsystem : 'n/a')
.setStringField('message', msg.message)
.setStringField(
'exception_message',
msg.exception_message ? msg.exception_message : ''
)
.setStringField('command', msg.command ? msg.command : '')
.setStringField('result_code', msg.result_code ? msg.result_code : '')
.setStringField('origin', msg.origin ? msg.origin : '')
.setStringField('context', msg.context ? msg.context : '')
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0)
point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.result_code?.length > 0) point.setTag('result_code', msg.result_code);
} else if (msg.source === 'qseow-qix-perf') {
// QIX Performance fields: app_id, process_time, work_time, lock_time, validate_time, traverse_time, handle, net_ram, peak_ram, raw_event
point = new Point3('log_event')
.setTag('host', msg.host ? msg.host : '<Unknown>')
.setTag('level', msg.level ? msg.level : '<Unknown>')
.setTag('source', msg.source ? msg.source : '<Unknown>')
.setTag('log_row', msg.log_row ? msg.log_row : '-1')
.setTag('subsystem', msg.subsystem ? msg.subsystem : '<Unknown>')
.setTag('method', msg.method ? msg.method : '<Unknown>')
.setTag('object_type', msg.object_type ? msg.object_type : '<Unknown>')
.setTag(
'proxy_session_id',
msg.proxy_session_id ? msg.proxy_session_id : '-1'
)
.setTag('session_id', msg.session_id ? msg.session_id : '-1')
.setTag(
'event_activity_source',
msg.event_activity_source ? msg.event_activity_source : '<Unknown>'
)
.setStringField('app_id', msg.app_id ? msg.app_id : '')
.setFloatField('process_time', msg.process_time)
.setFloatField('work_time', msg.work_time)
.setFloatField('lock_time', msg.lock_time)
.setFloatField('validate_time', msg.validate_time)
.setFloatField('traverse_time', msg.traverse_time)
.setIntegerField('handle', msg.handle)
.setIntegerField('net_ram', msg.net_ram)
.setIntegerField('peak_ram', msg.peak_ram)
.setStringField('raw_event', JSON.stringify(msg));
// Conditional tags
if (msg?.user_full?.length > 0) point.setTag('user_full', msg.user_full);
if (msg?.user_directory?.length > 0)
point.setTag('user_directory', msg.user_directory);
if (msg?.user_id?.length > 0) point.setTag('user_id', msg.user_id);
if (msg?.app_id?.length > 0) point.setTag('app_id', msg.app_id);
if (msg?.app_name?.length > 0) point.setTag('app_name', msg.app_name);
if (msg?.object_id?.length > 0) point.setTag('object_id', msg.object_id);
}
// Add log event categories to tags if available
if (msg?.category?.length > 0) {
msg.category.forEach((category) => {
point.setTag(category.name, category.value);
});
}
// Add custom tags from config file
if (
globals.config.has('Butler-SOS.logEvents.tags') &&
globals.config.get('Butler-SOS.logEvents.tags') !== null &&
globals.config.get('Butler-SOS.logEvents.tags').length > 0
) {
const configTags = globals.config.get('Butler-SOS.logEvents.tags');
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
try {
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.debug(`LOG EVENT INFLUXDB: Wrote data to InfluxDB v3`);
globals.logger.verbose(
'LOG EVENT INFLUXDB: Sent Butler SOS log event data to InfluxDB'
);
@@ -1576,7 +2029,7 @@ export async function storeEventCountInfluxDB() {
try {
globals.influx.writePoints(points);
} catch (err) {
globals.logger.error(`EVENT COUNT INFLUXDB: Error saving data to InfluxDB v1! ${err}`);
logError('EVENT COUNT INFLUXDB: Error saving data to InfluxDB v1!', err);
return;
}
@@ -1698,7 +2151,109 @@ export async function storeEventCountInfluxDB() {
'EVENT COUNT INFLUXDB: Sent Butler SOS event count data to InfluxDB'
);
} catch (err) {
globals.logger.error(`EVENT COUNT INFLUXDB: Error getting write API: ${err}`);
logError('EVENT COUNT INFLUXDB: Error getting write API', err);
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
try {
// Store data for each log event
for (const logEvent of logEvents) {
const tags = {
butler_sos_instance: globals.options.instanceTag,
event_type: 'log',
source: logEvent.source,
host: logEvent.host,
subsystem: logEvent.subsystem,
};
// Add static tags defined in config file, if any
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
Array.isArray(
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags')
)
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
configTags.forEach((tag) => {
tags[tag.name] = tag.value;
});
}
const point = new Point3(
globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName'
)
)
.setTag('event_type', 'log')
.setTag('source', logEvent.source)
.setTag('host', logEvent.host)
.setTag('subsystem', logEvent.subsystem)
.setIntegerField('counter', logEvent.counter);
// Add tags to point
Object.keys(tags).forEach((key) => {
point.setTag(key, tags[key]);
});
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.debug(`EVENT COUNT INFLUXDB: Wrote data to InfluxDB v3`);
}
// Loop through data in user events and create datapoints.
for (const event of userEvents) {
const tags = {
butler_sos_instance: globals.options.instanceTag,
event_type: 'user',
source: event.source,
host: event.host,
subsystem: event.subsystem,
};
// Add static tags defined in config file, if any
if (
globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') &&
Array.isArray(
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags')
)
) {
const configTags = globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags'
);
configTags.forEach((tag) => {
tags[tag.name] = tag.value;
});
}
const point = new Point3(
globals.config.get(
'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName'
)
)
.setTag('event_type', 'user')
.setTag('source', event.source)
.setTag('host', event.host)
.setTag('subsystem', event.subsystem)
.setIntegerField('counter', event.counter);
// Add tags to point
Object.keys(tags).forEach((key) => {
point.setTag(key, tags[key]);
});
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.debug(`EVENT COUNT INFLUXDB: Wrote user event data to InfluxDB v3`);
}
globals.logger.verbose(
'EVENT COUNT INFLUXDB: Sent Butler SOS event count data to InfluxDB'
);
} catch (err) {
logError('EVENT COUNT INFLUXDB: Error getting write API', err);
}
}
}
@@ -1940,7 +2495,86 @@ export async function storeRejectedEventCountInfluxDB() {
'REJECT LOG EVENT INFLUXDB: Sent Butler SOS rejected event count data to InfluxDB'
);
} catch (err) {
globals.logger.error(`REJECTED LOG EVENT INFLUXDB: Error getting write API: ${err}`);
logError('REJECTED LOG EVENT INFLUXDB: Error getting write API', err);
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
try {
const points = [];
const measurementName = globals.config.get(
'Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName'
);
rejectedLogEvents.forEach((event) => {
globals.logger.debug(`REJECTED LOG EVENT INFLUXDB 3: ${JSON.stringify(event)}`);
if (event.source === 'qseow-qix-perf') {
let point = new Point3(measurementName)
.setTag('source', event.source)
.setTag('object_type', event.objectType)
.setTag('method', event.method)
.setIntegerField('counter', event.counter)
.setFloatField('process_time', event.processTime);
// Add app_id and app_name if available
if (event?.appId) {
point.setTag('app_id', event.appId);
}
if (event?.appName?.length > 0) {
point.setTag('app_name', event.appName);
point.setTag('app_name_set', 'true');
} else {
point.setTag('app_name_set', 'false');
}
// Add static tags defined in config file, if any
if (
globals.config.has(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
) &&
Array.isArray(
globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
)
)
) {
const configTags = globals.config.get(
'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags'
);
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
points.push(point);
} else {
let point = new Point3(measurementName)
.setTag('source', event.source)
.setIntegerField('counter', event.counter);
points.push(point);
}
});
// Write to InfluxDB
try {
for (const point of points) {
await globals.influx.write(point.toLineProtocol(), database);
}
globals.logger.debug(`REJECT LOG EVENT INFLUXDB: Wrote data to InfluxDB v3`);
} catch (err) {
globals.logger.error(
`REJECTED LOG EVENT INFLUXDB: Error saving data to InfluxDB v3! ${err}`
);
return;
}
globals.logger.verbose(
'REJECT LOG EVENT INFLUXDB: Sent Butler SOS rejected event count data to InfluxDB'
);
} catch (err) {
logError('REJECTED LOG EVENT INFLUXDB: Error getting write API', err);
}
}
}
@@ -2089,6 +2723,56 @@ export async function postUserEventQueueMetricsToInfluxdb() {
);
return;
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// InfluxDB 3.x
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
try {
const point = new Point3(measurementName)
.setTag('queue_type', 'user_events')
.setTag('host', globals.hostInfo.hostname)
.setIntegerField('queue_size', metrics.queueSize)
.setIntegerField('queue_max_size', metrics.queueMaxSize)
.setFloatField('queue_utilization_pct', metrics.queueUtilizationPct)
.setIntegerField('queue_pending', metrics.queuePending)
.setIntegerField('messages_received', metrics.messagesReceived)
.setIntegerField('messages_queued', metrics.messagesQueued)
.setIntegerField('messages_processed', metrics.messagesProcessed)
.setIntegerField('messages_failed', metrics.messagesFailed)
.setIntegerField('messages_dropped_total', metrics.messagesDroppedTotal)
.setIntegerField(
'messages_dropped_rate_limit',
metrics.messagesDroppedRateLimit
)
.setIntegerField(
'messages_dropped_queue_full',
metrics.messagesDroppedQueueFull
)
.setIntegerField('messages_dropped_size', metrics.messagesDroppedSize)
.setFloatField('processing_time_avg_ms', metrics.processingTimeAvgMs)
.setFloatField('processing_time_p95_ms', metrics.processingTimeP95Ms)
.setFloatField('processing_time_max_ms', metrics.processingTimeMaxMs)
.setIntegerField('rate_limit_current', metrics.rateLimitCurrent)
.setIntegerField('backpressure_active', metrics.backpressureActive);
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.verbose(
'USER EVENT QUEUE METRICS INFLUXDB: Sent queue metrics data to InfluxDB v3'
);
} catch (err) {
globals.logger.error(
`USER EVENT QUEUE METRICS INFLUXDB: Error saving data to InfluxDB v3! ${err}`
);
return;
}
}
// Clear metrics after writing
@@ -2242,6 +2926,56 @@ export async function postLogEventQueueMetricsToInfluxdb() {
);
return;
}
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// InfluxDB 3.x
const database = globals.config.get('Butler-SOS.influxdbConfig.v3Config.database');
try {
const point = new Point3(measurementName)
.setTag('queue_type', 'log_events')
.setTag('host', globals.hostInfo.hostname)
.setIntegerField('queue_size', metrics.queueSize)
.setIntegerField('queue_max_size', metrics.queueMaxSize)
.setFloatField('queue_utilization_pct', metrics.queueUtilizationPct)
.setIntegerField('queue_pending', metrics.queuePending)
.setIntegerField('messages_received', metrics.messagesReceived)
.setIntegerField('messages_queued', metrics.messagesQueued)
.setIntegerField('messages_processed', metrics.messagesProcessed)
.setIntegerField('messages_failed', metrics.messagesFailed)
.setIntegerField('messages_dropped_total', metrics.messagesDroppedTotal)
.setIntegerField(
'messages_dropped_rate_limit',
metrics.messagesDroppedRateLimit
)
.setIntegerField(
'messages_dropped_queue_full',
metrics.messagesDroppedQueueFull
)
.setIntegerField('messages_dropped_size', metrics.messagesDroppedSize)
.setFloatField('processing_time_avg_ms', metrics.processingTimeAvgMs)
.setFloatField('processing_time_p95_ms', metrics.processingTimeP95Ms)
.setFloatField('processing_time_max_ms', metrics.processingTimeMaxMs)
.setIntegerField('rate_limit_current', metrics.rateLimitCurrent)
.setIntegerField('backpressure_active', metrics.backpressureActive);
// Add static tags from config file
if (configTags && configTags.length > 0) {
for (const item of configTags) {
point.setTag(item.name, item.value);
}
}
await globals.influx.write(point.toLineProtocol(), database);
globals.logger.verbose(
'LOG EVENT QUEUE METRICS INFLUXDB: Sent queue metrics data to InfluxDB v3'
);
} catch (err) {
globals.logger.error(
`LOG EVENT QUEUE METRICS INFLUXDB: Error saving data to InfluxDB v3! ${err}`
);
return;
}
}
// Clear metrics after writing

19
src/lib/post-to-mqtt.js
View File

@@ -1,4 +1,5 @@
import globals from '../globals.js';
import { logError } from './log-error.js';
/**
* Posts health metrics from Qlik Sense engine healthcheck API to MQTT.
@@ -131,9 +132,9 @@ export function postUserSessionsToMQTT(host, virtualProxy, body) {
* @param {string} [msg.appId] - Optional app ID
* @param {string} [msg.appName] - Optional app name
* @param {object} [msg.ua] - Optional user agent information
* @returns {void}
* @returns {Promise<void>}
*/
export function postUserEventToMQTT(msg) {
export async function postUserEventToMQTT(msg) {
try {
// Create payload
const payload = {
@@ -231,7 +232,10 @@ export function postUserEventToMQTT(msg) {
globals.mqttClient.publish(topic, JSON.stringify(payload));
}
} catch (err) {
globals.logger.error(`USER EVENT MQTT: Failed posting message to MQTT ${err}.`);
// Track error count
await globals.errorTracker.incrementError('MQTT_PUBLISH', '');
logError('USER EVENT MQTT: Failed posting message to MQTT', err);
}
}
@@ -248,9 +252,9 @@ export function postUserEventToMQTT(msg) {
* @param {string} msg.message - The log message content
* @param {string} [msg.timestamp] - The timestamp of the log event
* @param {string} [msg.hostname] - The hostname where the log event occurred
* @returns {void}
* @returns {Promise<void>}
*/
export function postLogEventToMQTT(msg) {
export async function postLogEventToMQTT(msg) {
try {
// Get MQTT root topic
let baseTopic = globals.config.get('Butler-SOS.logEvents.sendToMQTT.baseTopic');
@@ -296,6 +300,9 @@ export function postLogEventToMQTT(msg) {
globals.mqttClient.publish(baseTopic, JSON.stringify(msg));
}
} catch (err) {
globals.logger.error(`LOG EVENT MQTT: Failed posting message to MQTT ${err}.`);
// Track error count
await globals.errorTracker.incrementError('MQTT_PUBLISH', '');
logError('LOG EVENT MQTT: Failed posting message to MQTT', err);
}
}

17
src/lib/post-to-new-relic.js
View File

@@ -2,6 +2,7 @@ import crypto from 'crypto';
import axios from 'axios';
import globals from '../globals.js';
import { logError } from './log-error.js';
// const sessionAppPrefix = 'SessionApp';
@@ -350,8 +351,11 @@ export async function postHealthMetricsToNewRelic(_host, body, tags) {
}
}
} catch (error) {
// Track error count
await globals.errorTracker.incrementError('NEW_RELIC_POST', '');
// handle error
globals.logger.error(`HEALTH METRICS NEW RELIC: Error sending proxy sessions: ${error}`);
logError('HEALTH METRICS NEW RELIC: Error sending proxy sessions', error);
}
}
@@ -511,8 +515,11 @@ export async function postProxySessionsToNewRelic(userSessions) {
}
}
} catch (error) {
// Track error count
await globals.errorTracker.incrementError('NEW_RELIC_POST', '');
// handle error
globals.logger.error(`PROXY SESSIONS NEW RELIC: Error sending proxy sessions: ${error}`);
logError('PROXY SESSIONS NEW RELIC: Error sending proxy sessions', error);
}
}
@@ -687,7 +694,7 @@ export async function postButlerSOSUptimeToNewRelic(fields) {
}
} catch (error) {
// handle error
globals.logger.error(`UPTIME NEW RELIC: Error sending uptime: ${error}`);
logError('UPTIME NEW RELIC: Error sending uptime', error);
}
}
@@ -842,7 +849,7 @@ export async function postUserEventToNewRelic(msg) {
}
}
} catch (err) {
globals.logger.error(`USER EVENT NEW RELIC: Error saving user event to New Relic! ${err}`);
logError('USER EVENT NEW RELIC: Error saving user event to New Relic!', err);
}
}
@@ -1136,6 +1143,6 @@ export async function postLogEventToNewRelic(msg) {
}
}
} catch (err) {
globals.logger.error(`LOG EVENT NEW RELIC: Error saving event to New Relic! ${err}`);
logError('LOG EVENT NEW RELIC: Error saving event to New Relic!', err);
}
}

41
src/lib/proxysessionmetrics.js
View File

@@ -6,14 +6,17 @@ import https from 'https';
import path from 'path';
import axios from 'axios';
import { Point } from '@influxdata/influxdb-client';
import { Point as Point3 } from '@influxdata/influxdb3-client';
import globals from '../globals.js';
import { postProxySessionsToInfluxdb } from './post-to-influxdb.js';
import { postProxySessionsToInfluxdb } from './influxdb/index.js';
import { postProxySessionsToNewRelic } from './post-to-new-relic.js';
import { applyTagsToPoint3 } from './influxdb/shared/utils.js';
import { postUserSessionsToMQTT } from './post-to-mqtt.js';
import { getServerTags } from './servertags.js';
import { saveUserSessionMetricsToPrometheus } from './prom-client.js';
import { getCertificates, createCertificateOptions } from './cert-utils.js';
import { logError } from './log-error.js';
/**
* Prepares user session metrics data for storage/forwarding to various destinations.
@@ -98,6 +101,19 @@ function prepUserSessionMetrics(serverName, host, virtualProxy, body, tags) {
.uintField('session_count', userProxySessionsData.sessionCount)
.stringField('session_user_id_list', userProxySessionsData.uniqueUserList),
];
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Create data points for InfluxDB v3
const summaryPoint = new Point3('user_session_summary')
.setIntegerField('session_count', userProxySessionsData.sessionCount)
.setStringField('session_user_id_list', userProxySessionsData.uniqueUserList);
applyTagsToPoint3(summaryPoint, userProxySessionsData.tags);
const listPoint = new Point3('user_session_list')
.setIntegerField('session_count', userProxySessionsData.sessionCount)
.setStringField('session_user_id_list', userProxySessionsData.uniqueUserList);
applyTagsToPoint3(listPoint, userProxySessionsData.tags);
userProxySessionsData.datapointInfluxdb = [summaryPoint, listPoint];
}
// Prometheus specific.
@@ -184,11 +200,26 @@ function prepUserSessionMetrics(serverName, host, virtualProxy, body, tags) {
.stringField('session_id', bodyItem.SessionId)
.stringField('user_directory', bodyItem.UserDirectory)
.stringField('user_id', bodyItem.UserId);
} else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Create data point for InfluxDB v3
sessionDatapoint = new Point3('user_session_details')
.setStringField('session_id', bodyItem.SessionId)
.setStringField('user_directory', bodyItem.UserDirectory)
.setStringField('user_id', bodyItem.UserId);
// Apply all tags including server tags and session-specific tags
applyTagsToPoint3(sessionDatapoint, userProxySessionsData.tags);
// Add individual session tags
sessionDatapoint
.setTag('user_session_id', bodyItem.SessionId)
.setTag('user_session_user_directory', bodyItem.UserDirectory)
.setTag('user_session_user_id', bodyItem.UserId);
}
if (sessionDatapoint) {
userProxySessionsData.datapointInfluxdb.push(sessionDatapoint);
}
}
}
resolve(userProxySessionsData);
} catch (err) {
@@ -316,8 +347,12 @@ export async function getProxySessionStatsFromSense(serverName, host, virtualPro
}
}
} catch (err) {
globals.logger.error(
`PROXY SESSIONS: Error when calling proxy session API for server '${serverName}' (${host}), virtual proxy '${virtualProxy}': ${globals.getErrorMessage(err)}`
// Track error count
await globals.errorTracker.incrementError('PROXY_API', serverName);
logError(
`PROXY SESSIONS: Error when calling proxy session API for server '${serverName}' (${host}), virtual proxy '${virtualProxy}'`,
err
);
}
}

3
src/lib/serverheaders.js
View File

@@ -1,4 +1,5 @@
import globals from '../globals.js';
import { logError } from './log-error.js';
/**
* Extracts HTTP headers from a server configuration object.
@@ -33,7 +34,7 @@ export function getServerHeaders(server) {
return headers;
} catch (err) {
globals.logger.error(`SERVERTAGS: ${err}`);
logError('SERVERTAGS', err);
return [];
}
}

2
src/lib/service_uptime.js
View File

@@ -2,7 +2,7 @@ import later from '@breejs/later';
import { Duration } from 'luxon';
import globals from '../globals.js';
import { postButlerSOSMemoryUsageToInfluxdb } from './post-to-influxdb.js';
import { postButlerSOSMemoryUsageToInfluxdb } from './influxdb/index.js';
import { postButlerSOSUptimeToNewRelic } from './post-to-new-relic.js';
const fullUnits = ['years', 'months', 'days', 'hours', 'minutes', 'seconds'];

2
src/lib/udp-event.js
View File

@@ -1,7 +1,7 @@
import { Mutex } from 'async-mutex';
import globals from '../globals.js';
import { storeRejectedEventCountInfluxDB, storeEventCountInfluxDB } from './post-to-influxdb.js';
import { storeRejectedEventCountInfluxDB, storeEventCountInfluxDB } from './influxdb/index.js';
/**
* Class for tracking counts of UDP events received from Qlik Sense.

4
src/lib/udp_handlers/log_events/__tests__/sanitization.test.js
View File

@@ -217,7 +217,7 @@ describe('Log Event Handler Sanitization', () => {
});
describe('QIX Performance Event Handler', () => {
it('should sanitize method and object_type fields', () => {
it('should sanitize method and object_type fields', async () => {
const msg = [
'/qseow-qix-perf/',
'1',
@@ -247,7 +247,7 @@ describe('Log Event Handler Sanitization', () => {
'linechart\x02', // Field 25: object_type
];
const result = processQixPerfEvent(msg);
const result = await processQixPerfEvent(msg);
if (result) {
expect(result.method).not.toMatch(/[\x00-\x1F\x7F]/);
expect(result.object_type).not.toMatch(/[\x00-\x1F\x7F]/);

30
src/lib/udp_handlers/log_events/handlers/qix-perf-handler.js
View File

@@ -39,9 +39,9 @@ import { sanitizeField } from '../../../udp-queue-manager.js';
* 25: Object type. Ex: <Unknown>, AppPropsList, SheetList, StoryList, VariableList, linechart, barchart, map, listbox, CurrentSelection
*
* @param {Array} msg - The message parts
* @returns {object | null} Processed message object or null if event should be skipped
* @returns {Promise<object | null>} Processed message object or null if event should be skipped
*/
export function processQixPerfEvent(msg) {
export async function processQixPerfEvent(msg) {
globals.logger.verbose(
`LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, ${msg[9]}\\${msg[10]}, ${msg[13]}, ${msg[15]}, Object type: ${msg[25]}`
);
@@ -51,6 +51,32 @@ export function processQixPerfEvent(msg) {
globals.logger.debug(
'LOG EVENT: Qix performance monitoring is disabled in the configuration. Skipping event.'
);
// Is logging of event counts enabled?
if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) {
// Get source, host and subsystem if they exist, otherwise set to 'Unknown'
let source = 'Unknown';
let host = 'Unknown';
let subsystem = 'Unknown';
if (msg.length > 0) {
source = msg[0].toLowerCase().replace('/', '').replace('/', '');
}
if (msg.length > 5) {
host = msg[5];
}
if (msg.length > 6) {
subsystem = msg[6];
}
// Increase counter for log events when detailed monitoring is disabled
await globals.udpEvents.addLogEvent({
source: source,
host: host,
subsystem: subsystem,
});
}
return null;
}

79
src/lib/udp_handlers/log_events/message-event.js
View File

@@ -1,8 +1,9 @@
import globals from '../../../globals.js';
import { postLogEventToInfluxdb } from '../../post-to-influxdb.js';
import { postLogEventToInfluxdb } from '../../influxdb/index.js';
import { postLogEventToNewRelic } from '../../post-to-new-relic.js';
import { postLogEventToMQTT } from '../../post-to-mqtt.js';
import { categoriseLogEvent } from '../../log-event-categorise.js';
import { logError } from '../../log-error.js';
// Import handlers for different log event sources
import { processEngineEvent } from './handlers/engine-handler.js';
@@ -72,7 +73,7 @@ export async function messageEventHandler(message, _remote) {
msgObj = processSchedulerEvent(msgParts);
break;
case 'qseow-qix-perf':
msgObj = processQixPerfEvent(msgParts);
msgObj = await processQixPerfEvent(msgParts);
// If null is returned, it means the event should be skipped
if (msgObj === null) {
return;
@@ -80,9 +81,52 @@ export async function messageEventHandler(message, _remote) {
break;
default:
globals.logger.warn(`LOG EVENT: Unknown source: ${msgParts[0]}`);
// Is logging of event counts enabled?
if (
globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true
) {
// Increase counter for unknown log events
await globals.udpEvents.addLogEvent({
source: 'Unknown',
host: 'Unknown',
subsystem: 'Unknown',
});
}
return;
}
// Add counter for received log events
// Is logging of event counts enabled?
if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) {
globals.logger.debug(
`LOG EVENT: Received message that is a recognised log event: ${msgParts[0]}`
);
// Get source, host and subsystem if they exist, otherwise set to 'Unknown'
let source = 'Unknown';
let host = 'Unknown';
let subsystem = 'Unknown';
if (msgObj.source.length > 0) {
source = msgObj.source;
}
if (msgObj.host.length > 0) {
host = msgObj.host;
}
if (msgObj.subsystem.length > 0) {
subsystem = msgObj.subsystem;
}
// Increase counter for log events
await globals.udpEvents.addLogEvent({
source: source,
host: host,
subsystem: subsystem,
});
}
// If message parsing was done and categorisation is enabled, categorise the log event
if (
Object.keys(msgObj).length !== 0 &&
@@ -130,8 +174,37 @@ export async function messageEventHandler(message, _remote) {
globals.logger.debug(
`LOG EVENT: Log event source not recognized or not enabled in configuration, skipping message: ${msgParts[0]}`
);
// Is logging of event counts enabled?
if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) {
// Get source, host and subsystem if they exist, otherwise set to 'Unknown'
let source = 'Unknown';
let host = 'Unknown';
let subsystem = 'Unknown';
if (msgParts.length > 0) {
source = msgParts[0].toLowerCase().replace('/', '').replace('/', '');
}
if (msgParts.length > 1) {
host = msgParts[1];
}
if (msgParts.length > 5) {
subsystem = msgParts[5];
}
globals.logger.warn(
`LOG EVENT: Received message that is an unrecognized log event: ${source}`
);
// Increase counter for log events
await globals.udpEvents.addLogEvent({
source: source,
host: host,
subsystem: subsystem,
});
}
}
} catch (err) {
globals.logger.error(`LOG EVENT: Error handling message: ${globals.getErrorMessage(err)}`);
logError('LOG EVENT: Error handling message', err);
}
}

7
src/lib/udp_handlers/user_events/message-event.js
View File

@@ -4,9 +4,10 @@ import { UAParser } from 'ua-parser-js';
// Load global variables and functions
import globals from '../../../globals.js';
import { sanitizeField } from '../../udp-queue-manager.js';
import { postUserEventToInfluxdb } from '../../post-to-influxdb.js';
import { postUserEventToInfluxdb } from '../../influxdb/index.js';
import { postUserEventToNewRelic } from '../../post-to-new-relic.js';
import { postUserEventToMQTT } from '../../post-to-mqtt.js';
import { logError } from '../../log-error.js';
/**
* Handler for UDP messages relating to user events from Qlik Sense Proxy service.
@@ -237,8 +238,6 @@ export async function messageEventHandler(message, _remote) {
postUserEventToNewRelic(msgObj);
}
} catch (err) {
globals.logger.error(
`USER EVENT: Error processing user activity event: ${globals.getErrorMessage(err)}`
);
logError('USER EVENT: Error processing user activity event', err);
}
}

7
src/lib/udp_handlers_log_events.js
View File

@@ -1,6 +1,7 @@
// Load global variables and functions
import globals from '../globals.js';
import { listeningEventHandler, messageEventHandler } from './udp_handlers/log_events/index.js';
import { logError } from './log-error.js';
// --------------------------------------------------------
// Set up UDP server for acting on Sense log events
@@ -57,15 +58,13 @@ export function udpInitLogEventServer() {
globals.logger.debug(`[UDP Queue] Log event message dropped due to full queue`);
}
} catch (err) {
globals.logger.error(
`[UDP Queue] Error handling log event message: ${globals.getErrorMessage(err)}`
);
logError('[UDP Queue] Error handling log event message', err);
}
});
// Handler for UDP server errors
globals.udpServerLogEvents.socket.on('error', (err) => {
globals.logger.error(`[UDP] Log events server error: ${globals.getErrorMessage(err)}`);
logError('[UDP] Log events server error', err);
});
// Handler for UDP server close event

7
src/lib/udp_handlers_user_activity.js
View File

@@ -1,6 +1,7 @@
// Load global variables and functions
import globals from '../globals.js';
import { listeningEventHandler, messageEventHandler } from './udp_handlers/user_events/index.js';
import { logError } from './log-error.js';
// --------------------------------------------------------
// Set up UDP server for acting on Sense user activity events
@@ -49,15 +50,13 @@ export function udpInitUserActivityServer() {
globals.logger.debug(`[UDP Queue] User activity message dropped due to full queue`);
}
} catch (err) {
globals.logger.error(
`[UDP Queue] Error handling user activity message: ${globals.getErrorMessage(err)}`
);
logError('[UDP Queue] Error handling user activity message', err);
}
});
// Handler for UDP server errors
globals.udpServerUserActivity.socket.on('error', (err) => {
globals.logger.error(`[UDP] User activity server error: ${globals.getErrorMessage(err)}`);
logError('[UDP] User activity server error', err);
});
// Handler for UDP server close event