jprdonnelly/butler-sos
1
0
Fork 0
mirror of https://github.com/ptarmiganlabs/butler-sos.git synced 2025-12-19 17:58:18 -05:00
Code Issues Projects Releases Wiki Activity

work in progress

Browse Source
This commit is contained in:
Göran Sander
2025-09-26 13:03:23 +02:00
parent 956d69b3fa
commit bedfeca420
13 changed files with 1095 additions and 1014 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
docs/docker-compose/.env_influxdb_v3 Normal file
View File

@@ -0,0 +1,17 @@
# Adapted from https://github.com/InfluxCommunity/TIG-Stack-using-InfluxDB-3/blob/main/.env
# Butler SOS configuration
BUTLER_SOS_CONFIG_FILE=/production_influxdb_v3.yaml # File placed in ./config directory
# InfluxDB Configuration
INFLUXDB_HTTP_PORT=8181 # for influxdb3 enterprise database, change this to port 8182
INFLUXDB_HOST=influxdb3-core # for influxdb3 enterprise database, change this to "influxdb3-enterprise"
INFLUXDB_TOKEN=
INFLUXDB_BUCKET=local_system # Your Database name
INFLUXDB_ORG=local_org
INFLUXDB_NODE_ID=node0
# Grafana Configuration
GRAFANA_PORT=3000
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin

95
docs/docker-compose/README.md
View File

@@ -5,26 +5,29 @@ This directory contains Docker Compose configurations for running Butler SOS wit
## Available Configurations
### InfluxDB v1.x
- **File**: `docker-compose_fullstack_influxdb_v1.yml`
- **InfluxDB Image**: `influxdb:1.8-alpine`
- **Features**: Traditional InfluxDB with SQL-like query language
- **Configuration**: Set `Butler-SOS.influxdbConfig.version: 1` in your config file
- **Environment**: Set `NODE_ENV=production_influxdb_v1`
### InfluxDB v2.x
### InfluxDB v2.x
- **File**: `docker-compose_fullstack_influxdb_v2.yml`
- **InfluxDB Image**: `influxdb:2.7-alpine`
- **Features**: Modern InfluxDB with Flux query language, unified time series platform
- **Configuration**: Set `Butler-SOS.influxdbConfig.version: 2` in your config file
- **Environment**: Set `NODE_ENV=production_influxdb_v2`
- **Default Credentials**:
- Username: `admin`
- Password: `butlersos123`
- Organization: `butler-sos`
- Bucket: `butler-sos`
- Token: `butlersos-token`
- Username: `admin`
- Password: `butlersos123`
- Organization: `butler-sos`
- Bucket: `butler-sos`
- Token: `butlersos-token`
### InfluxDB v3.x
- **File**: `docker-compose_fullstack_influxdb_v3.yml`
- **InfluxDB Image**: `influxdb:latest`
- **Features**: Latest InfluxDB architecture with enhanced performance and cloud-native design
@@ -32,11 +35,6 @@ This directory contains Docker Compose configurations for running Butler SOS wit
- **Environment**: Set `NODE_ENV=production_influxdb_v3`
- **Default Credentials**: Same as v2.x but with database concept support
### Legacy/Default
- **File**: `docker-compose_fullstack_influxdb.yml`
- **Purpose**: Backward compatibility (defaults to v1.x)
- **Recommendation**: Use version-specific files for new deployments
## Usage
1. Choose the appropriate docker-compose file for your InfluxDB version
@@ -47,53 +45,56 @@ This directory contains Docker Compose configurations for running Butler SOS wit
## Configuration Requirements
### For InfluxDB v1.x
```yaml
Butler-SOS:
influxdbConfig:
enable: true
version: 1
host: influxdb-v1
port: 8086
v1Config:
auth:
enable: false
dbName: SenseOps
retentionPolicy:
name: 10d
duration: 10d
influxdbConfig:
enable: true
version: 1
host: influxdb-v1
port: 8086
v1Config:
auth:
enable: false
dbName: SenseOps
retentionPolicy:
name: 10d
duration: 10d
```
### For InfluxDB v2.x
```yaml
Butler-SOS:
influxdbConfig:
enable: true
version: 2
host: influxdb-v2
port: 8086
v2Config:
org: butler-sos
bucket: butler-sos
token: butlersos-token
description: Butler SOS metrics
retentionDuration: 10d
influxdbConfig:
enable: true
version: 2
host: influxdb-v2
port: 8086
v2Config:
org: butler-sos
bucket: butler-sos
token: butlersos-token
description: Butler SOS metrics
retentionDuration: 10d
```
### For InfluxDB v3.x
```yaml
Butler-SOS:
influxdbConfig:
enable: true
version: 3
host: influxdb-v3
port: 8086
v3Config:
org: butler-sos
bucket: butler-sos
database: butler-sos
token: butlersos-token
description: Butler SOS metrics
retentionDuration: 10d
influxdbConfig:
enable: true
version: 3
host: influxdb-v3
port: 8086
v3Config:
org: butler-sos
bucket: butler-sos
database: butler-sos
token: butlersos-token
description: Butler SOS metrics
retentionDuration: 10d
```
## Migration Notes
@@ -102,4 +103,4 @@ Butler-SOS:
- **v2 to v3**: Uses similar client libraries but different internal architecture
- **v1 to v3**: Significant migration required, consider using InfluxDB migration tools
For detailed configuration options, refer to the main Butler SOS documentation.
For detailed configuration options, refer to the main Butler SOS documentation.

27
docs/docker-compose/docker-compose.yml
View File

@@ -1,27 +0,0 @@
# docker-compose.yml
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
command:
- 'node'
- 'src/butler-sos.js'
- '--configfile'
- '/nodeapp/config/production.yaml'
ports:
- '9997:9997' # UDP user events
- '9996:9996' # UDP log events
- '9842:9842' # Prometheus metrics
- '3100:3100' # Config file visualization
volumes:
# Make config file accessible outside of container
- './config:/nodeapp/config'
- './log:/nodeapp/log'
environment:
- 'NODE_ENV=production' # Means that Butler SOS will read config data from production.yaml
logging:
driver: 'json-file'
options:
max-file: '5'
max-size: '5m'

57
docs/docker-compose/docker-compose_fullstack_influxdb.yml
View File

@@ -1,57 +0,0 @@
# docker-compose_fullstack_influxdb.yml
# Default InfluxDB configuration (v1.x) for backward compatibility
# For version-specific configurations, use:
# - docker-compose_fullstack_influxdb_v1.yml for InfluxDB v1.x
# - docker-compose_fullstack_influxdb_v2.yml for InfluxDB v2.x
# - docker-compose_fullstack_influxdb_v3.yml for InfluxDB v3.x
version: "3.3"
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_influxdb" # Means that Butler SOS will read config data from production_influxdb.yaml
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "5m"
networks:
- senseops
influxdb:
image: influxdb:1.12.2
container_name: influxdb
restart: always
volumes:
- ./influxdb/data:/var/lib/influxdb # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb/ # Mount for influxdb configuration
ports:
# The API for InfluxDB is served on port 8086
- "8086:8086"
- "8082:8082"
environment:
# Disable usage reporting
- "INFLUXDB_REPORTING_DISABLED=true"
networks:
- senseops
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: always
ports:
- "3000:3000"
volumes:
- ./grafana/data:/var/lib/grafana
networks:
- senseops
networks:
senseops:
driver: bridge

17
docs/docker-compose/docker-compose_fullstack_influxdb_v1.yml
View File

@@ -1,16 +1,19 @@
# docker-compose_fullstack_influxdb_v1.yml
version: "3.3"
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
restart: unless-stopped
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_influxdb_v1" # Means that Butler SOS will read config data from production_influxdb_v1.yaml
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_influxdb_v1.yaml"]
logging:
driver: "json-file"
options:
@@ -20,9 +23,9 @@ services:
- senseops
influxdb:
image: influxdb:1.8-alpine
image: influxdb:1.12.2
container_name: influxdb-v1
restart: always
restart: unless-stopped
volumes:
- ./influxdb/data:/var/lib/influxdb # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb/ # Mount for influxdb configuration
@@ -39,7 +42,7 @@ services:
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: always
restart: unless-stopped
ports:
- "3000:3000"
volumes:

15
docs/docker-compose/docker-compose_fullstack_influxdb_v2.yml
View File

@@ -1,16 +1,19 @@
# docker-compose_fullstack_influxdb_v2.yml
version: "3.3"
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
restart: unless-stopped
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_influxdb_v2" # Means that Butler SOS will read config data from production_influxdb_v2.yaml
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_influxdb_v2.yaml"]
logging:
driver: "json-file"
options:
@@ -22,7 +25,7 @@ services:
influxdb:
image: influxdb:2.7-alpine
container_name: influxdb-v2
restart: always
restart: unless-stopped
volumes:
- ./influxdb/data:/var/lib/influxdb2 # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb2/ # Mount for influxdb configuration
@@ -44,7 +47,7 @@ services:
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: always
restart: unless-stopped
ports:
- "3000:3000"
volumes:

67
docs/docker-compose/docker-compose_fullstack_influxdb_v3.yml
View File

@@ -1,58 +1,81 @@
# docker-compose_fullstack_influxdb_v3.yml
# InfluxDB v3.x (Core) - using the InfluxDB 3.x Community Edition
version: "3.3"
# Inspiration from https://github.com/InfluxCommunity/TIG-Stack-using-InfluxDB-3/blob/main/docker-compose.yml
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
restart: unless-stopped
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_influxdb_v3" # Means that Butler SOS will read config data from production_influxdb_v3.yaml
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/${BUTLER_SOS_CONFIG_FILE}"]
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "5m"
depends_on:
# Or switch to influxdb3-enterprise as needed
- influxdb-v3-core
networks:
- senseops
influxdb:
influxdb-v3-core:
# Note: InfluxDB v3 Core is available as influxdb3 image
# For production use, consider InfluxDB Cloud or Enterprise
image: influxdb:latest
container_name: influxdb-v3
restart: always
image: influxdb:3-core
container_name: influxdb-v3-core
restart: unless-stopped
ports:
- ${INFLUXDB_HTTP_PORT}:8181
command:
- influxdb3
- serve
- --node-id=${INFLUXDB_NODE_ID}
- --object-store=file
- --data-dir=/var/lib/influxdb3
volumes:
- ./influxdb/data:/var/lib/influxdb3 # Mount for influxdb data directory
- ./influxdb/config/:/etc/influxdb3/ # Mount for influxdb configuration
ports:
# The API for InfluxDB is served on port 8086
- "8086:8086"
environment:
# environment:
# InfluxDB v3 setup - uses similar setup to v2 but different internal architecture
- "DOCKER_INFLUXDB_INIT_MODE=setup"
- "DOCKER_INFLUXDB_INIT_USERNAME=admin"
- "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123"
- "DOCKER_INFLUXDB_INIT_ORG=butler-sos"
- "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos"
- "DOCKER_INFLUXDB_INIT_DATABASE=butler-sos" # v3 uses database concept
- "DOCKER_INFLUXDB_INIT_RETENTION=10d"
- "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token"
# - "DOCKER_INFLUXDB_INIT_MODE=setup"
# - "DOCKER_INFLUXDB_INIT_USERNAME=admin"
# - "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123"
# - "DOCKER_INFLUXDB_INIT_ORG=butler-sos"
# - "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos"
# - "DOCKER_INFLUXDB_INIT_DATABASE=butler-sos" # v3 uses database concept
# - "DOCKER_INFLUXDB_INIT_RETENTION=10d"
# - "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token"
healthcheck:
test: ["CMD-SHELL", "curl -f -H 'Authorization: Bearer ${INFLUXDB_TOKEN}' http://localhost:8181/health || exit 1"]
interval: 30s
timeout: 10s
retries: 3
networks:
- senseops
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: always
restart: unless-stopped
ports:
- "3000:3000"
- "${GRAFANA_PORT}:3000"
environment:
- GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER}
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
volumes:
- ./grafana/data:/var/lib/grafana
depends_on:
# Or switch to influxdb3-enterprise as needed
- influxdb-v3-core
networks:
- senseops

9
docs/docker-compose/docker-compose_fullstack_prometheus.yml Executable file → Normal file
View File

@@ -1,16 +1,19 @@
# docker-compose_fullstack_prometheus.yml
version: "3.3"
services:
butler-sos:
image: ptarmiganlabs/butler-sos:latest
container_name: butler-sos
restart: always
ports:
- "9997:9997" # UDP user events
- "9996:9996" # UDP log events
- "9842:9842" # Prometheus metrics
- "3100:3100" # Config file visualization
volumes:
# Make config file and log files accessible outside of container
- "./config:/nodeapp/config"
- "./log:/nodeapp/log"
environment:
- "NODE_ENV=production_prometheus" # Means that Butler SOS will read config data from production_prometheus.yaml
command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_prometheus.yaml"]
logging:
driver: "json-file"
options:

1748
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
package.json
View File

@@ -5,6 +5,7 @@
"main": "butler-sos.js",
"scripts": {
"build": "npx jsdoc-to-markdown 'src/**/*.js' > docs/src-code-overview.md",
"build:docker": "docker build -t butler-sos:latest .",
"butler-sos": "node src/butler-sos.js",
"jest": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js",
"test": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js && snyk test && npm run format",
@@ -52,6 +53,7 @@
"@fastify/static": "^8.3.0",
"@influxdata/influxdb-client": "^1.35.0",
"@influxdata/influxdb-client-apis": "^1.35.0",
"@influxdata/influxdb3-client": "^1.4.0",
"ajv": "^8.17.1",
"ajv-keywords": "^5.1.0",
"async-mutex": "^0.5.0",

8
src/config/production_template.yaml
View File

@@ -514,6 +514,12 @@ Butler-SOS:
host: influxdb.mycompany.com # InfluxDB host, hostname, FQDN or IP address
port: 8086 # Port where InfluxDBdb is listening, usually 8086
version: 1 # Is the InfluxDB instance version 1.x or 2.x? Valid values are 1 or 2
v3Config: # Settings for InfluxDB v3.x only, i.e. Butler-SOS.influxdbConfig.version=3
org: myorg
bucket: mybucket
description: Butler SOS metrics
token: mytoken
retentionDuration: 10d
v2Config: # Settings for InfluxDB v2.x only, i.e. Butler-SOS.influxdbConfig.version=2
org: myorg
bucket: mybucket
@@ -525,7 +531,7 @@ Butler-SOS:
enable: false # Does influxdb instance require authentication (true/false)?
username: <username> # Username for Influxdb authentication. Mandatory if auth.enable=true
password: <password> # Password for Influxdb authentication. Mandatory if auth.enable=true
dbName: SenseOps
dbName: senseops
# Default retention policy that should be created in InfluxDB when Butler SOS creates a new database there.
# Any data older than retention policy threshold will be purged from InfluxDB.
retentionPolicy:

44
src/globals.js
View File

@@ -8,10 +8,28 @@ import winston from 'winston';
import 'winston-daily-rotate-file';
import si from 'systeminformation';
import { readFileSync } from 'fs';
import Influx from 'influx';
import { Command, Option } from 'commander';
import { InfluxDB, HttpError, DEFAULT_WriteOptions } from '@influxdata/influxdb-client';
// Note on InfluxDB libraries:
// v1 client library: https://github.com/node-influx/node-influx
// v2 client library: https://influxdata.github.io/influxdb-client-js/
// v3 client library: https://github.com/InfluxCommunity/influxdb3-js
// v1
import Influx from 'influx';
// v2
// Import InfluxDB as const InfluxDB2 to avoid name clash with Influx from 'influx' above
import {
InfluxDB as InfluxDB2,
HttpError,
DEFAULT_WriteOptions,
} from '@influxdata/influxdb-client';
import { OrgsAPI, BucketsAPI } from '@influxdata/influxdb-client-apis';
// v3
import { InfluxDBClient as InfluxDBClient3, Point as Point3 } from '@influxdata/influxdb3-client';
import { fileURLToPath } from 'url';
import sea from './lib/sea-wrapper.js';
@@ -135,9 +153,6 @@ class Settings {
this.appVersion = appVersion;
// Make copy of influxdb client
const InfluxDB2 = InfluxDB;
// Command line parameters
const program = new Command();
program
@@ -705,9 +720,6 @@ Configuration File:
this.logger.info(
`CONFIG: Influxdb organisation: ${this.config.get('Butler-SOS.influxdbConfig.v3Config.org')}`
);
this.logger.info(
`CONFIG: Influxdb database: ${this.config.get('Butler-SOS.influxdbConfig.v3Config.database')}`
);
this.logger.info(
`CONFIG: Influxdb bucket name: ${this.config.get('Butler-SOS.influxdbConfig.v3Config.bucket')}`
);
@@ -876,7 +888,7 @@ Configuration File:
const token = this.config.get('Butler-SOS.influxdbConfig.v2Config.token');
try {
this.influx = new InfluxDB({ url, token });
this.influx = new InfluxDB2({ url, token });
} catch (err) {
this.logger.error(
`INFLUXDB2 INIT: Error creating InfluxDB 2 client: ${this.getErrorMessage(err)}`
@@ -884,14 +896,14 @@ Configuration File:
this.logger.error(`INFLUXDB2 INIT: Exiting.`);
}
} else if (this.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Set up Influxdb v3 client (uses same client library as v2)
// Set up Influxdb v3 client (uses its own client library, NOT same as v2)
const url = `http://${this.config.get('Butler-SOS.influxdbConfig.host')}:${this.config.get(
'Butler-SOS.influxdbConfig.port'
)}`;
const token = this.config.get('Butler-SOS.influxdbConfig.v3Config.token');
try {
this.influx = new InfluxDB({ url, token });
this.influx = new InfluxDBClient3({ url, token });
} catch (err) {
this.logger.error(
`INFLUXDB3 INIT: Error creating InfluxDB 3 client: ${this.getErrorMessage(err)}`
@@ -1118,8 +1130,8 @@ Configuration File:
maxRetries: 2, // do not retry writes
// ... there are more write options that can be customized, see
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html
// https://influxdata.github.io/influxdb-client-js/interfaces/_influxdata_influxdb-client.WriteOptions.html
// https://influxdata.github.io/influxdb-client-js/interfaces/_influxdata_influxdb-client.WriteRetryOptions.html
};
try {
@@ -1145,7 +1157,6 @@ Configuration File:
} else if (this.config.get('Butler-SOS.influxdbConfig.version') === 3) {
// Get config
const org = this.config.get('Butler-SOS.influxdbConfig.v3Config.org');
const database = this.config.get('Butler-SOS.influxdbConfig.v3Config.database');
const bucketName = this.config.get('Butler-SOS.influxdbConfig.v3Config.bucket');
const description = this.config.get('Butler-SOS.influxdbConfig.v3Config.description');
const token = this.config.get('Butler-SOS.influxdbConfig.v3Config.token');
@@ -1157,7 +1168,6 @@ Configuration File:
this.influx &&
this.config.get('Butler-SOS.influxdbConfig.enable') === true &&
org?.length > 0 &&
database?.length > 0 &&
bucketName?.length > 0 &&
token?.length > 0 &&
retentionDuration?.length > 0
@@ -1166,9 +1176,9 @@ Configuration File:
}
if (enableInfluxdb) {
// For InfluxDB v3, we use the database directly
// For InfluxDB v3, we use the bucket directly
this.logger.info(
`INFLUXDB3: Using organization "${org}" with database "${database}"`
`INFLUXDB3: Using organization "${org}" with bucket "${bucketName}"`
);
// Create array of per-server writeAPI objects for v3

3
src/lib/config-schemas/destinations.js
View File

@@ -321,12 +321,11 @@ export const destinationsSchema = {
properties: {
org: { type: 'string' },
bucket: { type: 'string' },
database: { type: 'string' },
description: { type: 'string' },
token: { type: 'string' },
retentionDuration: { type: 'string' },
},
required: ['org', 'bucket', 'database', 'description', 'token', 'retentionDuration'],
required: ['org', 'bucket', 'description', 'token', 'retentionDuration'],
additionalProperties: false,
},
v2Config: {