From 99ab4579dcf3e233e042ae2621091e08556abded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 18 Dec 2025 15:48:21 +0100 Subject: [PATCH 1/3] feat(docker): New full-stack compose files for InfluxDB v1/v2/v3 + Grafana --- .gitignore | 2 +- docs/docker-compose/.env_influxdb_v3 | 2 +- docs/docker-compose/config/admin_token.txt | 4 + .../config/production_influxdb_v1.yaml | 302 ++++++++++++++++++ .../config/production_influxdb_v2.yaml | 302 ++++++++++++++++++ .../config/production_influxdb_v3.yaml | 302 ++++++++++++++++++ .../docker-compose_fullstack_influxdb_v1.yml | 57 ++-- .../docker-compose_fullstack_influxdb_v2.yml | 67 ++-- .../docker-compose_fullstack_influxdb_v3.yml | 94 ++++-- .../grafana/dashboards/butler_sos_test.json | 272 ++++++++++++++++ .../provisioning/dashboards/dashboard.yaml | 11 + .../datasources/datasource_v1.yaml | 14 + .../datasources/datasource_v2.yaml | 14 + .../datasources/datasource_v3.yaml | 14 + src/globals.js | 4 +- 15 files changed, 1381 insertions(+), 80 deletions(-) create mode 100644 docs/docker-compose/config/admin_token.txt create mode 100644 docs/docker-compose/config/production_influxdb_v1.yaml create mode 100644 docs/docker-compose/config/production_influxdb_v2.yaml create mode 100644 docs/docker-compose/config/production_influxdb_v3.yaml create mode 100644 docs/docker-compose/grafana/dashboards/butler_sos_test.json create mode 100644 docs/docker-compose/grafana/provisioning/dashboards/dashboard.yaml create mode 100644 docs/docker-compose/grafana/provisioning/datasources/datasource_v1.yaml create mode 100644 docs/docker-compose/grafana/provisioning/datasources/datasource_v2.yaml create mode 100644 docs/docker-compose/grafana/provisioning/datasources/datasource_v3.yaml diff --git a/.gitignore b/.gitignore index 38f849f..4acd5cc 100755 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ ssl/* *.tmp snyk.sarif build +test_data # Logs log/ @@ -60,4 +61,3 @@ butler-sos src/config/production_vac.yaml tmp/ docs/docker-compose/.env -docs/docker-compose/config/production_influxdb_v3.yaml diff --git a/docs/docker-compose/.env_influxdb_v3 b/docs/docker-compose/.env_influxdb_v3 index c1a8b41..8abf6de 100644 --- a/docs/docker-compose/.env_influxdb_v3 +++ b/docs/docker-compose/.env_influxdb_v3 @@ -6,7 +6,7 @@ BUTLER_SOS_CONFIG_FILE=/production_influxdb_v3.yaml # File placed in ./config di # InfluxDB Configuration INFLUXDB_HTTP_PORT=8181 # for influxdb3 enterprise database, change this to port 8182 INFLUXDB_HOST=influxdb3-core # for influxdb3 enterprise database, change this to "influxdb3-enterprise" -INFLUXDB_TOKEN= +INFLUXDB_TOKEN=apiv3_99RtQcbFuBYP_wO1fK5aHvIBQqOqOVRAV4cAqAJxm_gpQRUFIpQkONXWfG0cBt_1wOI21Dc2l5K2izMa3gW7mA INFLUXDB_DATABASE=local_system # Your Database name INFLUXDB_ORG=local_org INFLUXDB_NODE_ID=node0 diff --git a/docs/docker-compose/config/admin_token.txt b/docs/docker-compose/config/admin_token.txt new file mode 100644 index 0000000..7246300 --- /dev/null +++ b/docs/docker-compose/config/admin_token.txt @@ -0,0 +1,4 @@ +{ + "token": "apiv3_99RtQcbFuBYP_wO1fK5aHvIBQqOqOVRAV4cAqAJxm_gpQRUFIpQkONXWfG0cBt_1wOI21Dc2l5K2izMa3gW7mA", + "name": "_admin" +} \ No newline at end of file diff --git a/docs/docker-compose/config/production_influxdb_v1.yaml b/docs/docker-compose/config/production_influxdb_v1.yaml new file mode 100644 index 0000000..4cb33db --- /dev/null +++ b/docs/docker-compose/config/production_influxdb_v1.yaml @@ -0,0 +1,302 @@ +--- +Butler-SOS: + logLevel: info + fileLogging: true + logDirectory: log + systemInfo: + enable: true + anonTelemetry: true + configVisualisation: + enable: true + host: 0.0.0.0 + port: 3100 + obfuscate: true + heartbeat: + enable: false + remoteURL: http://localhost + frequency: every 1 hour + dockerHealthCheck: + enable: true + port: 12398 + uptimeMonitor: + enable: true + frequency: every 15 seconds + logLevel: verbose + storeInInfluxdb: + butlerSOSMemoryUsage: true + instanceTag: docker-v1 + storeNewRelic: + enable: false + destinationAccount: [] + metric: + dynamic: + butlerMemoryUsage: + enable: false + butlerUptime: + enable: false + attribute: + static: [] + dynamic: + butlerVersion: + enable: false + qlikSenseEvents: + influxdb: + enable: true + writeFrequency: 5000 + eventCount: + enable: true + influxdb: + measurementName: event_count + tags: + - name: qs_env + value: docker-v1 + rejectedEventCount: + enable: false + influxdb: + measurementName: rejected_event_count + influxdbConfig: + enable: true + host: influxdb-v1 + port: 8086 + version: 1 + maxBatchSize: 100 + v3Config: + database: mydatabase + description: Butler SOS metrics + token: mytoken + retentionDuration: 10d + writeTimeout: 10000 + queryTimeout: 60000 + v2Config: + org: myorg + bucket: mybucket + description: Butler SOS metrics + token: mytoken + retentionDuration: 10d + v1Config: + auth: + enable: false + username: '' + password: '' + dbName: senseops + retentionPolicy: + name: 10d + duration: 10d + includeFields: + activeDocs: false + loadedDocs: false + inMemoryDocs: false + serversToMonitor: + pollingInterval: 30000 + rejectUnauthorized: false + serverTagsDefinition: [] + servers: [] + thirdPartyToolsCredentials: + newRelic: [] + userEvents: + enable: false + excludeUser: [] + udpServerConfig: + serverHost: 0.0.0.0 + portUserActivityEvents: 9997 + messageQueue: + maxConcurrent: 10 + maxSize: 200 + backpressureThreshold: 80 + rateLimit: + enable: false + maxMessagesPerMinute: 600 + maxMessageSize: 65507 + queueMetrics: + influxdb: + enable: false + writeFrequency: 20000 + measurementName: user_events_queue + tags: [] + tags: [] + sendToMQTT: + enable: false + postTo: + everythingTopic: + enable: false + topic: qliksense/userevent + sessionStartTopic: + enable: false + topic: qliksense/userevent/session/start + sessionStopTopic: + enable: false + topic: qliksense/userevent/session/stop + connectionOpenTopic: + enable: false + topic: qliksense/userevent/connection/open + connectionCloseTopic: + enable: false + topic: qliksense/userevent/connection/close + sendToInfluxdb: + enable: false + sendToNewRelic: + enable: false + destinationAccount: [] + scramble: true + logEvents: + udpServerConfig: + serverHost: 0.0.0.0 + portLogEvents: 9996 + messageQueue: + maxConcurrent: 10 + maxSize: 200 + backpressureThreshold: 80 + rateLimit: + enable: false + maxMessagesPerMinute: 600 + maxMessageSize: 65507 + queueMetrics: + influxdb: + enable: false + writeFrequency: 20000 + measurementName: log_events_queue + tags: [] + tags: [] + source: + engine: + enable: false + proxy: + enable: false + repository: + enable: false + scheduler: + enable: false + qixPerf: + enable: false + categorise: + enable: false + rules: [] + ruleDefault: + enable: false + category: [] + enginePerformanceMonitor: + enable: false + appNameLookup: + enable: false + trackRejectedEvents: + enable: false + tags: [] + monitorFilter: + allApps: + enable: false + appExclude: [] + objectType: + allObjectTypes: true + allObjectTypesExclude: [] + someObjectTypesInclude: [] + method: + allMethods: true + allMethodsExclude: [] + someMethodsInclude: [] + appSpecific: + enable: false + app: [] + sendToMQTT: + enable: false + baseTopic: qliksense/logevent + postTo: + baseTopic: true + subsystemTopics: true + sendToInfluxdb: + enable: false + sendToNewRelic: + enable: false + destinationAccount: [] + source: + engine: + enable: false + logLevel: + error: true + warn: true + proxy: + enable: false + logLevel: + error: true + warn: true + repository: + enable: false + logLevel: + error: true + warn: true + scheduler: + enable: false + logLevel: + error: true + warn: true + cert: + clientCert: /nodeapp/config/certificate/client.pem + clientCertKey: /nodeapp/config/certificate/client_key.pem + clientCertCA: /nodeapp/config/certificate/root.pem + clientCertPassphrase: '' + mqttConfig: + enable: false + brokerHost: localhost + brokerPort: 1883 + baseTopic: butler-sos/ + newRelic: + enable: false + event: + url: https://insights-collector.eu01.nr-data.net + header: [] + attribute: + static: [] + dynamic: + butlerSosVersion: + enable: false + metric: + destinationAccount: [] + url: https://insights-collector.eu01.nr-data.net/metric/v1 + header: [] + dynamic: + engine: + memory: + enable: false + cpu: + enable: false + calls: + enable: false + selections: + enable: false + sessions: + enable: false + users: + enable: false + saturated: + enable: false + apps: + docCount: + enable: false + activeDocs: + enable: false + loadedDocs: + enable: false + inMemoryDocs: + enable: false + cache: + cache: + enable: false + proxy: + sessions: + enable: false + attribute: + static: [] + dynamic: + butlerSosVersion: + enable: false + prometheus: + enable: false + host: 0.0.0.0 + port: 9842 + appNames: + enableAppNameExtract: false + extractInterval: 60000 + hostIP: localhost + userSessions: + enableSessionExtract: false + pollingInterval: 30000 + excludeUser: [] diff --git a/docs/docker-compose/config/production_influxdb_v2.yaml b/docs/docker-compose/config/production_influxdb_v2.yaml new file mode 100644 index 0000000..0333182 --- /dev/null +++ b/docs/docker-compose/config/production_influxdb_v2.yaml @@ -0,0 +1,302 @@ +--- +Butler-SOS: + logLevel: info + fileLogging: true + logDirectory: log + systemInfo: + enable: true + anonTelemetry: true + configVisualisation: + enable: true + host: 0.0.0.0 + port: 3100 + obfuscate: true + heartbeat: + enable: false + remoteURL: http://localhost + frequency: every 1 hour + dockerHealthCheck: + enable: true + port: 12398 + uptimeMonitor: + enable: true + frequency: every 15 seconds + logLevel: verbose + storeInInfluxdb: + butlerSOSMemoryUsage: true + instanceTag: docker-v2 + storeNewRelic: + enable: false + destinationAccount: [] + metric: + dynamic: + butlerMemoryUsage: + enable: false + butlerUptime: + enable: false + attribute: + static: [] + dynamic: + butlerVersion: + enable: false + qlikSenseEvents: + influxdb: + enable: true + writeFrequency: 5000 + eventCount: + enable: true + influxdb: + measurementName: event_count + tags: + - name: qs_env + value: docker-v2 + rejectedEventCount: + enable: false + influxdb: + measurementName: rejected_event_count + influxdbConfig: + enable: true + host: influxdb-v2 + port: 8086 + version: 2 + maxBatchSize: 100 + v3Config: + database: mydatabase + description: Butler SOS metrics + token: mytoken + retentionDuration: 10d + writeTimeout: 10000 + queryTimeout: 60000 + v2Config: + org: butler-sos + bucket: butler-sos + description: Butler SOS metrics + token: butlersos-token + retentionDuration: 10d + v1Config: + auth: + enable: false + username: '' + password: '' + dbName: senseops + retentionPolicy: + name: 10d + duration: 10d + includeFields: + activeDocs: false + loadedDocs: false + inMemoryDocs: false + serversToMonitor: + pollingInterval: 30000 + rejectUnauthorized: false + serverTagsDefinition: [] + servers: [] + thirdPartyToolsCredentials: + newRelic: [] + userEvents: + enable: false + excludeUser: [] + udpServerConfig: + serverHost: 0.0.0.0 + portUserActivityEvents: 9997 + messageQueue: + maxConcurrent: 10 + maxSize: 200 + backpressureThreshold: 80 + rateLimit: + enable: false + maxMessagesPerMinute: 600 + maxMessageSize: 65507 + queueMetrics: + influxdb: + enable: false + writeFrequency: 20000 + measurementName: user_events_queue + tags: [] + tags: [] + sendToMQTT: + enable: false + postTo: + everythingTopic: + enable: false + topic: qliksense/userevent + sessionStartTopic: + enable: false + topic: qliksense/userevent/session/start + sessionStopTopic: + enable: false + topic: qliksense/userevent/session/stop + connectionOpenTopic: + enable: false + topic: qliksense/userevent/connection/open + connectionCloseTopic: + enable: false + topic: qliksense/userevent/connection/close + sendToInfluxdb: + enable: false + sendToNewRelic: + enable: false + destinationAccount: [] + scramble: true + logEvents: + udpServerConfig: + serverHost: 0.0.0.0 + portLogEvents: 9996 + messageQueue: + maxConcurrent: 10 + maxSize: 200 + backpressureThreshold: 80 + rateLimit: + enable: false + maxMessagesPerMinute: 600 + maxMessageSize: 65507 + queueMetrics: + influxdb: + enable: false + writeFrequency: 20000 + measurementName: log_events_queue + tags: [] + tags: [] + source: + engine: + enable: false + proxy: + enable: false + repository: + enable: false + scheduler: + enable: false + qixPerf: + enable: false + categorise: + enable: false + rules: [] + ruleDefault: + enable: false + category: [] + enginePerformanceMonitor: + enable: false + appNameLookup: + enable: false + trackRejectedEvents: + enable: false + tags: [] + monitorFilter: + allApps: + enable: false + appExclude: [] + objectType: + allObjectTypes: true + allObjectTypesExclude: [] + someObjectTypesInclude: [] + method: + allMethods: true + allMethodsExclude: [] + someMethodsInclude: [] + appSpecific: + enable: false + app: [] + sendToMQTT: + enable: false + baseTopic: qliksense/logevent + postTo: + baseTopic: true + subsystemTopics: true + sendToInfluxdb: + enable: false + sendToNewRelic: + enable: false + destinationAccount: [] + source: + engine: + enable: false + logLevel: + error: true + warn: true + proxy: + enable: false + logLevel: + error: true + warn: true + repository: + enable: false + logLevel: + error: true + warn: true + scheduler: + enable: false + logLevel: + error: true + warn: true + cert: + clientCert: /nodeapp/config/certificate/client.pem + clientCertKey: /nodeapp/config/certificate/client_key.pem + clientCertCA: /nodeapp/config/certificate/root.pem + clientCertPassphrase: '' + mqttConfig: + enable: false + brokerHost: localhost + brokerPort: 1883 + baseTopic: butler-sos/ + newRelic: + enable: false + event: + url: https://insights-collector.eu01.nr-data.net + header: [] + attribute: + static: [] + dynamic: + butlerSosVersion: + enable: false + metric: + destinationAccount: [] + url: https://insights-collector.eu01.nr-data.net/metric/v1 + header: [] + dynamic: + engine: + memory: + enable: false + cpu: + enable: false + calls: + enable: false + selections: + enable: false + sessions: + enable: false + users: + enable: false + saturated: + enable: false + apps: + docCount: + enable: false + activeDocs: + enable: false + loadedDocs: + enable: false + inMemoryDocs: + enable: false + cache: + cache: + enable: false + proxy: + sessions: + enable: false + attribute: + static: [] + dynamic: + butlerSosVersion: + enable: false + prometheus: + enable: false + host: 0.0.0.0 + port: 9842 + appNames: + enableAppNameExtract: false + extractInterval: 60000 + hostIP: localhost + userSessions: + enableSessionExtract: false + pollingInterval: 30000 + excludeUser: [] diff --git a/docs/docker-compose/config/production_influxdb_v3.yaml b/docs/docker-compose/config/production_influxdb_v3.yaml new file mode 100644 index 0000000..2f7c264 --- /dev/null +++ b/docs/docker-compose/config/production_influxdb_v3.yaml @@ -0,0 +1,302 @@ +--- +Butler-SOS: + logLevel: info + fileLogging: true + logDirectory: log + systemInfo: + enable: true + anonTelemetry: true + configVisualisation: + enable: true + host: 0.0.0.0 + port: 3100 + obfuscate: true + heartbeat: + enable: false + remoteURL: http://localhost + frequency: every 1 hour + dockerHealthCheck: + enable: true + port: 12398 + uptimeMonitor: + enable: true + frequency: every 15 seconds + logLevel: verbose + storeInInfluxdb: + butlerSOSMemoryUsage: true + instanceTag: docker-v3 + storeNewRelic: + enable: false + destinationAccount: [] + metric: + dynamic: + butlerMemoryUsage: + enable: false + butlerUptime: + enable: false + attribute: + static: [] + dynamic: + butlerVersion: + enable: false + qlikSenseEvents: + influxdb: + enable: true + writeFrequency: 5000 + eventCount: + enable: true + influxdb: + measurementName: event_count + tags: + - name: qs_env + value: docker-v3 + rejectedEventCount: + enable: false + influxdb: + measurementName: rejected_event_count + influxdbConfig: + enable: true + host: influxdb-v3-core + port: 8181 + version: 3 + maxBatchSize: 100 + v3Config: + database: butler-sos + description: Butler SOS metrics + token: apiv3_99RtQcbFuBYP_wO1fK5aHvIBQqOqOVRAV4cAqAJxm_gpQRUFIpQkONXWfG0cBt_1wOI21Dc2l5K2izMa3gW7mA + retentionDuration: 10d + writeTimeout: 10000 + queryTimeout: 60000 + v2Config: + org: myorg + bucket: mybucket + description: Butler SOS metrics + token: mytoken + retentionDuration: 10d + v1Config: + auth: + enable: false + username: '' + password: '' + dbName: senseops + retentionPolicy: + name: 10d + duration: 10d + includeFields: + activeDocs: false + loadedDocs: false + inMemoryDocs: false + serversToMonitor: + pollingInterval: 30000 + rejectUnauthorized: false + serverTagsDefinition: [] + servers: [] + thirdPartyToolsCredentials: + newRelic: [] + userEvents: + enable: false + excludeUser: [] + udpServerConfig: + serverHost: 0.0.0.0 + portUserActivityEvents: 9997 + messageQueue: + maxConcurrent: 10 + maxSize: 200 + backpressureThreshold: 80 + rateLimit: + enable: false + maxMessagesPerMinute: 600 + maxMessageSize: 65507 + queueMetrics: + influxdb: + enable: false + writeFrequency: 20000 + measurementName: user_events_queue + tags: [] + tags: [] + sendToMQTT: + enable: false + postTo: + everythingTopic: + enable: false + topic: qliksense/userevent + sessionStartTopic: + enable: false + topic: qliksense/userevent/session/start + sessionStopTopic: + enable: false + topic: qliksense/userevent/session/stop + connectionOpenTopic: + enable: false + topic: qliksense/userevent/connection/open + connectionCloseTopic: + enable: false + topic: qliksense/userevent/connection/close + sendToInfluxdb: + enable: false + sendToNewRelic: + enable: false + destinationAccount: [] + scramble: true + logEvents: + udpServerConfig: + serverHost: 0.0.0.0 + portLogEvents: 9996 + messageQueue: + maxConcurrent: 10 + maxSize: 200 + backpressureThreshold: 80 + rateLimit: + enable: false + maxMessagesPerMinute: 600 + maxMessageSize: 65507 + queueMetrics: + influxdb: + enable: false + writeFrequency: 20000 + measurementName: log_events_queue + tags: [] + tags: [] + source: + engine: + enable: false + proxy: + enable: false + repository: + enable: false + scheduler: + enable: false + qixPerf: + enable: false + categorise: + enable: false + rules: [] + ruleDefault: + enable: false + category: [] + enginePerformanceMonitor: + enable: false + appNameLookup: + enable: false + trackRejectedEvents: + enable: false + tags: [] + monitorFilter: + allApps: + enable: false + appExclude: [] + objectType: + allObjectTypes: true + allObjectTypesExclude: [] + someObjectTypesInclude: [] + method: + allMethods: true + allMethodsExclude: [] + someMethodsInclude: [] + appSpecific: + enable: false + app: [] + sendToMQTT: + enable: false + baseTopic: qliksense/logevent + postTo: + baseTopic: true + subsystemTopics: true + sendToInfluxdb: + enable: false + sendToNewRelic: + enable: false + destinationAccount: [] + source: + engine: + enable: false + logLevel: + error: true + warn: true + proxy: + enable: false + logLevel: + error: true + warn: true + repository: + enable: false + logLevel: + error: true + warn: true + scheduler: + enable: false + logLevel: + error: true + warn: true + cert: + clientCert: /nodeapp/config/certificate/client.pem + clientCertKey: /nodeapp/config/certificate/client_key.pem + clientCertCA: /nodeapp/config/certificate/root.pem + clientCertPassphrase: '' + mqttConfig: + enable: false + brokerHost: localhost + brokerPort: 1883 + baseTopic: butler-sos/ + newRelic: + enable: false + event: + url: https://insights-collector.eu01.nr-data.net + header: [] + attribute: + static: [] + dynamic: + butlerSosVersion: + enable: false + metric: + destinationAccount: [] + url: https://insights-collector.eu01.nr-data.net/metric/v1 + header: [] + dynamic: + engine: + memory: + enable: false + cpu: + enable: false + calls: + enable: false + selections: + enable: false + sessions: + enable: false + users: + enable: false + saturated: + enable: false + apps: + docCount: + enable: false + activeDocs: + enable: false + loadedDocs: + enable: false + inMemoryDocs: + enable: false + cache: + cache: + enable: false + proxy: + sessions: + enable: false + attribute: + static: [] + dynamic: + butlerSosVersion: + enable: false + prometheus: + enable: false + host: 0.0.0.0 + port: 9842 + appNames: + enableAppNameExtract: false + extractInterval: 60000 + hostIP: localhost + userSessions: + enableSessionExtract: false + pollingInterval: 30000 + excludeUser: [] diff --git a/docs/docker-compose/docker-compose_fullstack_influxdb_v1.yml b/docs/docker-compose/docker-compose_fullstack_influxdb_v1.yml index a32663a..cf56586 100644 --- a/docs/docker-compose/docker-compose_fullstack_influxdb_v1.yml +++ b/docs/docker-compose/docker-compose_fullstack_influxdb_v1.yml @@ -1,24 +1,27 @@ -# docker-compose_fullstack_influxdb_v1.yml +name: butler-sos-v1 services: butler-sos: - image: ptarmiganlabs/butler-sos:latest + image: butler-sos:local container_name: butler-sos restart: unless-stopped ports: - - "9997:9997" # UDP user events - - "9996:9996" # UDP log events - - "9842:9842" # Prometheus metrics - - "3100:3100" # Config file visualization + - '9997:9997' # UDP user events + - '9996:9996' # UDP log events + - '9842:9842' # Prometheus metrics + - '3100:3100' # Config file visualization volumes: # Make config file and log files accessible outside of container - - "./config:/nodeapp/config" - - "./log:/nodeapp/log" - command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_influxdb_v1.yaml"] + - './config:/nodeapp/config' + - './test_data/v1/log:/nodeapp/log' + command: ['node', 'src/butler-sos.js', '-c', '/nodeapp/config/production_influxdb_v1.yaml'] logging: - driver: "json-file" + driver: 'json-file' options: - max-file: "5" - max-size: "5m" + max-file: '5' + max-size: '5m' + depends_on: + influxdb: + condition: service_healthy networks: - senseops @@ -27,15 +30,20 @@ services: container_name: influxdb-v1 restart: unless-stopped volumes: - - ./influxdb/data:/var/lib/influxdb # Mount for influxdb data directory - - ./influxdb/config/:/etc/influxdb/ # Mount for influxdb configuration + - ./test_data/v1/influxdb/data:/var/lib/influxdb # Mount for influxdb data directory + - ./test_data/v1/influxdb/config/:/etc/influxdb/ # Mount for influxdb configuration ports: # The API for InfluxDB is served on port 8086 - - "8086:8086" - - "8082:8082" + - '8086:8086' + - '8082:8082' environment: # Disable usage reporting - - "INFLUXDB_REPORTING_DISABLED=true" + - 'INFLUXDB_REPORTING_DISABLED=true' + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:8086/ping'] + interval: 30s + timeout: 10s + retries: 5 networks: - senseops @@ -44,12 +52,21 @@ services: container_name: grafana restart: unless-stopped ports: - - "3000:3000" + - '3000:3000' volumes: - - ./grafana/data:/var/lib/grafana + - ./test_data/v1/grafana/data:/var/lib/grafana + - ./grafana/provisioning/datasources/datasource_v1.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + - ./grafana/provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml + - ./grafana/dashboards:/var/lib/grafana/dashboards + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + depends_on: + influxdb: + condition: service_healthy networks: - senseops networks: senseops: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/docs/docker-compose/docker-compose_fullstack_influxdb_v2.yml b/docs/docker-compose/docker-compose_fullstack_influxdb_v2.yml index ef89bbf..c77b867 100644 --- a/docs/docker-compose/docker-compose_fullstack_influxdb_v2.yml +++ b/docs/docker-compose/docker-compose_fullstack_influxdb_v2.yml @@ -1,24 +1,27 @@ -# docker-compose_fullstack_influxdb_v2.yml +name: butler-sos-v2 services: butler-sos: - image: ptarmiganlabs/butler-sos:latest + image: butler-sos:local container_name: butler-sos restart: unless-stopped ports: - - "9997:9997" # UDP user events - - "9996:9996" # UDP log events - - "9842:9842" # Prometheus metrics - - "3100:3100" # Config file visualization + - '9997:9997' # UDP user events + - '9996:9996' # UDP log events + - '9842:9842' # Prometheus metrics + - '3100:3100' # Config file visualization volumes: # Make config file and log files accessible outside of container - - "./config:/nodeapp/config" - - "./log:/nodeapp/log" - command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/production_influxdb_v2.yaml"] + - './config:/nodeapp/config' + - './test_data/v2/log:/nodeapp/log' + command: ['node', 'src/butler-sos.js', '-c', '/nodeapp/config/production_influxdb_v2.yaml'] logging: - driver: "json-file" + driver: 'json-file' options: - max-file: "5" - max-size: "5m" + max-file: '5' + max-size: '5m' + depends_on: + influxdb: + condition: service_healthy networks: - senseops @@ -27,20 +30,25 @@ services: container_name: influxdb-v2 restart: unless-stopped volumes: - - ./influxdb/data:/var/lib/influxdb2 # Mount for influxdb data directory - - ./influxdb/config/:/etc/influxdb2/ # Mount for influxdb configuration + - ./test_data/v2/influxdb/data:/var/lib/influxdb2 # Mount for influxdb data directory + - ./test_data/v2/influxdb/config/:/etc/influxdb2/ # Mount for influxdb configuration ports: # The API for InfluxDB is served on port 8086 - - "8086:8086" + - '8086:8086' environment: # Initial setup parameters - - "DOCKER_INFLUXDB_INIT_MODE=setup" - - "DOCKER_INFLUXDB_INIT_USERNAME=admin" - - "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123" - - "DOCKER_INFLUXDB_INIT_ORG=butler-sos" - - "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos" - - "DOCKER_INFLUXDB_INIT_RETENTION=10d" - - "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token" + - 'DOCKER_INFLUXDB_INIT_MODE=setup' + - 'DOCKER_INFLUXDB_INIT_USERNAME=admin' + - 'DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123' + - 'DOCKER_INFLUXDB_INIT_ORG=butler-sos' + - 'DOCKER_INFLUXDB_INIT_BUCKET=butler-sos' + - 'DOCKER_INFLUXDB_INIT_RETENTION=10d' + - 'DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token' + healthcheck: + test: ['CMD', 'influx', 'ping'] + interval: 30s + timeout: 10s + retries: 5 networks: - senseops @@ -49,12 +57,21 @@ services: container_name: grafana restart: unless-stopped ports: - - "3000:3000" + - '3000:3000' volumes: - - ./grafana/data:/var/lib/grafana + - ./test_data/v2/grafana/data:/var/lib/grafana + - ./grafana/provisioning/datasources/datasource_v2.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + - ./grafana/provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml + - ./grafana/dashboards:/var/lib/grafana/dashboards + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + depends_on: + influxdb: + condition: service_healthy networks: - senseops networks: senseops: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/docs/docker-compose/docker-compose_fullstack_influxdb_v3.yml b/docs/docker-compose/docker-compose_fullstack_influxdb_v3.yml index efacbc8..72d1234 100644 --- a/docs/docker-compose/docker-compose_fullstack_influxdb_v3.yml +++ b/docs/docker-compose/docker-compose_fullstack_influxdb_v3.yml @@ -1,29 +1,31 @@ -# docker-compose_fullstack_influxdb_v3.yml +name: butler-sos-v3 # InfluxDB v3.x (Core) - using the InfluxDB 3.x Community Edition # Inspiration from https://github.com/InfluxCommunity/TIG-Stack-using-InfluxDB-3/blob/main/docker-compose.yml services: butler-sos: - image: ptarmiganlabs/butler-sos:latest + image: butler-sos:local container_name: butler-sos restart: unless-stopped ports: - - "9997:9997" # UDP user events - - "9996:9996" # UDP log events - - "9842:9842" # Prometheus metrics - - "3100:3100" # Config file visualization + - '9997:9997' # UDP user events + - '9996:9996' # UDP log events + - '9842:9842' # Prometheus metrics + - '3100:3100' # Config file visualization volumes: # Make config file and log files accessible outside of container - - "./config:/nodeapp/config" - - "./log:/nodeapp/log" - command: ["node", "src/butler-sos.js", "-c", "/nodeapp/config/${BUTLER_SOS_CONFIG_FILE}"] + - './config:/nodeapp/config' + - './test_data/v3/log:/nodeapp/log' + command: ['node', 'src/butler-sos.js', '-c', '/nodeapp/config/${BUTLER_SOS_CONFIG_FILE}'] logging: - driver: "json-file" + driver: 'json-file' options: - max-file: "5" - max-size: "5m" + max-file: '5' + max-size: '5m' depends_on: - # Or switch to influxdb3-enterprise as needed - - influxdb-v3-core + influxdb-v3-core: + condition: service_healthy + setup-influxdb: + condition: service_completed_successfully networks: - senseops @@ -41,44 +43,74 @@ services: - --node-id=${INFLUXDB_NODE_ID} - --object-store=file - --data-dir=/var/lib/influxdb3 + - --admin-token-file=/admin_token.txt volumes: - - ./influxdb/data:/var/lib/influxdb3 # Mount for influxdb data directory - - ./influxdb/config/:/etc/influxdb3/ # Mount for influxdb configuration + - ./test_data/v3/influxdb/data:/var/lib/influxdb3 # Mount for influxdb data directory + - ./test_data/v3/influxdb/config/:/etc/influxdb3/ # Mount for influxdb configuration + - ./config/admin_token.txt:/admin_token.txt # environment: - # InfluxDB v3 setup - uses similar setup to v2 but different internal architecture - # - "DOCKER_INFLUXDB_INIT_MODE=setup" - # - "DOCKER_INFLUXDB_INIT_USERNAME=admin" - # - "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123" - # - "DOCKER_INFLUXDB_INIT_ORG=butler-sos" - # - "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos" - # - "DOCKER_INFLUXDB_INIT_DATABASE=butler-sos" # v3 uses database concept - # - "DOCKER_INFLUXDB_INIT_RETENTION=10d" - # - "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token" + # InfluxDB v3 setup - uses similar setup to v2 but different internal architecture + # - "DOCKER_INFLUXDB_INIT_MODE=setup" + # - "DOCKER_INFLUXDB_INIT_USERNAME=admin" + # - "DOCKER_INFLUXDB_INIT_PASSWORD=butlersos123" + # - "DOCKER_INFLUXDB_INIT_ORG=butler-sos" + # - "DOCKER_INFLUXDB_INIT_BUCKET=butler-sos" + # - "DOCKER_INFLUXDB_INIT_DATABASE=butler-sos" # v3 uses database concept + # - "DOCKER_INFLUXDB_INIT_RETENTION=10d" + # - "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=butlersos-token" healthcheck: - test: ["CMD-SHELL", "curl -f -H 'Authorization: Bearer ${INFLUXDB_TOKEN}' http://localhost:8181/health || exit 1"] + test: + [ + 'CMD-SHELL', + "curl -f -H 'Authorization: Bearer ${INFLUXDB_TOKEN}' http://localhost:8181/health || exit 1", + ] interval: 30s timeout: 10s retries: 3 networks: - senseops + setup-influxdb: + image: influxdb:3-core + container_name: setup-influxdb + restart: no + entrypoint: ['/bin/sh', '-c'] + command: + - | + echo "Creating database $${INFLUXDB_DATABASE}..." + influxdb3 create database $${INFLUXDB_DATABASE} --host http://influxdb-v3-core:8181 --token $${INFLUXDB_TOKEN} || true + echo "Database creation attempt finished." + environment: + - INFLUXDB_TOKEN=${INFLUXDB_TOKEN} + - INFLUXDB_DATABASE=${INFLUXDB_DATABASE} + depends_on: + influxdb-v3-core: + condition: service_healthy + networks: + - senseops + grafana: image: grafana/grafana:latest container_name: grafana restart: unless-stopped ports: - - "${GRAFANA_PORT}:3000" + - '${GRAFANA_PORT}:3000' environment: - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER} - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin volumes: - - ./grafana/data:/var/lib/grafana + - ./test_data/v3/grafana/data:/var/lib/grafana + - ./grafana/provisioning/datasources/datasource_v3.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + - ./grafana/provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml + - ./grafana/dashboards:/var/lib/grafana/dashboards depends_on: - # Or switch to influxdb3-enterprise as needed - - influxdb-v3-core + influxdb-v3-core: + condition: service_healthy networks: - senseops networks: senseops: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/docs/docker-compose/grafana/dashboards/butler_sos_test.json b/docs/docker-compose/grafana/dashboards/butler_sos_test.json new file mode 100644 index 0000000..7d7bac1 --- /dev/null +++ b/docs/docker-compose/grafana/dashboards/butler_sos_test.json @@ -0,0 +1,272 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Verifies that the Butler SOS stack (Butler SOS, InfluxDB and Grafana) is working correctly.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "bf7gj7evi44xsd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "alias": "Heap total", + "datasource": { + "type": "influxdb", + "uid": "bf7gj7evi44xsd" + }, + "groupBy": [ + { + "params": ["$__interval"], + "type": "time" + }, + { + "params": ["none"], + "type": "fill" + } + ], + "measurement": "butlersos_memory_usage", + "orderByTime": "ASC", + "policy": "default", + "refId": "Heap total", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["heap_total"], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "Heap used", + "datasource": { + "type": "influxdb", + "uid": "bf7gj7evi44xsd" + }, + "groupBy": [ + { + "params": ["$__interval"], + "type": "time" + }, + { + "params": ["none"], + "type": "fill" + } + ], + "hide": false, + "measurement": "butlersos_memory_usage", + "orderByTime": "ASC", + "policy": "default", + "refId": "Heap used", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["heap_used"], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "Process memory", + "datasource": { + "type": "influxdb", + "uid": "bf7gj7evi44xsd" + }, + "groupBy": [ + { + "params": ["$__interval"], + "type": "time" + }, + { + "params": ["none"], + "type": "fill" + } + ], + "hide": false, + "measurement": "butlersos_memory_usage", + "orderByTime": "ASC", + "policy": "default", + "refId": "Process memory", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["process_memory"], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "External", + "datasource": { + "type": "influxdb", + "uid": "bf7gj7evi44xsd" + }, + "groupBy": [ + { + "params": ["$__interval"], + "type": "time" + }, + { + "params": ["none"], + "type": "fill" + } + ], + "hide": false, + "measurement": "butlersos_memory_usage", + "orderByTime": "ASC", + "policy": "default", + "refId": "External", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["external"], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "New panel", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "refresh": "10s", + "timepicker": {}, + "timezone": "browser", + "title": "Butler SOS Docker container test dashboard", + "uid": "adjpfhm", + "version": 2 +} diff --git a/docs/docker-compose/grafana/provisioning/dashboards/dashboard.yaml b/docs/docker-compose/grafana/provisioning/dashboards/dashboard.yaml new file mode 100644 index 0000000..d3826da --- /dev/null +++ b/docs/docker-compose/grafana/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'Default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/docs/docker-compose/grafana/provisioning/datasources/datasource_v1.yaml b/docs/docker-compose/grafana/provisioning/datasources/datasource_v1.yaml new file mode 100644 index 0000000..e8938de --- /dev/null +++ b/docs/docker-compose/grafana/provisioning/datasources/datasource_v1.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: senseops + type: influxdb + access: proxy + url: http://influxdb-v1:8086 + database: senseops + user: goran + secureJsonData: + password: mysecret + jsonData: + httpMode: GET + uid: bf7gj7evi44xsd diff --git a/docs/docker-compose/grafana/provisioning/datasources/datasource_v2.yaml b/docs/docker-compose/grafana/provisioning/datasources/datasource_v2.yaml new file mode 100644 index 0000000..7a0292c --- /dev/null +++ b/docs/docker-compose/grafana/provisioning/datasources/datasource_v2.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: Butler SOS + type: influxdb + access: proxy + url: http://influxdb:8086 + database: butler-sos + jsonData: + httpMode: GET + httpHeaderName1: Authorization + secureJsonData: + httpHeaderValue1: Token butlersos-token + uid: bf7gj7evi44xsd diff --git a/docs/docker-compose/grafana/provisioning/datasources/datasource_v3.yaml b/docs/docker-compose/grafana/provisioning/datasources/datasource_v3.yaml new file mode 100644 index 0000000..db73c3c --- /dev/null +++ b/docs/docker-compose/grafana/provisioning/datasources/datasource_v3.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: Butler SOS + type: influxdb + access: proxy + url: http://influxdb-v3-core:8181 + database: butler-sos + jsonData: + httpMode: GET + httpHeaderName1: Authorization + secureJsonData: + httpHeaderValue1: Token apiv3_99RtQcbFuBYP_wO1fK5aHvIBQqOqOVRAV4cAqAJxm_gpQRUFIpQkONXWfG0cBt_1wOI21Dc2l5K2izMa3gW7mA + uid: bf7gj7evi44xsd diff --git a/src/globals.js b/src/globals.js index 95e33a2..5ab0abc 100755 --- a/src/globals.js +++ b/src/globals.js @@ -965,7 +965,7 @@ Configuration File: this.logger.info(`INFLUXDB3 INIT: Port: ${port}`); this.logger.info(`INFLUXDB3 INIT: Database: ${database}`); this.logger.info(`INFLUXDB3 INIT: Token: ${tokenPreview}`); - this.logger.info(`INFLUXDB3 INIT: Socket timeout: ${timeout}ms`); + this.logger.info(`INFLUXDB3 INIT: Socket timeout: ${writeTimeout}ms`); this.logger.info(`INFLUXDB3 INIT: Query timeout: ${queryTimeout}ms`); } catch (testErr) { this.logger.warn( @@ -978,7 +978,7 @@ Configuration File: this.logger.info(`INFLUXDB3 INIT: Port: ${port}`); this.logger.info(`INFLUXDB3 INIT: Database: ${database}`); this.logger.info(`INFLUXDB3 INIT: Token: ${tokenPreview}`); - this.logger.info(`INFLUXDB3 INIT: Socket timeout: ${timeout}ms`); + this.logger.info(`INFLUXDB3 INIT: Socket timeout: ${writeTimeout}ms`); this.logger.info(`INFLUXDB3 INIT: Query timeout: ${queryTimeout}ms`); } } catch (err) { From b9dc4d888e0912910d49317cdbca8fa7de3ebf31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 18 Dec 2025 15:54:03 +0100 Subject: [PATCH 2/3] fix(docker): Stabilize the Docker image build process --- .dockerignore | 8 +++++++- Dockerfile | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index a6e5f3e..18b78a7 100755 --- a/.dockerignore +++ b/.dockerignore @@ -28,7 +28,6 @@ img log scriptlog node_modules -package-lock.json renovate.json release release-macos @@ -42,3 +41,10 @@ src/.snyk src/config src/docker-compose src/log + +coverage +build +jest.config.mjs +eslint.config.js +CHANGELOG.md +changelog_old.md diff --git a/Dockerfile b/Dockerfile index 81b9253..7f084b5 100755 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,8 @@ LABEL description="Real-time operational metrics for Qlik Sense Enterprise on Wi WORKDIR /nodeapp # Install app dependencies separately (creating a separate layer for node_modules, effectively caching them between image rebuilds) -COPY package.json . -RUN npm i +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev # Copy app's source files COPY . . From 127a191b9601627b43b0e1eaa807bbac2eeabbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 18 Dec 2025 16:03:21 +0100 Subject: [PATCH 3/3] Fix .env file for Influx v3 version of Docker compose file --- .gitignore | 1 - docs/docker-compose/{.env_influxdb_v3 => .env} | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) rename docs/docker-compose/{.env_influxdb_v3 => .env} (74%) diff --git a/.gitignore b/.gitignore index 4acd5cc..44f002d 100755 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,3 @@ build.cjs butler-sos src/config/production_vac.yaml tmp/ -docs/docker-compose/.env diff --git a/docs/docker-compose/.env_influxdb_v3 b/docs/docker-compose/.env similarity index 74% rename from docs/docker-compose/.env_influxdb_v3 rename to docs/docker-compose/.env index 8abf6de..9f7af83 100644 --- a/docs/docker-compose/.env_influxdb_v3 +++ b/docs/docker-compose/.env @@ -5,10 +5,8 @@ BUTLER_SOS_CONFIG_FILE=/production_influxdb_v3.yaml # File placed in ./config di # InfluxDB Configuration INFLUXDB_HTTP_PORT=8181 # for influxdb3 enterprise database, change this to port 8182 -INFLUXDB_HOST=influxdb3-core # for influxdb3 enterprise database, change this to "influxdb3-enterprise" INFLUXDB_TOKEN=apiv3_99RtQcbFuBYP_wO1fK5aHvIBQqOqOVRAV4cAqAJxm_gpQRUFIpQkONXWfG0cBt_1wOI21Dc2l5K2izMa3gW7mA -INFLUXDB_DATABASE=local_system # Your Database name -INFLUXDB_ORG=local_org +INFLUXDB_DATABASE=butler-sos # Your Database name INFLUXDB_NODE_ID=node0 # Grafana Configuration