Added Redis integration (#3)

This commit is contained in:
tess1o
2024-07-08 00:36:52 +03:00
committed by GitHub
parent 42b4c4a4b0
commit d75b24fa48
16 changed files with 260 additions and 22 deletions

View File

@@ -3,8 +3,7 @@ name: Docker Image CI
on:
push:
branches: [ "main" ]
tags:
- '*'
tags: ['*']
pull_request:
branches: [ "main" ]
@@ -27,7 +26,13 @@ jobs:
-
name: Extract GitHub tag
id: extract_tag
run: echo "GIT_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
run: |
echo "GIT_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
if: startsWith(github.ref, 'refs/tags/')
-
name: Print GitHub tag
run: |
echo "The value of GIT_TAG is: $GIT_TAG"
if: startsWith(github.ref, 'refs/tags/')
-
name: Build and push
@@ -37,4 +42,4 @@ jobs:
platforms: linux/amd64,linux/arm64
tags: |
tess1o/go-ecoflow-exporter:latest
tess1o/go-ecoflow-exporter:${{ steps.extract_tag.outputs.GIT_TAG || github.sha }}
tess1o/go-ecoflow-exporter:${{ env.GIT_TAG || github.sha }}

View File

@@ -34,9 +34,10 @@ start-prometheus:
start-timescale:
docker-compose -f docker-compose/timescale-compose.yml up -d
start-redis:
docker-compose -f docker-compose/redis-compose.yml up -d
start-exporter-local:
docker stop go_ecoflow_exporter
docker rm go_ecoflow_exporter
docker-compose -f docker-compose/exporter-local-compose.yml up --build --force-recreate --no-deps -d
start-exporter-remote:
@@ -50,4 +51,4 @@ stop-exporter:
exporter-logs:
docker logs -f go_ecoflow_exporter
.PHONY: build push build-push migrateup migrateup1 migratedown migratedown1 new_migration start-grafana start-prometheus start-timescale start-exporter-local start-exporter-remote stop-all
.PHONY: build push build-push migrateup migrateup1 migratedown migratedown1 new_migration start-grafana start-prometheus start-timescale start-redis start-exporter-local start-exporter-remote stop-all

View File

@@ -7,7 +7,7 @@ are:
1. Prometheus
2. TimescaleDB
3. Redis (planned, not implemented yet)
3. Redis
Depending on your configuration you can export the metrics to one of those systems or to all at once.
@@ -26,17 +26,18 @@ Other known to me projects use MQTT protocol to scrap the metrics, this implemen
6. Go to https://developer-eu.ecoflow.com/us/security and create new AccessKey and SecretKey
## How to run the Prometheus, Exporter and Grafana using docker-compose
See documentation here: [Prometheus](docs/prometheus.md)
## How to run the TimescaleDB, Exporter and Grafana using docker-compose
See documentation here: [TimescaleDB](docs/timescaledb.md)
TimescaleDB allows to build more complex logic if you want so. For instance, you can calculate how long you had power
outages and how long the grid power was on. Since all metrics are stored in a PostgreSQL database (TimescaleDB to be
precise), you have the power of SQL to build any kind of metrics or reports you want. Prometheus doesn't provide such
flexibility.
## How to run the Redis, Exporter and Grafana using docker-compose
See documentation here: [Redis](docs/redis.md)
## Compare to other exporters
This implementation is inspired by https://github.com/berezhinskiy/ecoflow_exporter, and it's fully

View File

@@ -26,7 +26,7 @@ GRAFANA_USERNAME=grafana
GRAFANA_PASSWORD=grafana
# Enable TimescaleDB integration
TIMESCALE_ENABLED=true
TIMESCALE_ENABLED=false
# TimescaleDB username
TIMESCALE_USERNAME=postgres
@@ -35,4 +35,19 @@ TIMESCALE_USERNAME=postgres
TIMESCALE_PASSWORD=postgres
# TimescaleDB connection string
TIMESCALE_URL=postgresql://postgres:postgres@timescaledb:5432/postgres?sslmode=disable
TIMESCALE_URL=postgresql://postgres:postgres@timescaledb:5432/postgres?sslmode=disable
# Redis Enabled
REDIS_ENABLED=false
# Redis URL
REDIS_URL=redis:6379
# Redis DB
REDIS_DB=0
# Redis username. Keep empty if not specified
REDIS_USER=
# Redis password. Keep empty if not specified
REDIS_PASSWORD=

View File

@@ -13,4 +13,9 @@ services:
PROMETHEUS_ENABLED: ${PROMETHEUS_ENABLED}
METRIC_PREFIX: ${METRIC_PREFIX}
TIMESCALE_ENABLED: ${TIMESCALE_ENABLED}
TIMESCALE_URL: ${TIMESCALE_URL}
TIMESCALE_URL: ${TIMESCALE_URL}
REDIS_ENABLED: ${REDIS_ENABLED}
REDIS_URL: ${REDIS_URL}
REDIS_DB: ${REDIS_DB}
REDIS_USER: ${REDIS_USER}
REDIS_PASSWORD: ${REDIS_PASSWORD}

View File

@@ -9,4 +9,9 @@ services:
ECOFLOW_ACCESS_KEY: ${ECOFLOW_ACCESS_KEY}
ECOFLOW_SECRET_KEY: ${ECOFLOW_SECRET_KEY}
METRIC_PREFIX: ${METRIC_PREFIX}
SCRAPING_INTERVAL: ${SCRAPING_INTERVAL}
SCRAPING_INTERVAL: ${SCRAPING_INTERVAL}
REDIS_ENABLED: ${REDIS_ENABLED}
REDIS_URL: ${REDIS_URL}
REDIS_DB: ${REDIS_DB}
REDIS_USER: ${REDIS_USER}
REDIS_PASSWORD: ${REDIS_PASSWORD}

View File

@@ -0,0 +1,16 @@
version: '3.8'
services:
redis:
image: redislabs/redistimeseries:latest
container_name: redis-timeseries
volumes:
- ./redis/redis.conf:/usr/local/etc/redis/redis.conf
- redis_data:/data
command: ["redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redistimeseries.so"]
ports:
- "6379:6379"
volumes:
redis_data:
driver: local

View File

@@ -0,0 +1,3 @@
save 3600 1
appendonly yes
appendfsync everysec

View File

@@ -1,4 +1,4 @@
## How to run the Exporter, Prometheus and Grafana using docker-compose
## How to run the Prometheus, Exporter and Grafana using docker-compose
1. Go to docker-compose folder: `cd docker-compose`
2. Update `.env` file with two mandatory parameters:
@@ -13,8 +13,8 @@
Rest API in order to get the data. Default value is 30 seconds. Align this value
with `docker-compose/prometheus/prometheus.yml`
- `DEBUG_ENABLED` - enable debug log messages. Default value is "false". To enable use values `true` or `1`
- `GRAFANA_USERNAME` - admin username in Grafana. Can be changed later in Grafana UI
- `GRAFANA_PASSWORD` - admin password in Grafana. Can be changed later in Grafana UI
- `GRAFANA_USERNAME` - admin username in Grafana. Default value: `grafana`. Can be changed later in Grafana UI
- `GRAFANA_PASSWORD` - admin password in Grafana. Default value: `grafana`. Can be changed later in Grafana UI
4. Save `.env` file with your changes.
5. Start all containers: `docker-compose -f docker-compose/grafana-compose.yml -f docker-compose/exporter-remote-compose.yml up -f docker-compose/prometheus-compose.yml up -d`
```

41
docs/redis.md Normal file
View File

@@ -0,0 +1,41 @@
## Data structure
All metrics are stored as TimeSeries with key structure:
`ts:%device_serial_number%:%metric_name%`
For instance:
```
ts:R123YHY5ABCE1346:ecoflow_bms_bms_status_cycles
ts:R123YHY5ABCE1346:ecoflow_bms_bms_status_soc
```
## How to run the Redis, Exporter and Grafana using docker-compose
1. Go to docker-compose folder: `cd docker-compose`
2. Update `.env` file with two mandatory parameters:
- `ECOFLOW_ACCESS_KEY` - the access key from the Ecoflow development website
- `ECOFLOW_SECRET_KEY` - the secret key from the Ecoflow development website
- `REDIS_ENABLED` - enable integration with Redis
3. (OPTIONALLY) Update other variables if you need to:
- `REDIS_URL` - Redis url. Default value: `localhost:6379`
- `REDIS_DB` - Redis database. Default value: `0`
- `REDIS_USER` - Redis username. Default value: no value
- `REDIS_PASSWORD` - Redis password. Default value: no value
- `METRIC_PREFIX`: the prefix that will be added to all metrics. Default value is `ecoflow`. For instance
metric `bms_bmsStatus.minCellTemp` will be exported to prometheus as `ecoflow.bms_bmsStatus.minCellTemp`.
- `SCRAPING_INTERVAL` - scrapping interval in seconds. How often should the exporter execute requests to Ecoflow
Rest API in order to get the data. Default value is 30 seconds.
- `DEBUG_ENABLED` - enable debug log messages. Default value is "false". To enable use values `true` or `1`
- `GRAFANA_USERNAME` - admin username in Grafana. Default value: `grafana`. Can be changed later in Grafana UI
- `GRAFANA_PASSWORD` - admin password in Grafana. Default value: `grafana`. Can be changed later in Grafana UI
4. Save `.env` file with your changes.
5. Adjust redis persistence configuration if needed at: `docker-compose/redis/redis.conf`
6. Start Redis container: `docker-compose -f docker-compose/redis-compose.yml up -d`
7. Start the exporter and
grafana: `docker-compose -f docker-compose/grafana-compose.yml -f docker-compose/exporter-remote-compose.yml up -d`
8. The services are available here:
- http://localhost:3000 - Grafana
- Redis is available at the value of `REDIS_URL` variable
9. Configure a new Redis datasource in Grafana according to example below:
10. Create your dashboard (TODO: add example of a dashboard)

View File

@@ -22,7 +22,7 @@ You can find usage examples below in this document.
There is no cleanup procedure implemented at the moment, so you might want to cleanup all records by yourself.
## How to run the Exporter, TimescaleDB and Grafana using docker-compose
## How to run the TimescaleDB, Exporter and Grafana using docker-compose
1. Go to docker-compose folder: `cd docker-compose`
2. Update `.env` file with two mandatory parameters:
@@ -40,8 +40,9 @@ There is no cleanup procedure implemented at the moment, so you might want to cl
- `SCRAPING_INTERVAL` - scrapping interval in seconds. How often should the exporter execute requests to Ecoflow
Rest API in order to get the data. Default value is 30 seconds.
- `DEBUG_ENABLED` - enable debug log messages. Default value is "false". To enable use values `true` or `1`
- `GRAFANA_USERNAME` - admin username in Grafana. Can be changed later in Grafana UI
- `GRAFANA_PASSWORD` - admin password in Grafana. Can be changed later in Grafana UI
- `GRAFANA_USERNAME` - admin username in Grafana. Default value: `grafana`. Can be changed later in Grafana UI
- `GRAFANA_PASSWORD` - admin password in Grafana. Default value: `grafana`. Can be changed later in Grafana UI
4. Save `.env` file with your changes.
5. Start timescaledb container: `docker-compose -f docker-compose/timescale-compose.yml up -d`
6. Start the exporter and

2
go.mod
View File

@@ -3,6 +3,7 @@ module go-ecoflow-exporter
go 1.22
require (
github.com/go-redis/redis/v8 v8.11.5
github.com/golang-migrate/migrate/v4 v4.17.1
github.com/jackc/pgx/v5 v5.6.0
github.com/prometheus/client_golang v1.19.1
@@ -12,6 +13,7 @@ require (
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect

16
go.sum
View File

@@ -9,6 +9,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dhui/dktest v0.4.1 h1:/w+IWuDXVymg3IrRJCHHOkMK10m9aNVMOyD0X12YVTg=
github.com/dhui/dktest v0.4.1/go.mod h1:DdOqcUpL7vgyP4GlF3X3w7HbSlz8cEQzwewPveYEQbA=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
@@ -19,6 +21,10 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4=
@@ -44,6 +50,12 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
@@ -86,6 +98,10 @@ golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

42
main.go
View File

@@ -2,6 +2,7 @@ package main
import (
"context"
"github.com/go-redis/redis/v8"
"github.com/tess1o/go-ecoflow"
"log"
"log/slog"
@@ -12,16 +13,28 @@ import (
"time"
)
// generic
const (
defaultMetricPrefix = "ecoflow"
defaultInterval = 30
defaultMetricsPort = "2112"
)
// prometheus
const (
defaultMetricsPort = "2112"
)
// timescaledb
const (
timescaleDbSource = "file://migrations/timescale"
)
// redis
const (
defaultRedisUrl = "localhost:6379"
defaultRedisDb = 0
)
type Shutdownable interface {
Close(ctx context.Context)
}
@@ -44,6 +57,7 @@ func main() {
handlers = enablePrometheus(metricPrefix, handlers)
handlers = enableTimescaleDb(metricPrefix, handlers)
handlers = enableRedis(metricPrefix, handlers)
if len(handlers) == 0 {
slog.Error("No metric handlers enabled, exiting")
@@ -80,6 +94,32 @@ func enableTimescaleDb(metricPrefix string, handlers []MetricHandler) []MetricHa
return handlers
}
func enableRedis(prefix string, handlers []MetricHandler) []MetricHandler {
if isOptionEnabled("REDIS_ENABLED") {
config := &redis.Options{
Addr: getStringOrDefault("REDIS_URL", defaultRedisUrl),
DB: getIntOrDefault("REDIS_DB", defaultRedisDb),
}
redisUser, exists := os.LookupEnv("REDIS_USER")
if exists {
config.Username = redisUser
}
redisPassword, exists := os.LookupEnv("REDIS_PASSWORD")
if exists {
config.Password = redisPassword
}
redisExporter := NewRedisExporter(&RedisExporterConfig{
Prefix: prefix,
RedisConfig: config,
})
handlers = append(handlers, redisExporter)
}
return handlers
}
func enablePrometheus(metricPrefix string, handlers []MetricHandler) []MetricHandler {
if isOptionEnabled("PROMETHEUS_ENABLED") {
port := getStringOrDefault("PROMETHEUS_PORT", defaultMetricsPort)

87
redis.go Normal file
View File

@@ -0,0 +1,87 @@
package main
import (
"context"
"fmt"
"github.com/go-redis/redis/v8"
"github.com/tess1o/go-ecoflow"
"log/slog"
"strings"
"time"
)
// check that RedisExporter implements MetricHandler
var _ MetricHandler = (*RedisExporter)(nil)
type RedisExporterConfig struct {
Prefix string
RedisConfig *redis.Options
}
type RedisExporter struct {
prefix string
client *redis.Client
}
func NewRedisExporter(config *RedisExporterConfig) *RedisExporter {
slog.Info("Creating redis exporter")
client := redis.NewClient(config.RedisConfig)
return &RedisExporter{
prefix: config.Prefix,
client: client,
}
}
func (r *RedisExporter) Handle(ctx context.Context, device ecoflow.DeviceInfo, rawParameters map[string]interface{}) {
if device.Online == 0 {
slog.Info("Device is offline. Setting all metrics to 0", "SN", device.SN)
rawParameters = r.handleOfflineDevice(rawParameters, device)
}
rawParameters["online"] = float64(device.Online)
r.handleTimeScaleMetrics(ctx, rawParameters, device)
}
func (r *RedisExporter) handleOfflineDevice(metrics map[string]interface{}, dev ecoflow.DeviceInfo) map[string]interface{} {
for k := range metrics {
if strings.Contains(k, dev.SN) {
metrics[k] = 0
}
}
return metrics
}
func (r *RedisExporter) handleTimeScaleMetrics(ctx context.Context, metrics map[string]interface{}, dev ecoflow.DeviceInfo) {
slog.Info("Handling metrics for device", "dev", dev.SN)
timestamp := time.Now().Unix()
pipe := r.client.Pipeline()
for field, val := range metrics {
metricName, _, err := generateMetricName(field, r.prefix, dev.SN)
if err != nil {
slog.Error("Unable to generate metric name", "metric", field)
continue
}
slog.Debug("Updating metric", "metric", metricName, "value", val, "device", dev.SN)
_, ok := val.([]interface{})
if ok {
slog.Debug("The value is an array, skipping it", "metric", metricName)
continue
}
floatVal, ok := val.(float64)
if ok {
tsKey := fmt.Sprintf("ts:%s:%s", dev.SN, metricName)
pipe.Do(ctx, "TS.ADD", tsKey, timestamp, floatVal)
} else {
slog.Error("Unable to convert value to float, skipping metric", "value", val, "metric", metricName)
}
}
_, err := pipe.Exec(ctx)
if err != nil {
slog.Error("Unable to insert metrics", "redis_error", err)
} else {
slog.Debug("Inserted metrics", "device", dev.SN)
}
}

View File

@@ -17,7 +17,7 @@ import (
"strings"
)
// check that PrometheusExporter implements MetricHandler
// check that TimescaleExporter implements MetricHandler
var _ MetricHandler = (*TimescaleExporter)(nil)
var _ Shutdownable = (*TimescaleExporter)(nil)