topaLE
/
fritzbox_exporter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

added support for caching results between collections

pull/4/head
sberk42 4 years ago
parent
320ad2d613
commit
33fc59a3c8
3 changed files with 139 additions and 47 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 141
      main.go
  2. 27
      metrics-lua.json
  3. 18
      metrics.json

141
main.go
View File

@ -17,7 +17,6 @@ package main
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log" "log"
@ -40,6 +39,9 @@ import (
const serviceLoadRetryTime = 1 * time.Minute const serviceLoadRetryTime = 1 * time.Minute
// minimum TTL for cached results in seconds
const minCacheTTL = 30
var ( var (
flag_test = flag.Bool("test", false, "print all available metrics to stdout") flag_test = flag.Bool("test", false, "print all available metrics to stdout")
flag_luatest = flag.Bool("testLua", false, "read luaTest.json file make all contained calls and dump results") flag_luatest = flag.Bool("testLua", false, "read luaTest.json file make all contained calls and dump results")
@ -69,6 +71,26 @@ var (
Help: "Number of lua collection errors.", Help: "Number of lua collection errors.",
}) })
) )
var collectLuaResultsCached = prometheus.NewCounter(prometheus.CounterOpts{
Name: "fritzbox_exporter_results_cached",
Help: "Number of results taken from cache.",
ConstLabels: prometheus.Labels{"Cache": "LUA"},
})
var collectUpnpResultsCached = prometheus.NewCounter(prometheus.CounterOpts{
Name: "fritzbox_exporter_results_cached",
Help: "Number of results taken from cache.",
ConstLabels: prometheus.Labels{"Cache": "UPNP"},
})
var collectLuaResultsLoaded = prometheus.NewCounter(prometheus.CounterOpts{
Name: "fritzbox_exporter_results_loaded",
Help: "Number of results loaded from fritzbox.",
ConstLabels: prometheus.Labels{"Cache": "LUA"},
})
var collectUpnpResultsLoaded = prometheus.NewCounter(prometheus.CounterOpts{
Name: "fritzbox_exporter_results_loaded",
Help: "Number of results loaded from fritzbox.",
ConstLabels: prometheus.Labels{"Cache": "UPNP"},
})
type JSON_PromDesc struct { type JSON_PromDesc struct {
FqName string `json:"fqName"` FqName string `json:"fqName"`
@ -93,6 +115,7 @@ type Metric struct {
OkValue string `json:"okValue"` OkValue string `json:"okValue"`
PromDesc JSON_PromDesc `json:"promDesc"` PromDesc JSON_PromDesc `json:"promDesc"`
PromType string `json:"promType"` PromType string `json:"promType"`
CacheEntryTTL int64 `json:"cacheEntryTTL"`
// initialized at startup // initialized at startup
Desc *prometheus.Desc Desc *prometheus.Desc
@ -111,13 +134,14 @@ type LuaLabelRename struct {
type LuaMetric struct { type LuaMetric struct {
// initialized loading JSON // initialized loading JSON
Path string `json:"path"` Path string `json:"path"`
Params string `json:"params"` Params string `json:"params"`
ResultPath string `json:"resultPath"` ResultPath string `json:"resultPath"`
ResultKey string `json:"resultKey"` ResultKey string `json:"resultKey"`
OkValue string `json:"okValue"` OkValue string `json:"okValue"`
PromDesc JSON_PromDesc `json:"promDesc"` PromDesc JSON_PromDesc `json:"promDesc"`
PromType string `json:"promType"` PromType string `json:"promType"`
CacheEntryTTL int64 `json:"cacheEntryTTL"`
// initialized at startup // initialized at startup
Desc *prometheus.Desc Desc *prometheus.Desc
@ -131,8 +155,20 @@ type LuaMetricsFile struct {
Metrics []*LuaMetric `json:"metrics"` Metrics []*LuaMetric `json:"metrics"`
} }
type UpnpCacheEntry struct {
Timestamp int64
Result *upnp.Result
}
type LuaCacheEntry struct {
Timestamp int64
Result *map[string]interface{}
}
var metrics []*Metric var metrics []*Metric
var luaMetrics []*LuaMetric var luaMetrics []*LuaMetric
var upnpCache map[string]*UpnpCacheEntry
var luaCache map[string]*LuaCacheEntry
type FritzboxCollector struct { type FritzboxCollector struct {
Url string Url string
@ -255,39 +291,50 @@ func (fc *FritzboxCollector) ReportMetric(ch chan<- prometheus.Metric, m *Metric
labels...) labels...)
} }
func (fc *FritzboxCollector) GetActionResult(result_map map[string]upnp.Result, serviceType string, actionName string, actionArg *upnp.ActionArgument) (upnp.Result, error) { func (fc *FritzboxCollector) GetActionResult(metric *Metric, actionName string, actionArg *upnp.ActionArgument) (upnp.Result, error) {
m_key := serviceType + "|" + actionName key := metric.Service + "|" + actionName
// for calls with argument also add arguement name and value to key // for calls with argument also add arguement name and value to key
if actionArg != nil { if actionArg != nil {
key += "|" + actionArg.Name + "|" + fmt.Sprintf("%v", actionArg.Value)
}
now := time.Now().Unix()
m_key += "|" + actionArg.Name + "|" + fmt.Sprintf("%v", actionArg.Value) cacheEntry := upnpCache[key]
if cacheEntry == nil {
cacheEntry = &UpnpCacheEntry{}
upnpCache[key] = cacheEntry
} else if now-cacheEntry.Timestamp > metric.CacheEntryTTL {
cacheEntry.Result = nil
} }
last_result := result_map[m_key] if cacheEntry.Result == nil {
if last_result == nil { service, ok := fc.Root.Services[metric.Service]
service, ok := fc.Root.Services[serviceType]
if !ok { if !ok {
return nil, errors.New(fmt.Sprintf("service %s not found", serviceType)) return nil, fmt.Errorf("service %s not found", metric.Service)
} }
action, ok := service.Actions[actionName] action, ok := service.Actions[actionName]
if !ok { if !ok {
return nil, errors.New(fmt.Sprintf("action %s not found in service %s", actionName, serviceType)) return nil, fmt.Errorf("action %s not found in service %s", actionName, metric.Service)
} }
var err error data, err := action.Call(actionArg)
last_result, err = action.Call(actionArg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
result_map[m_key] = last_result cacheEntry.Timestamp = now
cacheEntry.Result = &data
collectUpnpResultsCached.Inc()
} else {
collectUpnpResultsLoaded.Inc()
} }
return last_result, nil return *cacheEntry.Result, nil
} }
func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) { func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) {
@ -300,9 +347,6 @@ func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) {
return return
} }
// create a map for caching results
var result_map = make(map[string]upnp.Result)
for _, m := range metrics { for _, m := range metrics {
var actArg *upnp.ActionArgument var actArg *upnp.ActionArgument
if m.ActionArgument != nil { if m.ActionArgument != nil {
@ -311,7 +355,7 @@ func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) {
value = aa.Value value = aa.Value
if aa.ProviderAction != "" { if aa.ProviderAction != "" {
provRes, err := fc.GetActionResult(result_map, m.Service, aa.ProviderAction, nil) provRes, err := fc.GetActionResult(m, aa.ProviderAction, nil)
if err != nil { if err != nil {
fmt.Printf("Error getting provider action %s result for %s.%s: %s\n", aa.ProviderAction, m.Service, m.Action, err.Error()) fmt.Printf("Error getting provider action %s result for %s.%s: %s\n", aa.ProviderAction, m.Service, m.Action, err.Error())
@ -339,7 +383,7 @@ func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) {
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
actArg = &upnp.ActionArgument{Name: aa.Name, Value: i} actArg = &upnp.ActionArgument{Name: aa.Name, Value: i}
result, err := fc.GetActionResult(result_map, m.Service, m.Action, actArg) result, err := fc.GetActionResult(m, m.Action, actArg)
if err != nil { if err != nil {
fmt.Println(err.Error()) fmt.Println(err.Error())
@ -356,7 +400,7 @@ func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) {
} }
} }
result, err := fc.GetActionResult(result_map, m.Service, m.Action, actArg) result, err := fc.GetActionResult(m, m.Action, actArg)
if err != nil { if err != nil {
fmt.Println(err.Error()) fmt.Println(err.Error())
@ -375,13 +419,20 @@ func (fc *FritzboxCollector) Collect(ch chan<- prometheus.Metric) {
func (fc *FritzboxCollector) collectLua(ch chan<- prometheus.Metric) { func (fc *FritzboxCollector) collectLua(ch chan<- prometheus.Metric) {
// create a map for caching results // create a map for caching results
var result_map = make(map[string]map[string]interface{}) now := time.Now().Unix()
for _, lm := range luaMetrics { for _, lm := range luaMetrics {
key := lm.Path + "_" + lm.Params key := lm.Path + "_" + lm.Params
last_result := result_map[key] cacheEntry := luaCache[key]
if last_result == nil { if cacheEntry == nil {
cacheEntry = &LuaCacheEntry{}
luaCache[key] = cacheEntry
} else if now-cacheEntry.Timestamp > lm.CacheEntryTTL {
cacheEntry.Result = nil
}
if cacheEntry.Result == nil {
pageData, err := fc.LuaSession.LoadData(lm.LuaPage) pageData, err := fc.LuaSession.LoadData(lm.LuaPage)
if err != nil { if err != nil {
@ -390,17 +441,22 @@ func (fc *FritzboxCollector) collectLua(ch chan<- prometheus.Metric) {
continue continue
} }
last_result, err = lua.ParseJSON(pageData) var data map[string]interface{}
data, err = lua.ParseJSON(pageData)
if err != nil { if err != nil {
fmt.Printf("Error parsing JSON from %s for %s.%s: %s\n", lm.Path, lm.ResultPath, lm.ResultKey, err.Error()) fmt.Printf("Error parsing JSON from %s for %s.%s: %s\n", lm.Path, lm.ResultPath, lm.ResultKey, err.Error())
lua_collect_errors.Inc() lua_collect_errors.Inc()
continue continue
} }
result_map[key] = last_result cacheEntry.Result = &data
cacheEntry.Timestamp = now
collectLuaResultsLoaded.Inc()
} else {
collectLuaResultsCached.Inc()
} }
metricVals, err := lua.GetMetrics(fc.LabelRenames, last_result, lm.LuaMetricDef) metricVals, err := lua.GetMetrics(fc.LabelRenames, *cacheEntry.Result, lm.LuaMetricDef)
if err != nil { if err != nil {
fmt.Printf("Error getting metric values for %s.%s: %s\n", lm.ResultPath, lm.ResultKey, err.Error()) fmt.Printf("Error getting metric values for %s.%s: %s\n", lm.ResultPath, lm.ResultKey, err.Error())
@ -602,6 +658,9 @@ func main() {
return return
} }
// create a map for caching results
upnpCache = make(map[string]*UpnpCacheEntry)
var luaSession *lua.LuaSession var luaSession *lua.LuaSession
var luaLabelRenames *[]lua.LabelRename var luaLabelRenames *[]lua.LabelRename
if !*flag_disable_lua { if !*flag_disable_lua {
@ -618,6 +677,9 @@ func main() {
return return
} }
// create a map for caching results
luaCache = make(map[string]*LuaCacheEntry)
// init label renames // init label renames
lblRen := make([]lua.LabelRename, 0) lblRen := make([]lua.LabelRename, 0)
for _, ren := range lmf.LabelRenames { for _, ren := range lmf.LabelRenames {
@ -657,6 +719,11 @@ func main() {
OkValue: lm.OkValue, OkValue: lm.OkValue,
Labels: pd.VarLabels, Labels: pd.VarLabels,
} }
// init TTL
if lm.CacheEntryTTL < minCacheTTL {
lm.CacheEntryTTL = minCacheTTL
}
} }
luaSession = &lua.LuaSession{ luaSession = &lua.LuaSession{
@ -678,6 +745,11 @@ func main() {
m.Desc = prometheus.NewDesc(pd.FqName, pd.Help, labels, nil) m.Desc = prometheus.NewDesc(pd.FqName, pd.Help, labels, nil)
m.MetricType = getValueType(m.PromType) m.MetricType = getValueType(m.PromType)
// init TTL
if m.CacheEntryTTL < minCacheTTL {
m.CacheEntryTTL = minCacheTTL
}
} }
collector := &FritzboxCollector{ collector := &FritzboxCollector{
@ -715,8 +787,13 @@ func main() {
prometheus.MustRegister(collector) prometheus.MustRegister(collector)
prometheus.MustRegister(collect_errors) prometheus.MustRegister(collect_errors)
prometheus.MustRegister(collectUpnpResultsCached)
prometheus.MustRegister(collectUpnpResultsLoaded)
if luaSession != nil { if luaSession != nil {
prometheus.MustRegister(lua_collect_errors) prometheus.MustRegister(lua_collect_errors)
prometheus.MustRegister(collectLuaResultsCached)
prometheus.MustRegister(collectLuaResultsLoaded)
} }
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.Handler())

27
metrics-lua.json
View File

@ -42,7 +42,8 @@
"gateway", "name" "gateway", "name"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -57,7 +58,8 @@
"gateway", "name" "gateway", "name"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -71,7 +73,8 @@
"gateway" "gateway"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -85,7 +88,8 @@
"gateway" "gateway"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -102,7 +106,8 @@
"ram_type" : "Fixed" "ram_type" : "Fixed"
} }
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -119,7 +124,8 @@
"ram_type" : "Dynamic" "ram_type" : "Dynamic"
} }
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -136,7 +142,8 @@
"ram_type" : "Free" "ram_type" : "Free"
} }
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -150,7 +157,8 @@
"gateway", "deviceType", "deviceName" "gateway", "deviceType", "deviceName"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
}, },
{ {
"path": "data.lua", "path": "data.lua",
@ -164,7 +172,8 @@
"gateway", "deviceType", "deviceName" "gateway", "deviceType", "deviceName"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 300
} }
] ]
} }

18
metrics.json
View File

@ -88,7 +88,8 @@
"gateway" "gateway"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 60
}, },
{ {
"service": "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1", "service": "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1",
@ -101,7 +102,8 @@
"gateway" "gateway"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 60
}, },
{ {
"service": "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1", "service": "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1",
@ -115,7 +117,8 @@
"gateway" "gateway"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 60
}, },
{ {
"service": "urn:schemas-upnp-org:service:WANIPConnection:1", "service": "urn:schemas-upnp-org:service:WANIPConnection:1",
@ -129,7 +132,8 @@
"gateway" "gateway"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 60
}, },
{ {
"service": "urn:schemas-upnp-org:service:WANIPConnection:1", "service": "urn:schemas-upnp-org:service:WANIPConnection:1",
@ -314,7 +318,8 @@
"HostName" "HostName"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 60
}, },
{ {
"service": "urn:dslforum-org:service:X_AVM-DE_Dect:1", "service": "urn:dslforum-org:service:X_AVM-DE_Dect:1",
@ -336,6 +341,7 @@
"Model" "Model"
] ]
}, },
"promType": "GaugeValue" "promType": "GaugeValue",
"cacheEntryTTL": 120
} }
] ]

Write Preview
Loading…
Cancel
Save