first test for lua calls

fritzbox_lua/lua_client.go

@@ -0,0 +1,318 @@
+// client for fritzbox lua API
+package lua_client
+
+// Copyright 2020 Andreas Krebs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
+)
+
+// session XML from login_sid.lua
+type SessionInfo struct {
+	SID       string `xml:"SID"`
+	Challenge string `xml:"Challenge"`
+	BlockTime int    `xml:"BlockTime"`
+	Rights    string `xml:"Rights"`
+}
+
+type LuaSession struct {
+	BaseUrl     string
+	Username    string
+	Password    string
+	SID         string
+	SessionInfo SessionInfo
+}
+
+type LuaPage struct {
+	Path   string
+	Params string
+}
+
+type LuaMetricValueDefinition struct {
+	Path   string
+	Key    string
+	Labels []string
+}
+
+type LuaMetricValue struct {
+	Name   string
+	Value  string
+	Labels map[string]string
+}
+
+type LabelRename struct {
+	Pattern regexp.Regexp
+	Name    string
+}
+
+func (lua *LuaSession) do_Login(response string) error {
+	url_params := ""
+	if response != "" {
+		url_params = fmt.Sprintf("?response=%s&user=%s", response, lua.Username)
+	}
+
+	resp, err := http.Get(fmt.Sprintf("%s/login_sid.lua%s", lua.BaseUrl, url_params))
+	if err != nil {
+		return errors.New(fmt.Sprintf("Error calling login_sid.lua: %s", err.Error()))
+	}
+
+	defer resp.Body.Close()
+	dec := xml.NewDecoder(resp.Body)
+
+	err = dec.Decode(&lua.SessionInfo)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Error decoding SessionInfo: %s", err.Error()))
+	}
+
+	return nil
+}
+
+func (lua *LuaSession) Login() error {
+
+	err := lua.do_Login("")
+	if err != nil {
+		return err
+	}
+
+	challenge := lua.SessionInfo.Challenge
+	if lua.SessionInfo.SID == "0000000000000000" && challenge != "" {
+		// no SID, but challenge so calc response
+		hash := utf16leMd5(fmt.Sprintf("%s-%s", challenge, lua.Password))
+		response := fmt.Sprintf("%s-%x", challenge, hash)
+		err := lua.do_Login(response)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	sid := lua.SessionInfo.SID
+	if sid == "0000000000000000" || sid == "" {
+		return errors.New("LUA login failed - no SID received - check username and password")
+	}
+
+	lua.SID = sid
+
+	return nil
+}
+
+func (lua *LuaSession) LoadData(page LuaPage) ([]byte, error) {
+	data_url := fmt.Sprintf("%s/%s", lua.BaseUrl, page.Path)
+
+	callDone := false
+	var resp *http.Response
+	var err error
+	for !callDone {
+		// perform login if no SID or previous call failed with (403)
+		if lua.SID == "" || resp != nil {
+			err = lua.Login()
+			callDone = true // consider call done, since we tried login
+
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// send by UI for data.lua: xhr=1&sid=xxxxxxx&lang=de&page=energy&xhrId=all&no_sidrenew=
+		// but SID and page seem to be enough
+		params := "sid=" + lua.SID
+		if page.Params != "" {
+			params += "&" + page.Params
+		}
+
+		resp, err = http.Post(data_url, "application/x-www-form-urlencoded", bytes.NewBuffer([]byte(params)))
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode == http.StatusOK {
+			callDone = true
+		} else if resp.StatusCode == http.StatusForbidden && !callDone {
+			// we assume SID is expired, so retry login
+		} else {
+			return nil, errors.New(fmt.Sprintf("data.lua failed: %s", resp.Status))
+		}
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+func getRenamedLabel(labelRenames *[]LabelRename, label string) string {
+	if labelRenames != nil {
+		for _, lblRen := range *labelRenames {
+			if lblRen.Pattern.MatchString(label) {
+				return lblRen.Name
+			}
+		}
+	}
+
+	return label
+}
+
+func GetMetrics(labelRenames *[]LabelRename, data map[string]interface{}, metricDef LuaMetricValueDefinition) ([]LuaMetricValue, error) {
+
+	var values []map[string]interface{}
+	var err error
+	if metricDef.Path != "" {
+		pathItems := strings.Split(metricDef.Path, ".")
+		values, err = _getValues(data, pathItems, "")
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		values = make([]map[string]interface{}, 1)
+		values[0] = data
+	}
+
+	name := metricDef.Path
+	if name != "" {
+		name += "."
+	}
+	name += metricDef.Key
+
+	metrics := make([]LuaMetricValue, 0)
+	for _, valMap := range values {
+		value, exists := valMap[metricDef.Key]
+		if exists {
+			lmv := LuaMetricValue{
+				Name:   name,
+				Value:  toString(value),
+				Labels: make(map[string]string),
+			}
+
+			for _, l := range metricDef.Labels {
+				lv, exists := valMap[l]
+				if exists {
+					lmv.Labels[l] = getRenamedLabel(labelRenames, toString(lv))
+				}
+			}
+
+			metrics = append(metrics, lmv)
+		}
+	}
+
+	return metrics, nil
+}
+
+// from https://stackoverflow.com/questions/33710672/golang-encode-string-utf16-little-endian-and-hash-with-md5
+func utf16leMd5(s string) []byte {
+	enc := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewEncoder()
+	hasher := md5.New()
+	t := transform.NewWriter(hasher, enc)
+	t.Write([]byte(s))
+	return hasher.Sum(nil)
+}
+
+func ParseJSON(jsonData []byte) (map[string]interface{}, error) {
+	var data map[string]interface{}
+
+	// Unmarshal or Decode the JSON to the interface.
+	json.Unmarshal(jsonData, &data)
+
+	return data, nil
+}
+
+// helper for retrieving values from parsed JSON
+func _getValues(data interface{}, pathItems []string, parentPath string) ([]map[string]interface{}, error) {
+
+	value := data
+	curPath := parentPath
+
+	for i, p := range pathItems {
+		switch vv := value.(type) {
+		case []interface{}:
+			if p == "*" {
+
+				values := make([]map[string]interface{}, 0, len(vv))
+				for index, u := range vv {
+					subvals, err := _getValues(u, pathItems[i+1:], fmt.Sprintf("%s.%d", curPath, index))
+					if err != nil {
+						return nil, err
+					}
+
+					values = append(values, subvals...)
+				}
+
+				return values, nil
+			} else {
+				index, err := strconv.Atoi(p)
+				if err != nil {
+					return nil, errors.New(fmt.Sprintf("item '%s' is an array, but path item '%s' is neither '*' nor a number", curPath, p))
+				}
+
+				if index < 0 {
+					// this is an index from the end of the values
+					index += len(vv)
+				}
+
+				if index >= 0 && index < len(vv) {
+					value = vv[index]
+				} else {
+					return nil, errors.New(fmt.Sprintf("index %d is invalid for array '%s' with length %d", index, curPath, len(vv)))
+				}
+			}
+
+		case map[string]interface{}:
+			var exits bool
+			value, exits = vv[p]
+			if !exits {
+				return nil, errors.New(fmt.Sprintf("key '%s' not existing in hash '%s'", p, curPath))
+			}
+
+		default:
+			return nil, errors.New(fmt.Sprintf("item '%s' is neither a hash or array", curPath))
+		}
+
+		if curPath == "" {
+			curPath = p
+		} else {
+			curPath += "." + p
+		}
+	}
+
+	vm, isType := value.(map[string]interface{})
+	if !isType {
+		return nil, errors.New(fmt.Sprintf("item '%s' is not a hash", curPath))
+	}
+
+	values := make([]map[string]interface{}, 1)
+	values[0] = vm
+	return values, nil
+}
+
+func toString(value interface{}) string {
+	// should we better check or simple convert everything ????
+	return fmt.Sprintf("%v", value)
+}

main.go

@@ -15,30 +15,34 @@ package main
 // limitations under the License.
 
 import (
+	"bytes"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
-	"net/url"
 	"net/http"
-	"sync"
+	"net/url"
-	"time"
+	"regexp"
-	"encoding/json"
-	"io/ioutil"
 	"sort"
-	"bytes"
-	"errors"
-	"strings"
 	"strconv"
+	"strings"
+	"sync"
+	"time"
 
 	"github.com/namsral/flag"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
+	lua "github.com/sberk42/fritzbox_exporter/fritzbox_lua"
 	upnp "github.com/sberk42/fritzbox_exporter/fritzbox_upnp"
 )
 
 const serviceLoadRetryTime = 1 * time.Minute
 
 var (
+	flag_luacall = flag.Bool("testLua", false, "test LUA") // TODO cleanup once completed
+
 	flag_test    = flag.Bool("test", false, "print all available metrics to stdout")
 	flag_collect = flag.Bool("collect", false, "print configured metrics to stdout and exit")
 	flag_jsonout = flag.String("json-out", "", "store metrics also to JSON file when running test")
@@ -47,6 +51,7 @@ var (
 	flag_metrics_file = flag.String("metrics-file", "metrics.json", "The JSON file with the metric definitions.")
 
 	flag_gateway_url      = flag.String("gateway-url", "http://fritz.box:49000", "The URL of the FRITZ!Box")
+	flag_gateway_luaurl   = flag.String("gateway-luaurl", "http://fritz.box", "The URL of the FRITZ!Box UI")
 	flag_gateway_username = flag.String("username", "", "The user for the FRITZ!Box UPnP service")
 	flag_gateway_password = flag.String("password", "", "The password for the FRITZ!Box UPnP service")
 )
@@ -58,7 +63,6 @@ var (
 	})
 )
 
-
 type JSON_PromDesc struct {
 	FqName    string   `json:"fqName"`
 	Help      string   `json:"help"`
@@ -87,7 +91,7 @@ type Metric struct {
 	MetricType prometheus.ValueType
 }
 
-var metrics []*Metric;
+var metrics []*Metric
 
 type FritzboxCollector struct {
 	Url      string
@@ -191,7 +195,7 @@ func (fc *FritzboxCollector) ReportMetric(ch chan<- prometheus.Metric, m *Metric
 			}
 
 			// convert hostname and MAC tolower to avoid problems with labels
-			if(l == "HostName" || l == "MACAddress" ) {
+			if l == "HostName" || l == "MACAddress" {
 				labels[i] = strings.ToLower(fmt.Sprintf("%v", lval))
 			} else {
 				labels[i] = fmt.Sprintf("%v", lval)
@@ -216,7 +220,7 @@ func (fc *FritzboxCollector) GetActionResult(result_map map[string]upnp.Result,
 		m_key += "|" + actionArg.Name + "|" + fmt.Sprintf("%v", actionArg.Value)
 	}
 
-	last_result	:= result_map[m_key];
+	last_result := result_map[m_key]
 	if last_result == nil {
 		service, ok := fc.Root.Services[serviceType]
 		if !ok {
@@ -229,7 +233,7 @@ func (fc *FritzboxCollector) GetActionResult(result_map map[string]upnp.Result,
 		}
 
 		var err error
-		last_result, err = action.Call(actionArg);
+		last_result, err = action.Call(actionArg)
 
 		if err != nil {
 			return nil, err
@@ -360,7 +364,7 @@ func test() {
 			// TODO also create JSON templates for input actionParams
 			for _, arg := range a.Arguments {
 				// create new json entry
-				if(newEntry) {
+				if newEntry {
 					json.WriteString(",\n")
 				} else {
 					newEntry = true
@@ -399,17 +403,79 @@ func test() {
 	}
 }
 
+func testLuaCall() {
+	var luaSession lua.LuaSession
+	luaSession.BaseUrl = *flag_gateway_luaurl
+	luaSession.Username = *flag_gateway_username
+	luaSession.Password = *flag_gateway_password
+
+	var jsonData []byte
+	var err error
+
+	fmt.Println("calling login")
+	page := lua.LuaPage{"data.lua", "page=energy"}
+	jsonData, err = luaSession.LoadData(page)
+
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	fmt.Println(fmt.Sprintf("JSON: %s", string(jsonData)))
+
+	data, err := lua.ParseJSON(jsonData)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	labelRenames := make([]lua.LabelRename, 0)
+	labelRenames = addLabelRename(labelRenames, "(?i)prozessor", "CPU")
+	labelRenames = addLabelRename(labelRenames, "(?i)system", "System")
+	labelRenames = addLabelRename(labelRenames, "(?i)FON", "Phone")
+	labelRenames = addLabelRename(labelRenames, "(?i)WLAN", "WLAN")
+	labelRenames = addLabelRename(labelRenames, "(?i)USB", "USB")
+
+	pidMetric := lua.LuaMetricValueDefinition{"", "pid", nil}
+	powerMetric := lua.LuaMetricValueDefinition{"data.drain.*", "actPerc", []string{"name"}}
+	//	fmt.Println(fmt.Sprintf("DATA: %v", data))
+	dumpMetric(&labelRenames, data, pidMetric)
+	dumpMetric(&labelRenames, data, powerMetric)
+}
+
+func dumpMetric(labelRenames *[]lua.LabelRename, data map[string]interface{}, metricDef lua.LuaMetricValueDefinition) {
+
+	metrics, err := lua.GetMetrics(labelRenames, data, metricDef)
+
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	fmt.Println(fmt.Sprintf("Metrics: %v", metrics))
+}
+
+func addLabelRename(labelRenames []lua.LabelRename, pattern string, name string) []lua.LabelRename {
+	regex, err := regexp.Compile(pattern)
+
+	if err == nil {
+		return append(labelRenames, lua.LabelRename{*regex, name})
+	}
+
+	return labelRenames
+}
+
 func getValueType(vt string) prometheus.ValueType {
 	switch vt {
 	case "CounterValue":
-		return prometheus.CounterValue;
+		return prometheus.CounterValue
 	case "GaugeValue":
-		return prometheus.GaugeValue;
+		return prometheus.GaugeValue
 	case "UntypedValue":
-		return prometheus.UntypedValue;
+		return prometheus.UntypedValue
 	}
 
-	return prometheus.UntypedValue;
+	return prometheus.UntypedValue
 }
 
 func main() {
@@ -426,6 +492,10 @@ func main() {
 		return
 	}
 
+	if *flag_luacall {
+		testLuaCall()
+		return
+	}
 	// read metrics
 	jsonData, err := ioutil.ReadFile(*flag_metrics_file)
 	if err != nil {