From 1d6549a9e26380ed9ad691f6c1cd2896b75d454f Mon Sep 17 00:00:00 2001 From: Eldwan Brianne Date: Wed, 4 Nov 2020 17:10:50 +0100 Subject: [PATCH] Added https support, added more metrics, cleaned up dns_queries types --- config/configuration.go | 8 +- go.mod | 1 + go.sum | 2 + internal/adguard/client.go | 218 ++++++++++++++++++++++-------------- internal/adguard/model.go | 76 +++++++++---- internal/metrics/metrics.go | 22 ++++ main.go | 19 +--- 7 files changed, 218 insertions(+), 128 deletions(-) diff --git a/config/configuration.go b/config/configuration.go index 451e1f3..2658bbc 100644 --- a/config/configuration.go +++ b/config/configuration.go @@ -17,22 +17,22 @@ import ( type Config struct { AdguardProtocol string `config:"adguard_protocol"` AdguardHostname string `config:"adguard_hostname"` - AdguardPort uint16 `config:"adguard_port"` AdguardUsername string `config:"adguard_username"` AdguardPassword string `config:"adguard_password"` - Port string `config:"port"` + ServerPort string `config:"server_port"` Interval time.Duration `config:"interval"` + LogLimit string `config:"log_limit"` } func getDefaultConfig() *Config { return &Config{ AdguardProtocol: "http", AdguardHostname: "127.0.0.1", - AdguardPort: 80, AdguardUsername: "", AdguardPassword: "", - Port: "9617", + ServerPort: "9617", Interval: 10 * time.Second, + LogLimit: "1000", } } diff --git a/go.mod b/go.mod index 158e578..d2fddf8 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/heetch/confita v0.9.2 + github.com/mitchellh/mapstructure v1.1.2 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 ) diff --git a/go.sum b/go.sum index 3d60f49..ea21dad 100644 --- a/go.sum +++ b/go.sum @@ -105,6 +105,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -187,6 +188,7 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190508220229-2d0786266e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/adguard/client.go b/internal/adguard/client.go index 587976d..6770e68 100644 --- a/internal/adguard/client.go +++ b/internal/adguard/client.go @@ -1,47 +1,66 @@ package adguard import ( + "crypto/tls" "encoding/json" "fmt" "io/ioutil" "log" "net/http" "os" + "strconv" "time" "github.com/ebrianne/adguard-exporter/internal/metrics" + "github.com/mitchellh/mapstructure" ) var ( + port uint16 + statusURLPattern = "%s://%s:%d/control/status" statsURLPattern = "%s://%s:%d/control/stats" - logstatsURLPattern = "%s://%s:%d/control/querylog" + logstatsURLPattern = "%s://%s:%d/control/querylog?limit=%s&response_status=\"all\"" m map[string]int ) // Client struct is a AdGuard client to request an instance of a AdGuard ad blocker. type Client struct { - httpClient http.Client - interval time.Duration - protocol string - hostname string - port uint16 - b64password string + httpClient http.Client + interval time.Duration + logLimit string + protocol string + hostname string + port uint16 + username string + password string } // NewClient method initializes a new AdGuard client. -func NewClient(protocol, hostname string, port uint16, b64password string, interval time.Duration) *Client { - if protocol != "http" { - log.Printf("protocol %s is invalid. Must be http.", protocol) +func NewClient(protocol, hostname string, username, password string, interval time.Duration, logLimit string) *Client { + if protocol != "http" && protocol != "https" { + log.Printf("protocol %s is invalid. Must be http or https.", protocol) os.Exit(1) } + port = 80 + if protocol == "https" { + port = 443 + } + return &Client{ - protocol: protocol, - hostname: hostname, - port: port, - b64password: b64password, - interval: interval, - httpClient: http.Client{}, + protocol: protocol, + hostname: hostname, + port: port, + username: username, + password: password, + interval: interval, + logLimit: logLimit, + httpClient: http.Client{ + Transport: &http.Transport{TLSClientConfig: GetTlsConfig()}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, } } @@ -50,20 +69,30 @@ func NewClient(protocol, hostname string, port uint16, b64password string, inter func (c *Client) Scrape() { for range time.Tick(c.interval) { - //Get the general stats - stats := c.getStatistics() - c.setMetrics(stats) + allstats := c.getStatistics() + //Set the metrics + c.setMetrics(allstats.status, allstats.stats, allstats.logStats) - //Get the log stats - logdata := c.getLogStatistics() - c.setLogMetrics(logdata) - - log.Printf("New tick of statistics: %s", stats.ToString()) + log.Printf("New tick of statistics: %s", allstats.stats.ToString()) } } -// Function to set the general stats -func (c *Client) setMetrics(stats *Stats) { +// Function to set the prometheus metrics +func (c *Client) setMetrics(status *Status, stats *Stats, logstats *LogStats) { + //Status + var isRunning int = 0 + if status.Running == true { + isRunning = 1 + } + metrics.Running.WithLabelValues(c.hostname).Set(float64(isRunning)) + + var isProtected int = 0 + if status.ProtectionEnabled == true { + isProtected = 1 + } + metrics.ProtectionEnabled.WithLabelValues(c.hostname).Set(float64(isProtected)) + + //Stats metrics.AvgProcessingTime.WithLabelValues(c.hostname).Set(float64(stats.AvgProcessingTime)) metrics.DnsQueries.WithLabelValues(c.hostname).Set(float64(stats.DnsQueries)) metrics.BlockedFiltering.WithLabelValues(c.hostname).Set(float64(stats.BlockedFiltering)) @@ -88,51 +117,28 @@ func (c *Client) setMetrics(stats *Stats) { metrics.TopClients.WithLabelValues(c.hostname, source).Set(float64(value)) } } -} -// Function to get the general stats -func (c *Client) getStatistics() *Stats { - log.Printf("Getting general statistics") - - var stats Stats - statsURL := fmt.Sprintf(statsURLPattern, c.protocol, c.hostname, c.port) - - req, err := http.NewRequest("GET", statsURL, nil) - if err != nil { - log.Fatal("An error has occurred when creating HTTP statistics request ", err) - } - - if c.isUsingPassword() { - c.authenticateRequest(req) - } - - resp, err := c.httpClient.Do(req) - if err != nil { - log.Printf("An error has occurred during login to Adguard: %v", err) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println("Unable to read Adguard statistics HTTP response", err) - } - - err = json.Unmarshal(body, &stats) - if err != nil { - log.Println("Unable to unmarshal Adguard statistics to statistics struct model", err) - } - - return &stats -} - -// Function to get the log metrics -func (c *Client) setLogMetrics(logdata *LogData) { + //LogQuery m = make(map[string]int) - for i := range logdata.Data { - logstats := logdata.Data[i] - if logstats.DNS != nil { - for j := range logstats.DNS { - dnsType := logstats.DNS[j].Type - m[dnsType] += 1 + logdata := logstats.Data + for i := range logdata { + dnsanswer := logdata[i].Answer + if dnsanswer != nil && len(dnsanswer) > 0 { + for j := range dnsanswer { + var dnsType string + //Check the type of dnsanswer[j].Value, if string leave it be, otherwise get back the object to get the correct DNS type + switch v := dnsanswer[j].Value.(type) { + case string: + dnsType = dnsanswer[j].Type + m[dnsType] += 1 + case map[string]interface{}: + var dns65 Type65 + mapstructure.Decode(v, &dns65) + dnsType = "TYPE" + strconv.Itoa(dns65.Hdr.Rrtype) + m[dnsType] += 1 + default: + continue + } } } } @@ -141,49 +147,87 @@ func (c *Client) setLogMetrics(logdata *LogData) { metrics.QueryTypes.WithLabelValues(c.hostname, key).Set(float64(value)) } + //clear the map for k := range m { delete(m, k) } } -// Function to get the log stats -func (c *Client) getLogStatistics() *LogData { - log.Printf("Getting log statistics") +// Function to get the general stats +func (c *Client) getStatistics() *AllStats { - var logdata LogData - logstatsURL := fmt.Sprintf(logstatsURLPattern, c.protocol, c.hostname, c.port) - - req, err := http.NewRequest("GET", logstatsURL, nil) + var status Status + statusURL := fmt.Sprintf(statusURLPattern, c.protocol, c.hostname, c.port) + body := c.MakeRequest(statusURL) + err := json.Unmarshal(body, &status) if err != nil { - log.Fatal("An error has occurred when creating HTTP statistics request ", err) + log.Println("Unable to unmarshal Adguard log statistics to log statistics struct model", err) } + var stats Stats + statsURL := fmt.Sprintf(statsURLPattern, c.protocol, c.hostname, c.port) + body = c.MakeRequest(statsURL) + err = json.Unmarshal(body, &stats) + if err != nil { + log.Println("Unable to unmarshal Adguard statistics to statistics struct model", err) + } + + var logstats LogStats + logstatsURL := fmt.Sprintf(logstatsURLPattern, c.protocol, c.hostname, c.port, c.logLimit) + body = c.MakeRequest(logstatsURL) + err = json.Unmarshal(body, &logstats) + if err != nil { + log.Println("Unable to unmarshal Adguard log statistics to log statistics struct model", err) + } + + var allstats AllStats + allstats.status = &status + allstats.stats = &stats + allstats.logStats = &logstats + + return &allstats +} + +func (c *Client) MakeRequest(url string) []byte { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Fatal("An error has occurred when creating HTTP statistics request", err) + } + + req.Host = "adguard.home-lab.io" + req.Header.Add("User-Agent", "Mozilla/5.0") + if c.isUsingPassword() { c.authenticateRequest(req) } resp, err := c.httpClient.Do(req) if err != nil { - log.Printf("An error has occurred during login to Adguard: %v", err) + log.Fatal("An error has occurred during login to Adguard", err) + } + + if resp.StatusCode != 200 { + log.Fatal("An error occured in the request, Status Code ", resp.StatusCode) } body, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Println("Unable to read Adguard statistics HTTP response", err) + log.Fatal("Unable to read Adguard statistics HTTP response", err) } - err = json.Unmarshal(body, &logdata) - if err != nil { - log.Println("Unable to unmarshal Adguard log statistics to log statistics struct model", err) - } - - return &logdata + return body } func (c *Client) isUsingPassword() bool { - return len(c.b64password) > 0 + return len(c.password) > 0 } func (c *Client) authenticateRequest(req *http.Request) { - req.Header.Add("Authorization", "Basic "+c.b64password) + req.SetBasicAuth(c.username, c.password) +} + +func GetTlsConfig() *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, + } } diff --git a/internal/adguard/model.go b/internal/adguard/model.go index 97c34dc..3c2f3c5 100644 --- a/internal/adguard/model.go +++ b/internal/adguard/model.go @@ -2,6 +2,25 @@ package adguard import "fmt" +// SllStats struct containing all Adguard statistics structs +type AllStats struct { + status *Status + stats *Stats + logStats *LogStats +} + +// Status struct is the Adguard statistics JSON API corresponding model. +type Status struct { + Dhcp bool `json:"dhcp_available"` + DNSAddresses []string `json:"dns_addresses"` + DNSPort int `json:"dns_port"` + HttpPort int `json:"http_port"` + Language string `json:"language"` + ProtectionEnabled bool `json:"protection_enabled"` + Running bool `json:"running"` + Version string `json:"version"` +} + // Stats struct is the Adguard statistics JSON API corresponding model. type Stats struct { AvgProcessingTime float64 `json:"avg_processing_time"` @@ -15,38 +34,51 @@ type Stats struct { TopClients []map[string]int `json:"top_clients"` } -// DNSAnswer struct from LogStats -type DNSAnswer struct { - Ttl float64 `json:"ttl"` - Type string `json:"type"` - Value string `json:"value"` +type DNSHeader struct { + Name string `json:"Name"` + Rrtype int `json:"Rrtype"` + Class int `json:"Class"` + TTL int `json:"Ttl"` + Rdlength int `json:"Rdlength"` } -// DNSQuery struct from LogStats +type Type65 struct { + Hdr DNSHeader `json:"Hdr"` + RData string `json:"Rdata"` +} + +// DNSAnswer struct from LogData +type DNSAnswer struct { + TTL float64 `json:"ttl"` + Type string `json:"type"` + Value interface{} `json:"value"` // DNSAnswer struct can change sometimes... value:string or value: { "Hdr": { "Name":string, "Rrtype":int, "Class":int, "Ttl":int, "Rdlength":int }, "RData":string } +} + +// DNSQuery struct from LogData type DNSQuery struct { Class string `json:"class"` Host string `json:"host"` Type string `json:"type"` } -// LogStats struct, sub struct of LogData to collect the dns stats from the log -type LogStats struct { - DNS []DNSAnswer `json:"answer"` - DNSSec bool `json:"answer_dnssec"` - Client string `json:"client"` - ClientProto string `json:"client_proto"` - Elapsed string `json:"elapsedMs"` - Question DNSQuery `json:"question"` - Reason string `json:"reason"` - Status string `json:"status"` - Time string `json:"time"` - Upstream string `json:"upstream"` +// LogData struct, sub struct of LogStats to collect the dns stats from the log +type LogData struct { + Answer []DNSAnswer `json:"answer"` + DNSSec bool `json:"answer_dnssec"` + Client string `json:"client"` + ClientProto string `json:"client_proto"` + Elapsed string `json:"elapsedMs"` + Question DNSQuery `json:"question"` + Reason string `json:"reason"` + Status string `json:"status"` + Time string `json:"time"` + Upstream string `json:"upstream"` } -// LogData struct for the Adguard log statistics JSON API corresponding model. -type LogData struct { - Data []LogStats `json:"data"` - Oldest string `json:"oldest"` +// LogStats struct for the Adguard log statistics JSON API corresponding model. +type LogStats struct { + Data []LogData `json:"data"` + Oldest string `json:"oldest"` } // ToString method returns a string of the current statistics struct. diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 1cbed02..7c5ea17 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -106,6 +106,26 @@ var ( }, []string{"hostname", "type"}, ) + + // Running - If Adguard is running + Running = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "running", + Namespace: "adguard", + Help: "This represent if Adguard is running", + }, + []string{"hostname"}, + ) + + // ProtectionEnable - If Adguard protection is enabled + ProtectionEnabled = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "protection_enabled", + Namespace: "adguard", + Help: "This represent if Adguard Protection is enabled", + }, + []string{"hostname"}, + ) ) // Init initializes all Prometheus metrics made available by AdGuard exporter. @@ -120,6 +140,8 @@ func Init() { initMetric("top_blocked_domains", TopBlocked) initMetric("top_clients", TopClients) initMetric("query_types", QueryTypes) + initMetric("running", Running) + initMetric("protection_enabled", ProtectionEnabled) } func initMetric(name string, metric *prometheus.GaugeVec) { diff --git a/main.go b/main.go index 3500d08..74b83c7 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/base64" "fmt" "os" "os/signal" @@ -27,24 +26,14 @@ func main() { metrics.Init() - initAdguardClient(conf.AdguardProtocol, conf.AdguardHostname, conf.AdguardPort, conf.AdguardUsername, conf.AdguardPassword, conf.Interval) - initHttpServer(conf.Port) + initAdguardClient(conf.AdguardProtocol, conf.AdguardHostname, conf.AdguardUsername, conf.AdguardPassword, conf.Interval, conf.LogLimit) + initHttpServer(conf.ServerPort) handleExitSignal() } -func basicAuth(username, password string) string { - auth := username + ":" + password - return base64.StdEncoding.EncodeToString([]byte(auth)) -} - -func initAdguardClient(protocol, hostname string, port uint16, username, password string, interval time.Duration) { - b64password := "" - if len(username) > 0 && len(password) > 0 { - b64password = basicAuth(username, password) - } - - client := adguard.NewClient(protocol, hostname, port, b64password, interval) +func initAdguardClient(protocol, hostname string, username, password string, interval time.Duration, logLimit string) { + client := adguard.NewClient(protocol, hostname, username, password, interval, logLimit) go client.Scrape() }