Added parsing of log statistics, added dns query types as prometheus metric

This commit is contained in:
Eldwan Brianne 2020-11-03 23:05:29 +01:00
parent e770c7fc00
commit 645a647497
5 changed files with 548 additions and 168 deletions

View File

@ -16,6 +16,12 @@
"name": "Grafana",
"version": "7.3.1"
},
{
"type": "panel",
"id": "grafana-piechart-panel",
"name": "Pie Chart",
"version": "1.5.0"
},
{
"type": "panel",
"id": "graph",
@ -65,7 +71,7 @@
"gnetId": 10176,
"graphTooltip": 0,
"id": null,
"iteration": 1604355788357,
"iteration": 1604439632531,
"links": [],
"panels": [
{
@ -197,7 +203,7 @@
},
"gridPos": {
"h": 3,
"w": 2,
"w": 3,
"x": 3,
"y": 1
},
@ -291,7 +297,7 @@
"gridPos": {
"h": 3,
"w": 3,
"x": 5,
"x": 6,
"y": 1
},
"id": 80,
@ -395,7 +401,7 @@
"gridPos": {
"h": 3,
"w": 3,
"x": 8,
"x": 9,
"y": 1
},
"id": 92,
@ -434,6 +440,171 @@
"transparent": true,
"type": "stat"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "80%",
"format": "short",
"gridPos": {
"h": 8,
"w": 4,
"x": 12,
"y": 1
},
"id": 95,
"interval": null,
"legend": {
"percentage": true,
"show": false,
"values": true
},
"legendType": "On graph",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "donut",
"pluginVersion": "7.3.1",
"strokeWidth": 1,
"targets": [
{
"expr": "adguard_query_types{hostname=~'$node'}",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ type }}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Replies by type",
"transparent": true,
"type": "grafana-piechart-panel",
"valueName": "avg"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "80%",
"format": "short",
"gridPos": {
"h": 8,
"w": 4,
"x": 16,
"y": 1
},
"id": 100,
"interval": null,
"legend": {
"percentage": true,
"show": false,
"values": true
},
"legendType": "On graph",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "donut",
"pluginVersion": "7.3.1",
"strokeWidth": 1,
"targets": [
{
"expr": "adguard_query_types{hostname=~'$node'}",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ type }}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Replies by type",
"transparent": true,
"type": "grafana-piechart-panel",
"valueName": "avg"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "80%",
"format": "short",
"gridPos": {
"h": 8,
"w": 4,
"x": 20,
"y": 1
},
"id": 99,
"interval": null,
"legend": {
"percentage": true,
"show": false,
"values": true
},
"legendType": "On graph",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "donut",
"pluginVersion": "7.3.1",
"strokeWidth": 1,
"targets": [
{
"expr": "adguard_top_blocked_domains{hostname=~'$node'}",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ domain }}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Top ads by domain",
"transparent": true,
"type": "grafana-piechart-panel",
"valueName": "avg"
},
{
"aliasColors": {
"pihole": "super-light-blue"
@ -452,7 +623,7 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"w": 12,
"x": 0,
"y": 4
},
@ -538,7 +709,13 @@
}
},
{
"columns": [],
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
@ -546,59 +723,140 @@
},
"overrides": []
},
"fontSize": "100%",
"fontSize": "80%",
"format": "short",
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 4
"h": 8,
"w": 4,
"x": 12,
"y": 9
},
"id": 74,
"id": 98,
"interval": null,
"legend": {
"percentage": true,
"show": false,
"values": true
},
"legendType": "On graph",
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"mappingType": 1,
"pattern": "Time",
"thresholds": [],
"type": "date",
"unit": "short"
}
],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "donut",
"pluginVersion": "7.3.1",
"strokeWidth": 1,
"targets": [
{
"expr": "adguard_top_queried_domains{hostname=~'$node'}",
"expr": "adguard_top_clients{hostname=~'$node'}",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ domain }}",
"legendFormat": "{{ client }}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Top queries",
"transform": "timeseries_to_rows",
"title": "Top clients by source",
"transparent": true,
"type": "table-old"
"type": "grafana-piechart-panel",
"valueName": "avg"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 9
},
"hiddenSeries": false,
"id": 97,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.3.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "adguard_query_types{hostname=~'$node'}",
"interval": "",
"legendFormat": "{{ type }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "DNS Query Types",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
@ -618,7 +876,7 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"w": 12,
"x": 0,
"y": 11
},
@ -715,10 +973,10 @@
},
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 11
"h": 8,
"w": 12,
"x": 12,
"y": 17
},
"id": 93,
"links": [],
@ -788,7 +1046,7 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"w": 12,
"x": 0,
"y": 18
},
@ -884,10 +1142,73 @@
},
"fontSize": "100%",
"gridPos": {
"h": 6,
"w": 8,
"x": 8,
"y": 18
"h": 8,
"w": 12,
"x": 0,
"y": 25
},
"id": 74,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"mappingType": 1,
"pattern": "Time",
"thresholds": [],
"type": "date",
"unit": "short"
}
],
"targets": [
{
"expr": "adguard_top_queried_domains{hostname=~'$node'}",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ domain }}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Top queries",
"transform": "timeseries_to_rows",
"transparent": true,
"type": "table-old"
},
{
"columns": [],
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "100%",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 25
},
"id": 90,
"links": [],
@ -1007,5 +1328,5 @@
"timezone": "",
"title": "Adguard Exporter",
"uid": "MIBVglomg",
"version": 10
"version": 16
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 243 KiB

After

Width:  |  Height:  |  Size: 283 KiB

View File

@ -1,169 +1,189 @@
package adguard
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"time"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"time"
"github.com/ebrianne/adguard-exporter/internal/metrics"
"github.com/ebrianne/adguard-exporter/internal/metrics"
)
var (
statsURLPattern = "%s://%s:%d/control/stats"
logstatsURLPattern = "%s://%s:%d/control/querylog"
statsURLPattern = "%s://%s:%d/control/stats"
logstatsURLPattern = "%s://%s:%d/control/querylog"
m map[string]int
)
// Client struct is a AdGuard client to request an instance of a AdGuard ad blocker.
type Client struct {
httpClient http.Client
interval time.Duration
protocol string
hostname string
port uint16
b64password string
httpClient http.Client
interval time.Duration
protocol string
hostname string
port uint16
b64password string
}
// NewClient method initializes a new AdGuard client.
func NewClient(protocol, hostname string, port uint16, b64password string, interval time.Duration) *Client {
if protocol != "http" {
log.Printf("protocol %s is invalid. Must be http.", protocol)
os.Exit(1)
}
if protocol != "http" {
log.Printf("protocol %s is invalid. Must be http.", protocol)
os.Exit(1)
}
return &Client{
protocol: protocol,
hostname: hostname,
port: port,
b64password: b64password,
interval: interval,
httpClient: http.Client{},
}
return &Client{
protocol: protocol,
hostname: hostname,
port: port,
b64password: b64password,
interval: interval,
httpClient: http.Client{},
}
}
// Scrape method authenticates and retrieves statistics from AdGuard JSON API
// and then pass them as Prometheus metrics.
func (c *Client) Scrape() {
for range time.Tick(c.interval) {
for range time.Tick(c.interval) {
//Get the general stats
stats := c.getStatistics()
c.setMetrics(stats)
//Get the general stats
stats := c.getStatistics()
c.setMetrics(stats)
//Get the log stats
logstats := c.getLogStatistics()
c.setLogMetrics(logstats)
//Get the log stats
logdata := c.getLogStatistics()
c.setLogMetrics(logdata)
log.Printf("New tick of statistics: %s", stats.ToString())
}
log.Printf("New tick of statistics: %s", stats.ToString())
}
}
// Function to set the general stats
func (c *Client) setMetrics(stats *Stats) {
metrics.AvgProcessingTime.WithLabelValues(c.hostname).Set(float64(stats.AvgProcessingTime))
metrics.DnsQueries.WithLabelValues(c.hostname).Set(float64(stats.DnsQueries))
metrics.BlockedFiltering.WithLabelValues(c.hostname).Set(float64(stats.BlockedFiltering))
metrics.ParentalFiltering.WithLabelValues(c.hostname).Set(float64(stats.ParentalFiltering))
metrics.SafeBrowsingFiltering.WithLabelValues(c.hostname).Set(float64(stats.SafeBrowsingFiltering))
metrics.SafeSearchFiltering.WithLabelValues(c.hostname).Set(float64(stats.SafeSearchFiltering))
metrics.AvgProcessingTime.WithLabelValues(c.hostname).Set(float64(stats.AvgProcessingTime))
metrics.DnsQueries.WithLabelValues(c.hostname).Set(float64(stats.DnsQueries))
metrics.BlockedFiltering.WithLabelValues(c.hostname).Set(float64(stats.BlockedFiltering))
metrics.ParentalFiltering.WithLabelValues(c.hostname).Set(float64(stats.ParentalFiltering))
metrics.SafeBrowsingFiltering.WithLabelValues(c.hostname).Set(float64(stats.SafeBrowsingFiltering))
metrics.SafeSearchFiltering.WithLabelValues(c.hostname).Set(float64(stats.SafeSearchFiltering))
for l := range stats.TopQueries {
for domain, value := range stats.TopQueries[l] {
metrics.TopQueries.WithLabelValues(c.hostname, domain).Set(float64(value))
}
}
for l := range stats.TopQueries {
for domain, value := range stats.TopQueries[l] {
metrics.TopQueries.WithLabelValues(c.hostname, domain).Set(float64(value))
}
}
for l := range stats.TopBlocked {
for domain, value := range stats.TopBlocked[l] {
metrics.TopBlocked.WithLabelValues(c.hostname, domain).Set(float64(value))
}
}
for l := range stats.TopBlocked {
for domain, value := range stats.TopBlocked[l] {
metrics.TopBlocked.WithLabelValues(c.hostname, domain).Set(float64(value))
}
}
for l := range stats.TopClients {
for source, value := range stats.TopClients[l] {
metrics.TopClients.WithLabelValues(c.hostname, source).Set(float64(value))
}
}
for l := range stats.TopClients {
for source, value := range stats.TopClients[l] {
metrics.TopClients.WithLabelValues(c.hostname, source).Set(float64(value))
}
}
}
// Function to get the general stats
func (c *Client) getStatistics() *Stats {
var stats Stats
log.Printf("Getting general statistics")
statsURL := fmt.Sprintf(statsURLPattern, c.protocol, c.hostname, c.port)
var stats Stats
statsURL := fmt.Sprintf(statsURLPattern, c.protocol, c.hostname, c.port)
req, err := http.NewRequest("GET", statsURL, nil)
if err != nil {
log.Fatal("An error has occurred when creating HTTP statistics request ", err)
}
req, err := http.NewRequest("GET", statsURL, nil)
if err != nil {
log.Fatal("An error has occurred when creating HTTP statistics request ", err)
}
if c.isUsingPassword() {
c.authenticateRequest(req)
}
if c.isUsingPassword() {
c.authenticateRequest(req)
}
resp, err := c.httpClient.Do(req)
if err != nil {
log.Printf("An error has occurred during login to Adguard: %v", err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
log.Printf("An error has occurred during login to Adguard: %v", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("Unable to read Adguard statistics HTTP response", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("Unable to read Adguard statistics HTTP response", err)
}
err = json.Unmarshal(body, &stats)
if err != nil {
log.Println("Unable to unmarshal Adguard statistics to statistics struct model", err)
}
err = json.Unmarshal(body, &stats)
if err != nil {
log.Println("Unable to unmarshal Adguard statistics to statistics struct model", err)
}
return &stats
}
func (c *Client) isUsingPassword() bool {
return len(c.b64password) > 0
}
func (c *Client) authenticateRequest(req *http.Request) {
req.Header.Add("Authorization", "Basic "+c.b64password)
return &stats
}
// Function to get the log metrics
func (c *Client) setLogMetrics(logstats *LogStats) {
func (c *Client) setLogMetrics(logdata *LogData) {
m = make(map[string]int)
for i := range logdata.Data {
logstats := logdata.Data[i]
if logstats.DNS != nil {
for j := range logstats.DNS {
dns_type := logstats.DNS[j].Type
m[dns_type] += 1
}
}
}
for key, value := range m {
metrics.QueryTypes.WithLabelValues(c.hostname, key).Set(float64(value))
}
for k := range m {
delete(m, k)
}
}
// Function to get the log stats
func (c *Client) getLogStatistics() *LogStats {
var logstats LogStats
func (c *Client) getLogStatistics() *LogData {
log.Printf("Getting log statistics")
logstatsURL := fmt.Sprintf(logstatsURLPattern, c.protocol, c.hostname, c.port)
var logdata LogData
logstatsURL := fmt.Sprintf(logstatsURLPattern, c.protocol, c.hostname, c.port)
req, err := http.NewRequest("GET", statsURL, nil)
if err != nil {
log.Fatal("An error has occurred when creating HTTP statistics request ", err)
}
req, err := http.NewRequest("GET", logstatsURL, nil)
if err != nil {
log.Fatal("An error has occurred when creating HTTP statistics request ", err)
}
if c.isUsingPassword() {
c.authenticateRequest(req)
}
if c.isUsingPassword() {
c.authenticateRequest(req)
}
resp, err := c.httpClient.Do(req)
if err != nil {
log.Printf("An error has occurred during login to Adguard: %v", err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
log.Printf("An error has occurred during login to Adguard: %v", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("Unable to read Adguard statistics HTTP response", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("Unable to read Adguard statistics HTTP response", err)
}
// err = json.Unmarshal(body, &logstats)
// if err != nil {
// log.Println("Unable to unmarshal Adguard log statistics to log statistics struct model", err)
// }
err = json.Unmarshal(body, &logdata)
if err != nil {
log.Println("Unable to unmarshal Adguard log statistics to log statistics struct model", err)
}
return &logstats
return &logdata
}
func (c *Client) isUsingPassword() bool {
return len(c.b64password) > 0
}
func (c *Client) authenticateRequest(req *http.Request) {
req.Header.Add("Authorization", "Basic "+c.b64password)
}

View File

@ -15,9 +15,37 @@ type Stats struct {
TopClients []map[string]int `json:"top_clients"`
}
// DNS answer struct in the log
type DNSAnswer struct {
Ttl float64 `json:"ttl"`
Type string `json:"type"`
Value string `json:"value"`
}
// DNS query struct in the log
type DNSQuery struct {
Class string `json:"class"`
Host string `json:"host"`
Type string `json:"type"`
}
// LogStats struct for the Adguard log statistics JSON API corresponding model.
type LogStats struct {
DNS []DNSAnswer `json:"answer"`
DNSSec bool `json:"answer_dnssec"`
Client string `json:"client"`
Client_proto string `json:"client_proto"`
Elapsed string `json:"elapsedMs"`
Question DNSQuery `json:"question"`
Reason string `json:"reason"`
Status string `json:"status"`
Time string `json:"time"`
Upstream string `json:"upstream"`
}
type LogData struct {
Data []LogStats `json:"data"`
Oldest string `json:"oldest"`
}
// ToString method returns a string of the current statistics struct.

View File

@ -96,6 +96,16 @@ var (
},
[]string{"hostname", "client"},
)
// Answer
QueryTypes = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "query_types",
Namespace: "adguard",
Help: "This represent the DNS query types",
},
[]string{"hostname", "type"},
)
)
// Init initializes all Prometheus metrics made available by AdGuard exporter.
@ -109,6 +119,7 @@ func Init() {
initMetric("top_queried_domains", TopQueries)
initMetric("top_blocked_domains", TopBlocked)
initMetric("top_clients", TopClients)
initMetric("query_types", QueryTypes)
}
func initMetric(name string, metric *prometheus.GaugeVec) {