* querylog: move code to a separate package

+ config: "querylog_interval" setting
/control/querylog_config, /control/querylog_info
+ POST /control/querylog_clear
This commit is contained in:
Simon Zolin 2019-08-26 11:54:38 +03:00
parent 8f9ca4cba7
commit 8104c902ee
11 changed files with 457 additions and 150 deletions

View File

@ -42,6 +42,9 @@ Contents:
* API: Clear statistics data * API: Clear statistics data
* API: Set statistics parameters * API: Set statistics parameters
* API: Get statistics parameters * API: Get statistics parameters
* Query logs
* API: Set querylog parameters
* API: Get querylog parameters
## First startup ## First startup
@ -976,3 +979,37 @@ Response:
{ {
"interval": 1 | 7 | 30 | 90 "interval": 1 | 7 | 30 | 90
} }
## Query logs
### API: Set querylog parameters
Request:
POST /control/querylog_config
{
"enabled": true | false
"interval": 1 | 7 | 30 | 90
}
Response:
200 OK
### API: Get querylog parameters
Request:
GET /control/querylog_info
Response:
200 OK
{
"enabled": true | false
"interval": 1 | 7 | 30 | 90
}

View File

@ -11,6 +11,7 @@ import (
"time" "time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats" "github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/dnsproxy/upstream"
@ -40,7 +41,7 @@ const (
type Server struct { type Server struct {
dnsProxy *proxy.Proxy // DNS proxy instance dnsProxy *proxy.Proxy // DNS proxy instance
dnsFilter *dnsfilter.Dnsfilter // DNS filter instance dnsFilter *dnsfilter.Dnsfilter // DNS filter instance
queryLog *queryLog // Query log instance queryLog querylog.QueryLog // Query log instance
stats stats.Stats stats stats.Stats
AllowedClients map[string]bool // IP addresses of whitelist clients AllowedClients map[string]bool // IP addresses of whitelist clients
@ -54,16 +55,11 @@ type Server struct {
} }
// NewServer creates a new instance of the dnsforward.Server // NewServer creates a new instance of the dnsforward.Server
// baseDir is the base directory for query logs
// Note: this function must be called only once // Note: this function must be called only once
func NewServer(baseDir string, stats stats.Stats) *Server { func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server {
s := &Server{ s := &Server{}
queryLog: newQueryLog(baseDir),
}
s.stats = stats s.stats = stats
s.queryLog = queryLog
log.Printf("Start DNS server periodic jobs")
go s.queryLog.periodicQueryLogRotate()
return s return s
} }
@ -75,6 +71,7 @@ type FilteringConfig struct {
BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests
BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600) BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600)
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
Ratelimit int `yaml:"ratelimit"` // max number of requests per second from a given IP (0 to disable) Ratelimit int `yaml:"ratelimit"` // max number of requests per second from a given IP (0 to disable)
RatelimitWhitelist []string `yaml:"ratelimit_whitelist"` // a list of whitelisted client IP addresses RatelimitWhitelist []string `yaml:"ratelimit_whitelist"` // a list of whitelisted client IP addresses
RefuseAny bool `yaml:"refuse_any"` // if true, refuse ANY requests RefuseAny bool `yaml:"refuse_any"` // if true, refuse ANY requests
@ -303,8 +300,7 @@ func (s *Server) stopInternal() error {
s.dnsFilter = nil s.dnsFilter = nil
} }
// flush remainder to file return nil
return s.queryLog.flushLogBuffer(true)
} }
// IsRunning returns true if the DNS server is running // IsRunning returns true if the DNS server is running
@ -343,13 +339,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.RUnlock() s.RUnlock()
} }
// GetQueryLog returns a map with the current query log ready to be converted to a JSON
func (s *Server) GetQueryLog() []map[string]interface{} {
s.RLock()
defer s.RUnlock()
return s.queryLog.getQueryLog()
}
// Return TRUE if this client should be blocked // Return TRUE if this client should be blocked
func (s *Server) isBlockedIP(ip string) bool { func (s *Server) isBlockedIP(ip string) bool {
if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 { if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 {
@ -469,12 +458,12 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
} }
elapsed := time.Since(start) elapsed := time.Since(start)
if s.conf.QueryLogEnabled && shouldLog { if s.conf.QueryLogEnabled && shouldLog && s.queryLog != nil {
upstreamAddr := "" upstreamAddr := ""
if d.Upstream != nil { if d.Upstream != nil {
upstreamAddr = d.Upstream.Address() upstreamAddr = d.Upstream.Address()
} }
_ = s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr) s.queryLog.Add(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
} }
s.updateStats(d, elapsed, *res) s.updateStats(d, elapsed, *res)

View File

@ -10,7 +10,6 @@ import (
"encoding/pem" "encoding/pem"
"math/big" "math/big"
"net" "net"
"os"
"sync" "sync"
"testing" "testing"
"time" "time"
@ -18,18 +17,15 @@ import (
"github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/proxy"
"github.com/miekg/dns" "github.com/miekg/dns"
"github.com/stretchr/testify/assert"
) )
const ( const (
tlsServerName = "testdns.adguard.com" tlsServerName = "testdns.adguard.com"
dataDir = "testData"
testMessagesCount = 10 testMessagesCount = 10
) )
func TestServer(t *testing.T) { func TestServer(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -45,10 +41,6 @@ func TestServer(t *testing.T) {
} }
assertGoogleAResponse(t, reply) assertGoogleAResponse(t, reply)
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
// message over TCP // message over TCP
req = createGoogleATestMessage() req = createGoogleATestMessage()
addr = s.dnsProxy.Addr("tcp") addr = s.dnsProxy.Addr("tcp")
@ -59,10 +51,6 @@ func TestServer(t *testing.T) {
} }
assertGoogleAResponse(t, reply) assertGoogleAResponse(t, reply)
// check query log and stats again
log = s.GetQueryLog()
assert.Equal(t, 2, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -72,7 +60,6 @@ func TestServer(t *testing.T) {
func TestServerWithProtectionDisabled(t *testing.T) { func TestServerWithProtectionDisabled(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
s.conf.ProtectionEnabled = false s.conf.ProtectionEnabled = false
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -88,10 +75,6 @@ func TestServerWithProtectionDisabled(t *testing.T) {
} }
assertGoogleAResponse(t, reply) assertGoogleAResponse(t, reply)
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -102,7 +85,6 @@ func TestDotServer(t *testing.T) {
// Prepare the proxy server // Prepare the proxy server
_, certPem, keyPem := createServerTLSConfig(t) _, certPem, keyPem := createServerTLSConfig(t)
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
s.conf.TLSConfig = TLSConfig{ s.conf.TLSConfig = TLSConfig{
TLSListenAddr: &net.TCPAddr{Port: 0}, TLSListenAddr: &net.TCPAddr{Port: 0},
@ -143,7 +125,6 @@ func TestDotServer(t *testing.T) {
func TestServerRace(t *testing.T) { func TestServerRace(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -168,7 +149,6 @@ func TestServerRace(t *testing.T) {
func TestSafeSearch(t *testing.T) { func TestSafeSearch(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
s.conf.SafeSearchEnabled = true s.conf.SafeSearchEnabled = true
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -210,7 +190,6 @@ func TestSafeSearch(t *testing.T) {
func TestInvalidRequest(t *testing.T) { func TestInvalidRequest(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -229,11 +208,6 @@ func TestInvalidRequest(t *testing.T) {
t.Fatalf("got a response to an invalid query") t.Fatalf("got a response to an invalid query")
} }
// check query log and stats
// invalid requests aren't written to the query log
log := s.GetQueryLog()
assert.Equal(t, 0, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -242,7 +216,6 @@ func TestInvalidRequest(t *testing.T) {
func TestBlockedRequest(t *testing.T) { func TestBlockedRequest(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -267,10 +240,6 @@ func TestBlockedRequest(t *testing.T) {
t.Fatalf("Wrong response: %s", reply.String()) t.Fatalf("Wrong response: %s", reply.String())
} }
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -280,7 +249,6 @@ func TestBlockedRequest(t *testing.T) {
func TestNullBlockedRequest(t *testing.T) { func TestNullBlockedRequest(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
s.conf.FilteringConfig.BlockingMode = "null_ip" s.conf.FilteringConfig.BlockingMode = "null_ip"
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -312,10 +280,6 @@ func TestNullBlockedRequest(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0]) t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
} }
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -324,7 +288,6 @@ func TestNullBlockedRequest(t *testing.T) {
func TestBlockedByHosts(t *testing.T) { func TestBlockedByHosts(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -356,10 +319,6 @@ func TestBlockedByHosts(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0]) t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
} }
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -368,7 +327,6 @@ func TestBlockedByHosts(t *testing.T) {
func TestBlockedBySafeBrowsing(t *testing.T) { func TestBlockedBySafeBrowsing(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
@ -411,10 +369,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0]) t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
} }
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop() err = s.Stop()
if err != nil { if err != nil {
t.Fatalf("DNS server failed to stop: %s", err) t.Fatalf("DNS server failed to stop: %s", err)
@ -422,7 +376,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
} }
func createTestServer(t *testing.T) *Server { func createTestServer(t *testing.T) *Server {
s := NewServer(createDataDir(t), nil) s := NewServer(nil, nil)
s.conf.UDPListenAddr = &net.UDPAddr{Port: 0} s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.conf.TCPListenAddr = &net.TCPAddr{Port: 0} s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
@ -489,21 +443,6 @@ func createServerTLSConfig(t *testing.T) (*tls.Config, []byte, []byte) {
return &tls.Config{Certificates: []tls.Certificate{cert}, ServerName: tlsServerName, MinVersion: tls.VersionTLS12}, certPem, keyPem return &tls.Config{Certificates: []tls.Certificate{cert}, ServerName: tlsServerName, MinVersion: tls.VersionTLS12}, certPem, keyPem
} }
func createDataDir(t *testing.T) string {
err := os.MkdirAll(dataDir, 0755)
if err != nil {
t.Fatalf("Cannot create %s: %s", dataDir, err)
}
return dataDir
}
func removeDataDir(t *testing.T) {
err := os.RemoveAll(dataDir)
if err != nil {
t.Fatalf("Cannot remove %s: %s", dataDir, err)
}
}
func sendTestMessageAsync(t *testing.T, conn *dns.Conn, g *sync.WaitGroup) { func sendTestMessageAsync(t *testing.T, conn *dns.Conn, g *sync.WaitGroup) {
defer func() { defer func() {
g.Done() g.Done()
@ -607,7 +546,6 @@ func TestIsBlockedIPAllowed(t *testing.T) {
s.conf.AllowedClients = []string{"1.1.1.1", "2.2.0.0/16"} s.conf.AllowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
err := s.Start(nil) err := s.Start(nil)
defer removeDataDir(t)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
} }
@ -631,7 +569,6 @@ func TestIsBlockedIPDisallowed(t *testing.T) {
s.conf.DisallowedClients = []string{"1.1.1.1", "2.2.0.0/16"} s.conf.DisallowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
err := s.Start(nil) err := s.Start(nil)
defer removeDataDir(t)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
} }
@ -655,7 +592,6 @@ func TestIsBlockedIPBlockedDomain(t *testing.T) {
s.conf.BlockedHosts = []string{"host1", "host2"} s.conf.BlockedHosts = []string{"host1", "host2"}
err := s.Start(nil) err := s.Start(nil)
defer removeDataDir(t)
if err != nil { if err != nil {
t.Fatalf("Failed to start server: %s", err) t.Fatalf("Failed to start server: %s", err)
} }

View File

@ -12,6 +12,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/dhcpd" "github.com/AdguardTeam/AdGuardHome/dhcpd"
"github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward" "github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats" "github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/golibs/file" "github.com/AdguardTeam/golibs/file"
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
@ -70,6 +71,7 @@ type configuration struct {
transport *http.Transport transport *http.Transport
client *http.Client client *http.Client
stats stats.Stats stats stats.Stats
queryLog querylog.QueryLog
// cached version.json to avoid hammering github.io for each page reload // cached version.json to avoid hammering github.io for each page reload
versionCheckJSON []byte versionCheckJSON []byte
@ -175,6 +177,7 @@ var config = configuration{
BlockingMode: "nxdomain", // mode how to answer filtered requests BlockingMode: "nxdomain", // mode how to answer filtered requests
BlockedResponseTTL: 10, // in seconds BlockedResponseTTL: 10, // in seconds
QueryLogEnabled: true, QueryLogEnabled: true,
QueryLogInterval: 1,
Ratelimit: 20, Ratelimit: 20,
RefuseAny: true, RefuseAny: true,
BootstrapDNS: defaultBootstrap, BootstrapDNS: defaultBootstrap,
@ -274,6 +277,10 @@ func parseConfig() error {
config.DNS.StatsInterval = 1 config.DNS.StatsInterval = 1
} }
if !checkQueryLogInterval(config.DNS.QueryLogInterval) {
config.DNS.QueryLogInterval = 1
}
for _, cy := range config.Clients { for _, cy := range config.Clients {
cli := Client{ cli := Client{
Name: cy.Name, Name: cy.Name,

View File

@ -146,35 +146,6 @@ func handleProtectionDisable(w http.ResponseWriter, r *http.Request) {
httpUpdateConfigReloadDNSReturnOK(w, r) httpUpdateConfigReloadDNSReturnOK(w, r)
} }
// -----
// stats
// -----
func handleQueryLogEnable(w http.ResponseWriter, r *http.Request) {
config.DNS.QueryLogEnabled = true
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleQueryLogDisable(w http.ResponseWriter, r *http.Request) {
config.DNS.QueryLogEnabled = false
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleQueryLog(w http.ResponseWriter, r *http.Request) {
data := config.dnsServer.GetQueryLog()
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
}
// ----------------------- // -----------------------
// upstreams configuration // upstreams configuration
// ----------------------- // -----------------------
@ -570,9 +541,6 @@ func registerControlHandlers() {
httpRegister(http.MethodGet, "/control/status", handleStatus) httpRegister(http.MethodGet, "/control/status", handleStatus)
httpRegister(http.MethodPost, "/control/enable_protection", handleProtectionEnable) httpRegister(http.MethodPost, "/control/enable_protection", handleProtectionEnable)
httpRegister(http.MethodPost, "/control/disable_protection", handleProtectionDisable) httpRegister(http.MethodPost, "/control/disable_protection", handleProtectionDisable)
httpRegister(http.MethodGet, "/control/querylog", handleQueryLog)
httpRegister(http.MethodPost, "/control/querylog_enable", handleQueryLogEnable)
httpRegister(http.MethodPost, "/control/querylog_disable", handleQueryLogDisable)
httpRegister(http.MethodPost, "/control/set_upstreams_config", handleSetUpstreamConfig) httpRegister(http.MethodPost, "/control/set_upstreams_config", handleSetUpstreamConfig)
httpRegister(http.MethodPost, "/control/test_upstream_dns", handleTestUpstreamDNS) httpRegister(http.MethodPost, "/control/test_upstream_dns", handleTestUpstreamDNS)
httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage) httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage)
@ -611,6 +579,7 @@ func registerControlHandlers() {
RegisterClientsHandlers() RegisterClientsHandlers()
registerRewritesHandlers() registerRewritesHandlers()
RegisterBlockedServicesHandlers() RegisterBlockedServicesHandlers()
RegisterQueryLogHandlers()
RegisterStatsHandlers() RegisterStatsHandlers()
http.HandleFunc("/dns-query", postInstall(handleDOH)) http.HandleFunc("/dns-query", postInstall(handleDOH))

91
home/control_querylog.go Normal file
View File

@ -0,0 +1,91 @@
package home
import (
"encoding/json"
"net/http"
"github.com/AdguardTeam/AdGuardHome/querylog"
)
func handleQueryLog(w http.ResponseWriter, r *http.Request) {
data := config.queryLog.GetData()
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
}
func handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
config.queryLog.Clear()
returnOK(w)
}
type qlogConfig struct {
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"`
}
// Get configuration
func handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = config.DNS.QueryLogEnabled
resp.Interval = config.DNS.QueryLogInterval
jsonVal, err := json.Marshal(resp)
if err != nil {
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "http write: %s", err)
}
}
// Set configuration
func handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
reqData := qlogConfig{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
httpError(w, http.StatusBadRequest, "json decode: %s", err)
return
}
if !checkQueryLogInterval(reqData.Interval) {
httpError(w, http.StatusBadRequest, "Unsupported interval")
return
}
config.DNS.QueryLogEnabled = reqData.Enabled
config.DNS.QueryLogInterval = reqData.Interval
_ = config.write()
conf := querylog.Config{
Interval: config.DNS.QueryLogInterval * 24,
}
config.queryLog.Configure(conf)
returnOK(w)
}
func checkQueryLogInterval(i uint32) bool {
return i == 1 || i == 7 || i == 30 || i == 90
}
// RegisterQueryLogHandlers - register handlers
func RegisterQueryLogHandlers() {
httpRegister(http.MethodGet, "/control/querylog", handleQueryLog)
httpRegister(http.MethodGet, "/control/querylog_info", handleQueryLogInfo)
httpRegister(http.MethodPost, "/control/querylog_clear", handleQueryLogClear)
httpRegister(http.MethodPost, "/control/querylog_config", handleQueryLogConfig)
}

View File

@ -9,6 +9,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward" "github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats" "github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/dnsproxy/upstream"
@ -40,7 +41,12 @@ func initDNSServer(baseDir string) {
if err != nil { if err != nil {
log.Fatal("Couldn't initialize statistics module") log.Fatal("Couldn't initialize statistics module")
} }
config.dnsServer = dnsforward.NewServer(baseDir, config.stats) conf := querylog.Config{
BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval * 24,
}
config.queryLog = querylog.New(conf)
config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)
initRDNS() initRDNS()
} }
@ -186,6 +192,7 @@ func stopDNSServer() error {
} }
config.stats.Close() config.stats.Close()
config.queryLog.Close()
return nil return nil
} }

View File

@ -1,8 +1,9 @@
package dnsforward package querylog
import ( import (
"fmt" "fmt"
"net" "net"
"os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
@ -15,16 +16,14 @@ import (
) )
const ( const (
logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk
queryLogTimeLimit = time.Hour * 24 // how far in the past we care about querylogs queryLogFileName = "querylog.json" // .gz added during compression
queryLogRotationPeriod = time.Hour * 24 // rotate the log every 24 hours queryLogSize = 5000 // maximum API response for /querylog
queryLogFileName = "querylog.json" // .gz added during compression
queryLogSize = 5000 // maximum API response for /querylog
queryLogTopSize = 500 // Keep in memory only top N values
) )
// queryLog is a structure that writes and reads the DNS query log // queryLog is a structure that writes and reads the DNS query log
type queryLog struct { type queryLog struct {
conf Config
logFile string // path to the log file logFile string // path to the log file
logBufferLock sync.RWMutex logBufferLock sync.RWMutex
@ -32,16 +31,53 @@ type queryLog struct {
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
flushPending bool // don't start another goroutine while the previous one is still running flushPending bool // don't start another goroutine while the previous one is still running
queryLogCache []*logEntry cache []*logEntry
queryLogLock sync.RWMutex lock sync.RWMutex
} }
// newQueryLog creates a new instance of the query log // newQueryLog creates a new instance of the query log
func newQueryLog(baseDir string) *queryLog { func newQueryLog(conf Config) *queryLog {
l := &queryLog{ l := queryLog{}
logFile: filepath.Join(baseDir, queryLogFileName), l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
l.conf = conf
go l.periodicQueryLogRotate()
go l.fillFromFile()
return &l
}
func (l *queryLog) Close() {
_ = l.flushLogBuffer(true)
}
func (l *queryLog) Configure(conf Config) {
l.conf = conf
}
// Clear memory buffer and remove the file
func (l *queryLog) Clear() {
l.fileFlushLock.Lock()
defer l.fileFlushLock.Unlock()
l.logBufferLock.Lock()
l.logBuffer = nil
l.flushPending = false
l.logBufferLock.Unlock()
l.lock.Lock()
l.cache = nil
l.lock.Unlock()
err := os.Remove(l.logFile + ".1")
if err != nil {
log.Error("file remove: %s: %s", l.logFile+".1", err)
} }
return l
err = os.Remove(l.logFile)
if err != nil {
log.Error("file remove: %s: %s", l.logFile, err)
}
log.Debug("Query log: cleared")
} }
type logEntry struct { type logEntry struct {
@ -54,17 +90,28 @@ type logEntry struct {
Upstream string `json:",omitempty"` // if empty, means it was cached Upstream string `json:",omitempty"` // if empty, means it was cached
} }
func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) *logEntry { // getIPString is a helper function that extracts IP address from net.Addr
func getIPString(addr net.Addr) string {
switch addr := addr.(type) {
case *net.UDPAddr:
return addr.IP.String()
case *net.TCPAddr:
return addr.IP.String()
}
return ""
}
func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) {
var q []byte var q []byte
var a []byte var a []byte
var err error var err error
ip := GetIPString(addr) ip := getIPString(addr)
if question != nil { if question != nil {
q, err = question.Pack() q, err = question.Pack()
if err != nil { if err != nil {
log.Printf("failed to pack question for querylog: %s", err) log.Printf("failed to pack question for querylog: %s", err)
return nil return
} }
} }
@ -72,7 +119,7 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
a, err = answer.Pack() a, err = answer.Pack()
if err != nil { if err != nil {
log.Printf("failed to pack answer for querylog: %s", err) log.Printf("failed to pack answer for querylog: %s", err)
return nil return
} }
} }
@ -101,13 +148,13 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
} }
} }
l.logBufferLock.Unlock() l.logBufferLock.Unlock()
l.queryLogLock.Lock() l.lock.Lock()
l.queryLogCache = append(l.queryLogCache, &entry) l.cache = append(l.cache, &entry)
if len(l.queryLogCache) > queryLogSize { if len(l.cache) > queryLogSize {
toremove := len(l.queryLogCache) - queryLogSize toremove := len(l.cache) - queryLogSize
l.queryLogCache = l.queryLogCache[toremove:] l.cache = l.cache[toremove:]
} }
l.queryLogLock.Unlock() l.lock.Unlock()
// if buffer needs to be flushed to disk, do it now // if buffer needs to be flushed to disk, do it now
if needFlush { if needFlush {
@ -115,16 +162,14 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
// do it in separate goroutine -- we are stalling DNS response this whole time // do it in separate goroutine -- we are stalling DNS response this whole time
go l.flushLogBuffer(false) // nolint go l.flushLogBuffer(false) // nolint
} }
return &entry
} }
// getQueryLogJson returns a map with the current query log ready to be converted to a JSON // getQueryLogJson returns a map with the current query log ready to be converted to a JSON
func (l *queryLog) getQueryLog() []map[string]interface{} { func (l *queryLog) GetData() []map[string]interface{} {
l.queryLogLock.RLock() l.lock.RLock()
values := make([]*logEntry, len(l.queryLogCache)) values := make([]*logEntry, len(l.cache))
copy(values, l.queryLogCache) copy(values, l.cache)
l.queryLogLock.RUnlock() l.lock.RUnlock()
// reverse it so that newest is first // reverse it so that newest is first
for left, right := 0, len(values)-1; left < right; left, right = left+1, right-1 { for left, right := 0, len(values)-1; left < right; left, right = left+1, right-1 {

33
querylog/querylog.go Normal file
View File

@ -0,0 +1,33 @@
package querylog
import (
"net"
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/miekg/dns"
)
// QueryLog - main interface
type QueryLog interface {
Close()
// Set new configuration at runtime
// Currently only 'Interval' field is supported.
Configure(conf Config)
Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string)
GetData() []map[string]interface{}
Clear()
}
// Config - configuration object
type Config struct {
BaseDir string // directory where log file is stored
Interval uint32 // interval to rotate logs (in hours)
}
// New - create instance
func New(conf Config) QueryLog {
return newQueryLog(conf)
}

View File

@ -1,4 +1,4 @@
package dnsforward package querylog
import ( import (
"bytes" "bytes"
@ -11,6 +11,7 @@ import (
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/go-test/deep" "github.com/go-test/deep"
"github.com/miekg/dns"
) )
var ( var (
@ -170,7 +171,7 @@ func (l *queryLog) rotateQueryLog() error {
} }
func (l *queryLog) periodicQueryLogRotate() { func (l *queryLog) periodicQueryLogRotate() {
for range time.Tick(queryLogRotationPeriod) { for range time.Tick(time.Duration(l.conf.Interval) * time.Hour) {
err := l.rotateQueryLog() err := l.rotateQueryLog()
if err != nil { if err != nil {
log.Error("Failed to rotate querylog: %s", err) log.Error("Failed to rotate querylog: %s", err)
@ -178,3 +179,152 @@ func (l *queryLog) periodicQueryLogRotate() {
} }
} }
} }
// Reader is the DB reader context
type Reader struct {
f *os.File
jd *json.Decoder
now time.Time
ql *queryLog
files []string
ifile int
count uint64 // returned elements counter
}
// OpenReader locks the file and returns reader object or nil on error
func (l *queryLog) OpenReader() *Reader {
r := Reader{}
r.ql = l
r.now = time.Now()
return &r
}
// Close closes the reader
func (r *Reader) Close() {
elapsed := time.Since(r.now)
var perunit time.Duration
if r.count > 0 {
perunit = elapsed / time.Duration(r.count)
}
log.Debug("querylog: read %d entries in %v, %v/entry",
r.count, elapsed, perunit)
if r.f != nil {
r.f.Close()
}
}
// BeginRead starts reading
func (r *Reader) BeginRead() {
r.files = []string{
r.ql.logFile,
r.ql.logFile + ".1",
}
}
// Next returns the next entry or nil if reading is finished
func (r *Reader) Next() *logEntry { // nolint
var err error
for {
// open file if needed
if r.f == nil {
if r.ifile == len(r.files) {
return nil
}
fn := r.files[r.ifile]
r.f, err = os.Open(fn)
if err != nil {
log.Error("Failed to open file \"%s\": %s", fn, err)
r.ifile++
continue
}
}
// open decoder if needed
if r.jd == nil {
r.jd = json.NewDecoder(r.f)
}
// check if there's data
if !r.jd.More() {
r.jd = nil
r.f.Close()
r.f = nil
r.ifile++
continue
}
// read data
var entry logEntry
err = r.jd.Decode(&entry)
if err != nil {
log.Error("Failed to decode: %s", err)
// next entry can be fine, try more
continue
}
r.count++
return &entry
}
}
// Total returns the total number of items
func (r *Reader) Total() int {
return 0
}
// Fill cache from file
func (l *queryLog) fillFromFile() {
now := time.Now()
validFrom := now.Unix() - int64(l.conf.Interval*60*60)
r := l.OpenReader()
if r == nil {
return
}
r.BeginRead()
for {
entry := r.Next()
if entry == nil {
break
}
if entry.Time.Unix() < validFrom {
continue
}
if len(entry.Question) == 0 {
log.Printf("entry question is absent, skipping")
continue
}
if entry.Time.After(now) {
log.Printf("t %v vs %v is in the future, ignoring", entry.Time, now)
continue
}
q := new(dns.Msg)
if err := q.Unpack(entry.Question); err != nil {
log.Printf("failed to unpack dns message question: %s", err)
continue
}
if len(q.Question) != 1 {
log.Printf("malformed dns message, has no questions, skipping")
continue
}
l.lock.Lock()
l.cache = append(l.cache, entry)
if len(l.cache) > queryLogSize {
toremove := len(l.cache) - queryLogSize
l.cache = l.cache[toremove:]
}
l.lock.Unlock()
}
r.Close()
}

43
querylog/querylog_test.go Normal file
View File

@ -0,0 +1,43 @@
package querylog
import (
"net"
"testing"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
)
func TestQueryLog(t *testing.T) {
conf := Config{
Interval: 1,
}
l := New(conf)
q := dns.Msg{}
q.Question = append(q.Question, dns.Question{
Name: "example.org.",
Qtype: dns.TypeA,
Qclass: dns.ClassINET,
})
a := dns.Msg{}
a.Question = append(a.Question, q.Question[0])
answer := new(dns.A)
answer.Hdr = dns.RR_Header{
Name: q.Question[0].Name,
Rrtype: dns.TypeA,
Class: dns.ClassINET,
}
answer.A = net.IP{1, 2, 3, 4}
a.Answer = append(a.Answer, answer)
res := dnsfilter.Result{}
l.Add(&q, &a, &res, 0, nil, "upstream")
d := l.GetData()
m := d[0]
mq := m["question"].(map[string]interface{})
assert.True(t, mq["host"].(string) == "example.org")
}