mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* querylog: move code to a separate package 

+ config: "querylog_interval" setting
/control/querylog_config, /control/querylog_info
+ POST /control/querylog_clear
This commit is contained in:
Simon Zolin 2019-08-26 11:54:38 +03:00
parent 8f9ca4cba7
commit 8104c902ee
11 changed files with 457 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
AGHTechDoc.md
View File

@ -42,6 +42,9 @@ Contents:
* API: Clear statistics data
* API: Set statistics parameters
* API: Get statistics parameters
* Query logs
* API: Set querylog parameters
* API: Get querylog parameters
## First startup
@ -976,3 +979,37 @@ Response:
{
"interval": 1 | 7 | 30 | 90
}
## Query logs
### API: Set querylog parameters
Request:
POST /control/querylog_config
{
"enabled": true | false
"interval": 1 | 7 | 30 | 90
}
Response:
200 OK
### API: Get querylog parameters
Request:
GET /control/querylog_info
Response:
200 OK
{
"enabled": true | false
"interval": 1 | 7 | 30 | 90
}

29
dnsforward/dnsforward.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@ -40,7 +41,7 @@ const (
type Server struct {
dnsProxy *proxy.Proxy // DNS proxy instance
dnsFilter *dnsfilter.Dnsfilter // DNS filter instance
queryLog *queryLog // Query log instance
queryLog querylog.QueryLog // Query log instance
stats stats.Stats
AllowedClients map[string]bool // IP addresses of whitelist clients
@ -54,16 +55,11 @@ type Server struct {
}
// NewServer creates a new instance of the dnsforward.Server
// baseDir is the base directory for query logs
// Note: this function must be called only once
func NewServer(baseDir string, stats stats.Stats) *Server {
s := &Server{
queryLog: newQueryLog(baseDir),
}
func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server {
s := &Server{}
s.stats = stats
log.Printf("Start DNS server periodic jobs")
go s.queryLog.periodicQueryLogRotate()
s.queryLog = queryLog
return s
}
@ -75,6 +71,7 @@ type FilteringConfig struct {
BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests
BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600)
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
Ratelimit int `yaml:"ratelimit"` // max number of requests per second from a given IP (0 to disable)
RatelimitWhitelist []string `yaml:"ratelimit_whitelist"` // a list of whitelisted client IP addresses
RefuseAny bool `yaml:"refuse_any"` // if true, refuse ANY requests
@ -303,8 +300,7 @@ func (s *Server) stopInternal() error {
s.dnsFilter = nil
}
// flush remainder to file
return s.queryLog.flushLogBuffer(true)
return nil
}
// IsRunning returns true if the DNS server is running
@ -343,13 +339,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.RUnlock()
}
// GetQueryLog returns a map with the current query log ready to be converted to a JSON
func (s *Server) GetQueryLog() []map[string]interface{} {
s.RLock()
defer s.RUnlock()
return s.queryLog.getQueryLog()
}
// Return TRUE if this client should be blocked
func (s *Server) isBlockedIP(ip string) bool {
if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 {
@ -469,12 +458,12 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
}
elapsed := time.Since(start)
if s.conf.QueryLogEnabled && shouldLog {
if s.conf.QueryLogEnabled && shouldLog && s.queryLog != nil {
upstreamAddr := ""
if d.Upstream != nil {
upstreamAddr = d.Upstream.Address()
}
_ = s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
s.queryLog.Add(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
}
s.updateStats(d, elapsed, *res)

66
dnsforward/dnsforward_test.go
View File

@ -10,7 +10,6 @@ import (
"encoding/pem"
"math/big"
"net"
"os"
"sync"
"testing"
"time"
@ -18,18 +17,15 @@ import (
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
)
const (
tlsServerName = "testdns.adguard.com"
dataDir = "testData"
testMessagesCount = 10
)
func TestServer(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -45,10 +41,6 @@ func TestServer(t *testing.T) {
}
assertGoogleAResponse(t, reply)
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
// message over TCP
req = createGoogleATestMessage()
addr = s.dnsProxy.Addr("tcp")
@ -59,10 +51,6 @@ func TestServer(t *testing.T) {
}
assertGoogleAResponse(t, reply)
// check query log and stats again
log = s.GetQueryLog()
assert.Equal(t, 2, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -72,7 +60,6 @@ func TestServer(t *testing.T) {
func TestServerWithProtectionDisabled(t *testing.T) {
s := createTestServer(t)
s.conf.ProtectionEnabled = false
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -88,10 +75,6 @@ func TestServerWithProtectionDisabled(t *testing.T) {
}
assertGoogleAResponse(t, reply)
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -102,7 +85,6 @@ func TestDotServer(t *testing.T) {
// Prepare the proxy server
_, certPem, keyPem := createServerTLSConfig(t)
s := createTestServer(t)
defer removeDataDir(t)
s.conf.TLSConfig = TLSConfig{
TLSListenAddr: &net.TCPAddr{Port: 0},
@ -143,7 +125,6 @@ func TestDotServer(t *testing.T) {
func TestServerRace(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -168,7 +149,6 @@ func TestServerRace(t *testing.T) {
func TestSafeSearch(t *testing.T) {
s := createTestServer(t)
s.conf.SafeSearchEnabled = true
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -210,7 +190,6 @@ func TestSafeSearch(t *testing.T) {
func TestInvalidRequest(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -229,11 +208,6 @@ func TestInvalidRequest(t *testing.T) {
t.Fatalf("got a response to an invalid query")
}
// check query log and stats
// invalid requests aren't written to the query log
log := s.GetQueryLog()
assert.Equal(t, 0, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -242,7 +216,6 @@ func TestInvalidRequest(t *testing.T) {
func TestBlockedRequest(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -267,10 +240,6 @@ func TestBlockedRequest(t *testing.T) {
t.Fatalf("Wrong response: %s", reply.String())
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -280,7 +249,6 @@ func TestBlockedRequest(t *testing.T) {
func TestNullBlockedRequest(t *testing.T) {
s := createTestServer(t)
s.conf.FilteringConfig.BlockingMode = "null_ip"
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -312,10 +280,6 @@ func TestNullBlockedRequest(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -324,7 +288,6 @@ func TestNullBlockedRequest(t *testing.T) {
func TestBlockedByHosts(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -356,10 +319,6 @@ func TestBlockedByHosts(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -368,7 +327,6 @@ func TestBlockedByHosts(t *testing.T) {
func TestBlockedBySafeBrowsing(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@ -411,10 +369,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@ -422,7 +376,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
}
func createTestServer(t *testing.T) *Server {
s := NewServer(createDataDir(t), nil)
s := NewServer(nil, nil)
s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
@ -489,21 +443,6 @@ func createServerTLSConfig(t *testing.T) (*tls.Config, []byte, []byte) {
return &tls.Config{Certificates: []tls.Certificate{cert}, ServerName: tlsServerName, MinVersion: tls.VersionTLS12}, certPem, keyPem
}
func createDataDir(t *testing.T) string {
err := os.MkdirAll(dataDir, 0755)
if err != nil {
t.Fatalf("Cannot create %s: %s", dataDir, err)
}
return dataDir
}
func removeDataDir(t *testing.T) {
err := os.RemoveAll(dataDir)
if err != nil {
t.Fatalf("Cannot remove %s: %s", dataDir, err)
}
}
func sendTestMessageAsync(t *testing.T, conn *dns.Conn, g *sync.WaitGroup) {
defer func() {
g.Done()
@ -607,7 +546,6 @@ func TestIsBlockedIPAllowed(t *testing.T) {
s.conf.AllowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
err := s.Start(nil)
defer removeDataDir(t)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
}
@ -631,7 +569,6 @@ func TestIsBlockedIPDisallowed(t *testing.T) {
s.conf.DisallowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
err := s.Start(nil)
defer removeDataDir(t)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
}
@ -655,7 +592,6 @@ func TestIsBlockedIPBlockedDomain(t *testing.T) {
s.conf.BlockedHosts = []string{"host1", "host2"}
err := s.Start(nil)
defer removeDataDir(t)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
}

7
home/config.go
View File

@ -12,6 +12,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/dhcpd"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/golibs/file"
"github.com/AdguardTeam/golibs/log"
@ -70,6 +71,7 @@ type configuration struct {
transport *http.Transport
client *http.Client
stats stats.Stats
queryLog querylog.QueryLog
// cached version.json to avoid hammering github.io for each page reload
versionCheckJSON []byte
@ -175,6 +177,7 @@ var config = configuration{
BlockingMode: "nxdomain", // mode how to answer filtered requests
BlockedResponseTTL: 10, // in seconds
QueryLogEnabled: true,
QueryLogInterval: 1,
Ratelimit: 20,
RefuseAny: true,
BootstrapDNS: defaultBootstrap,
@ -274,6 +277,10 @@ func parseConfig() error {
config.DNS.StatsInterval = 1
}
if !checkQueryLogInterval(config.DNS.QueryLogInterval) {
config.DNS.QueryLogInterval = 1
}
for _, cy := range config.Clients {
cli := Client{
Name: cy.Name,

33
home/control.go
View File

@ -146,35 +146,6 @@ func handleProtectionDisable(w http.ResponseWriter, r *http.Request) {
httpUpdateConfigReloadDNSReturnOK(w, r)
}
// -----
// stats
// -----
func handleQueryLogEnable(w http.ResponseWriter, r *http.Request) {
config.DNS.QueryLogEnabled = true
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleQueryLogDisable(w http.ResponseWriter, r *http.Request) {
config.DNS.QueryLogEnabled = false
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleQueryLog(w http.ResponseWriter, r *http.Request) {
data := config.dnsServer.GetQueryLog()
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
}
// -----------------------
// upstreams configuration
// -----------------------
@ -570,9 +541,6 @@ func registerControlHandlers() {
httpRegister(http.MethodGet, "/control/status", handleStatus)
httpRegister(http.MethodPost, "/control/enable_protection", handleProtectionEnable)
httpRegister(http.MethodPost, "/control/disable_protection", handleProtectionDisable)
httpRegister(http.MethodGet, "/control/querylog", handleQueryLog)
httpRegister(http.MethodPost, "/control/querylog_enable", handleQueryLogEnable)
httpRegister(http.MethodPost, "/control/querylog_disable", handleQueryLogDisable)
httpRegister(http.MethodPost, "/control/set_upstreams_config", handleSetUpstreamConfig)
httpRegister(http.MethodPost, "/control/test_upstream_dns", handleTestUpstreamDNS)
httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage)
@ -611,6 +579,7 @@ func registerControlHandlers() {
RegisterClientsHandlers()
registerRewritesHandlers()
RegisterBlockedServicesHandlers()
RegisterQueryLogHandlers()
RegisterStatsHandlers()
http.HandleFunc("/dns-query", postInstall(handleDOH))

91
home/control_querylog.go Normal file
View File

@ -0,0 +1,91 @@
package home
import (
"encoding/json"
"net/http"
"github.com/AdguardTeam/AdGuardHome/querylog"
)
func handleQueryLog(w http.ResponseWriter, r *http.Request) {
data := config.queryLog.GetData()
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
}
func handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
config.queryLog.Clear()
returnOK(w)
}
type qlogConfig struct {
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"`
}
// Get configuration
func handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = config.DNS.QueryLogEnabled
resp.Interval = config.DNS.QueryLogInterval
jsonVal, err := json.Marshal(resp)
if err != nil {
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "http write: %s", err)
}
}
// Set configuration
func handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
reqData := qlogConfig{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
httpError(w, http.StatusBadRequest, "json decode: %s", err)
return
}
if !checkQueryLogInterval(reqData.Interval) {
httpError(w, http.StatusBadRequest, "Unsupported interval")
return
}
config.DNS.QueryLogEnabled = reqData.Enabled
config.DNS.QueryLogInterval = reqData.Interval
_ = config.write()
conf := querylog.Config{
Interval: config.DNS.QueryLogInterval * 24,
}
config.queryLog.Configure(conf)
returnOK(w)
}
func checkQueryLogInterval(i uint32) bool {
return i == 1 || i == 7 || i == 30 || i == 90
}
// RegisterQueryLogHandlers - register handlers
func RegisterQueryLogHandlers() {
httpRegister(http.MethodGet, "/control/querylog", handleQueryLog)
httpRegister(http.MethodGet, "/control/querylog_info", handleQueryLogInfo)
httpRegister(http.MethodPost, "/control/querylog_clear", handleQueryLogClear)
httpRegister(http.MethodPost, "/control/querylog_config", handleQueryLogConfig)
}

9
home/dns.go
View File

@ -9,6 +9,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@ -40,7 +41,12 @@ func initDNSServer(baseDir string) {
if err != nil {
log.Fatal("Couldn't initialize statistics module")
}
config.dnsServer = dnsforward.NewServer(baseDir, config.stats)
conf := querylog.Config{
BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval * 24,
}
config.queryLog = querylog.New(conf)
config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)
initRDNS()
}
@ -186,6 +192,7 @@ func stopDNSServer() error {
}
config.stats.Close()
config.queryLog.Close()
return nil
}

105
dnsforward/querylog.go → querylog/qlog.go
View File

@ -1,8 +1,9 @@
package dnsforward
package querylog
import (
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
@ -15,16 +16,14 @@ import (
)
const (
logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk
queryLogTimeLimit = time.Hour * 24 // how far in the past we care about querylogs
queryLogRotationPeriod = time.Hour * 24 // rotate the log every 24 hours
queryLogFileName = "querylog.json" // .gz added during compression
queryLogSize = 5000 // maximum API response for /querylog
queryLogTopSize = 500 // Keep in memory only top N values
logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk
queryLogFileName = "querylog.json" // .gz added during compression
queryLogSize = 5000 // maximum API response for /querylog
)
// queryLog is a structure that writes and reads the DNS query log
type queryLog struct {
conf Config
logFile string // path to the log file
logBufferLock sync.RWMutex
@ -32,16 +31,53 @@ type queryLog struct {
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
flushPending bool // don't start another goroutine while the previous one is still running
queryLogCache []*logEntry
queryLogLock sync.RWMutex
cache []*logEntry
lock sync.RWMutex
}
// newQueryLog creates a new instance of the query log
func newQueryLog(baseDir string) *queryLog {
l := &queryLog{
logFile: filepath.Join(baseDir, queryLogFileName),
func newQueryLog(conf Config) *queryLog {
l := queryLog{}
l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
l.conf = conf
go l.periodicQueryLogRotate()
go l.fillFromFile()
return &l
}
func (l *queryLog) Close() {
_ = l.flushLogBuffer(true)
}
func (l *queryLog) Configure(conf Config) {
l.conf = conf
}
// Clear memory buffer and remove the file
func (l *queryLog) Clear() {
l.fileFlushLock.Lock()
defer l.fileFlushLock.Unlock()
l.logBufferLock.Lock()
l.logBuffer = nil
l.flushPending = false
l.logBufferLock.Unlock()
l.lock.Lock()
l.cache = nil
l.lock.Unlock()
err := os.Remove(l.logFile + ".1")
if err != nil {
log.Error("file remove: %s: %s", l.logFile+".1", err)
}
return l
err = os.Remove(l.logFile)
if err != nil {
log.Error("file remove: %s: %s", l.logFile, err)
}
log.Debug("Query log: cleared")
}
type logEntry struct {
@ -54,17 +90,28 @@ type logEntry struct {
Upstream string `json:",omitempty"` // if empty, means it was cached
}
func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) *logEntry {
// getIPString is a helper function that extracts IP address from net.Addr
func getIPString(addr net.Addr) string {
switch addr := addr.(type) {
case *net.UDPAddr:
return addr.IP.String()
case *net.TCPAddr:
return addr.IP.String()
}
return ""
}
func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) {
var q []byte
var a []byte
var err error
ip := GetIPString(addr)
ip := getIPString(addr)
if question != nil {
q, err = question.Pack()
if err != nil {
log.Printf("failed to pack question for querylog: %s", err)
return nil
return
}
}
@ -72,7 +119,7 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
a, err = answer.Pack()
if err != nil {
log.Printf("failed to pack answer for querylog: %s", err)
return nil
return
}
}
@ -101,13 +148,13 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
}
}
l.logBufferLock.Unlock()
l.queryLogLock.Lock()
l.queryLogCache = append(l.queryLogCache, &entry)
if len(l.queryLogCache) > queryLogSize {
toremove := len(l.queryLogCache) - queryLogSize
l.queryLogCache = l.queryLogCache[toremove:]
l.lock.Lock()
l.cache = append(l.cache, &entry)
if len(l.cache) > queryLogSize {
toremove := len(l.cache) - queryLogSize
l.cache = l.cache[toremove:]
}
l.queryLogLock.Unlock()
l.lock.Unlock()
// if buffer needs to be flushed to disk, do it now
if needFlush {
@ -115,16 +162,14 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
// do it in separate goroutine -- we are stalling DNS response this whole time
go l.flushLogBuffer(false) // nolint
}
return &entry
}
// getQueryLogJson returns a map with the current query log ready to be converted to a JSON
func (l *queryLog) getQueryLog() []map[string]interface{} {
l.queryLogLock.RLock()
values := make([]*logEntry, len(l.queryLogCache))
copy(values, l.queryLogCache)
l.queryLogLock.RUnlock()
func (l *queryLog) GetData() []map[string]interface{} {
l.lock.RLock()
values := make([]*logEntry, len(l.cache))
copy(values, l.cache)
l.lock.RUnlock()
// reverse it so that newest is first
for left, right := 0, len(values)-1; left < right; left, right = left+1, right-1 {

33
querylog/querylog.go Normal file
View File

@ -0,0 +1,33 @@
package querylog
import (
"net"
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/miekg/dns"
)
// QueryLog - main interface
type QueryLog interface {
Close()
// Set new configuration at runtime
// Currently only 'Interval' field is supported.
Configure(conf Config)
Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string)
GetData() []map[string]interface{}
Clear()
}
// Config - configuration object
type Config struct {
BaseDir string // directory where log file is stored
Interval uint32 // interval to rotate logs (in hours)
}
// New - create instance
func New(conf Config) QueryLog {
return newQueryLog(conf)
}

154
dnsforward/querylog_file.go → querylog/querylog_file.go
View File

@ -1,4 +1,4 @@
package dnsforward
package querylog
import (
"bytes"
@ -11,6 +11,7 @@ import (
"github.com/AdguardTeam/golibs/log"
"github.com/go-test/deep"
"github.com/miekg/dns"
)
var (
@ -170,7 +171,7 @@ func (l *queryLog) rotateQueryLog() error {
}
func (l *queryLog) periodicQueryLogRotate() {
for range time.Tick(queryLogRotationPeriod) {
for range time.Tick(time.Duration(l.conf.Interval) * time.Hour) {
err := l.rotateQueryLog()
if err != nil {
log.Error("Failed to rotate querylog: %s", err)
@ -178,3 +179,152 @@ func (l *queryLog) periodicQueryLogRotate() {
}
}
}
// Reader is the DB reader context
type Reader struct {
f *os.File
jd *json.Decoder
now time.Time
ql *queryLog
files []string
ifile int
count uint64 // returned elements counter
}
// OpenReader locks the file and returns reader object or nil on error
func (l *queryLog) OpenReader() *Reader {
r := Reader{}
r.ql = l
r.now = time.Now()
return &r
}
// Close closes the reader
func (r *Reader) Close() {
elapsed := time.Since(r.now)
var perunit time.Duration
if r.count > 0 {
perunit = elapsed / time.Duration(r.count)
}
log.Debug("querylog: read %d entries in %v, %v/entry",
r.count, elapsed, perunit)
if r.f != nil {
r.f.Close()
}
}
// BeginRead starts reading
func (r *Reader) BeginRead() {
r.files = []string{
r.ql.logFile,
r.ql.logFile + ".1",
}
}
// Next returns the next entry or nil if reading is finished
func (r *Reader) Next() *logEntry { // nolint
var err error
for {
// open file if needed
if r.f == nil {
if r.ifile == len(r.files) {
return nil
}
fn := r.files[r.ifile]
r.f, err = os.Open(fn)
if err != nil {
log.Error("Failed to open file \"%s\": %s", fn, err)
r.ifile++
continue
}
}
// open decoder if needed
if r.jd == nil {
r.jd = json.NewDecoder(r.f)
}
// check if there's data
if !r.jd.More() {
r.jd = nil
r.f.Close()
r.f = nil
r.ifile++
continue
}
// read data
var entry logEntry
err = r.jd.Decode(&entry)
if err != nil {
log.Error("Failed to decode: %s", err)
// next entry can be fine, try more
continue
}
r.count++
return &entry
}
}
// Total returns the total number of items
func (r *Reader) Total() int {
return 0
}
// Fill cache from file
func (l *queryLog) fillFromFile() {
now := time.Now()
validFrom := now.Unix() - int64(l.conf.Interval*60*60)
r := l.OpenReader()
if r == nil {
return
}
r.BeginRead()
for {
entry := r.Next()
if entry == nil {
break
}
if entry.Time.Unix() < validFrom {
continue
}
if len(entry.Question) == 0 {
log.Printf("entry question is absent, skipping")
continue
}
if entry.Time.After(now) {
log.Printf("t %v vs %v is in the future, ignoring", entry.Time, now)
continue
}
q := new(dns.Msg)
if err := q.Unpack(entry.Question); err != nil {
log.Printf("failed to unpack dns message question: %s", err)
continue
}
if len(q.Question) != 1 {
log.Printf("malformed dns message, has no questions, skipping")
continue
}
l.lock.Lock()
l.cache = append(l.cache, entry)
if len(l.cache) > queryLogSize {
toremove := len(l.cache) - queryLogSize
l.cache = l.cache[toremove:]
}
l.lock.Unlock()
}
r.Close()
}

43
querylog/querylog_test.go Normal file
View File

@ -0,0 +1,43 @@
package querylog
import (
"net"
"testing"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
)
func TestQueryLog(t *testing.T) {
conf := Config{
Interval: 1,
}
l := New(conf)
q := dns.Msg{}
q.Question = append(q.Question, dns.Question{
Name: "example.org.",
Qtype: dns.TypeA,
Qclass: dns.ClassINET,
})
a := dns.Msg{}
a.Question = append(a.Question, q.Question[0])
answer := new(dns.A)
answer.Hdr = dns.RR_Header{
Name: q.Question[0].Name,
Rrtype: dns.TypeA,
Class: dns.ClassINET,
}
answer.A = net.IP{1, 2, 3, 4}
a.Answer = append(a.Answer, answer)
res := dnsfilter.Result{}
l.Add(&q, &a, &res, 0, nil, "upstream")
d := l.GetData()
m := d[0]
mq := m["question"].(map[string]interface{})
assert.True(t, mq["host"].(string) == "example.org")
}