package querylog import ( "fmt" "net" "os" "path/filepath" "strconv" "strings" "sync" "time" "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/golibs/log" "github.com/miekg/dns" ) const ( queryLogFileName = "querylog.json" // .gz added during compression getDataLimit = 500 // GetData(): maximum log entries to return // maximum entries to parse when searching maxSearchEntries = 50000 ) // queryLog is a structure that writes and reads the DNS query log type queryLog struct { conf *Config lock sync.Mutex logFile string // path to the log file bufferLock sync.RWMutex buffer []*logEntry fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread flushPending bool // don't start another goroutine while the previous one is still running fileWriteLock sync.Mutex } // create a new instance of the query log func newQueryLog(conf Config) *queryLog { l := queryLog{} l.logFile = filepath.Join(conf.BaseDir, queryLogFileName) l.conf = &Config{} *l.conf = conf if !checkInterval(l.conf.Interval) { l.conf.Interval = 1 } return &l } func (l *queryLog) Start() { if l.conf.HTTPRegister != nil { l.initWeb() } go l.periodicRotate() } func (l *queryLog) Close() { _ = l.flushLogBuffer(true) } func checkInterval(days uint32) bool { return days == 1 || days == 7 || days == 30 || days == 90 } func (l *queryLog) WriteDiskConfig(dc *DiskConfig) { dc.Enabled = l.conf.Enabled dc.Interval = l.conf.Interval dc.MemSize = l.conf.MemSize dc.AnonymizeClientIP = l.conf.AnonymizeClientIP } // Clear memory buffer and remove log files func (l *queryLog) clear() { l.fileFlushLock.Lock() defer l.fileFlushLock.Unlock() l.bufferLock.Lock() l.buffer = nil l.flushPending = false l.bufferLock.Unlock() err := os.Remove(l.logFile + ".1") if err != nil && !os.IsNotExist(err) { log.Error("file remove: %s: %s", l.logFile+".1", err) } err = os.Remove(l.logFile) if err != nil && !os.IsNotExist(err) { log.Error("file remove: %s: %s", l.logFile, err) } log.Debug("Query log: cleared") } type logEntry struct { IP string `json:"IP"` Time time.Time `json:"T"` QHost string `json:"QH"` QType string `json:"QT"` QClass string `json:"QC"` Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net OrigAnswer []byte `json:",omitempty"` Result dnsfilter.Result Elapsed time.Duration Upstream string `json:",omitempty"` // if empty, means it was cached } func (l *queryLog) Add(params AddParams) { if !l.conf.Enabled { return } if params.Question == nil || len(params.Question.Question) != 1 || len(params.Question.Question[0].Name) == 0 || params.ClientIP == nil { return } if params.Result == nil { params.Result = &dnsfilter.Result{} } now := time.Now() entry := logEntry{ IP: l.getClientIP(params.ClientIP.String()), Time: now, Result: *params.Result, Elapsed: params.Elapsed, Upstream: params.Upstream, } q := params.Question.Question[0] entry.QHost = strings.ToLower(q.Name[:len(q.Name)-1]) // remove the last dot entry.QType = dns.Type(q.Qtype).String() entry.QClass = dns.Class(q.Qclass).String() if params.Answer != nil { a, err := params.Answer.Pack() if err != nil { log.Info("Querylog: Answer.Pack(): %s", err) return } entry.Answer = a } if params.OrigAnswer != nil { a, err := params.OrigAnswer.Pack() if err != nil { log.Info("Querylog: OrigAnswer.Pack(): %s", err) return } entry.OrigAnswer = a } l.bufferLock.Lock() l.buffer = append(l.buffer, &entry) needFlush := false if !l.flushPending { needFlush = len(l.buffer) >= int(l.conf.MemSize) if needFlush { l.flushPending = true } } l.bufferLock.Unlock() // if buffer needs to be flushed to disk, do it now if needFlush { // write to file // do it in separate goroutine -- we are stalling DNS response this whole time go l.flushLogBuffer(false) // nolint } } // Parameters for getData() type getDataParams struct { OlderThan time.Time // return entries that are older than this value Domain string // filter by domain name in question Client string // filter by client IP QuestionType string // filter by question type ResponseStatus responseStatusType // filter by response status StrictMatchDomain bool // if Domain value must be matched strictly StrictMatchClient bool // if Client value must be matched strictly } // Response status type responseStatusType int32 // Response status constants const ( responseStatusAll responseStatusType = iota + 1 responseStatusFiltered ) // Gets log entries func (l *queryLog) getData(params getDataParams) map[string]interface{} { now := time.Now() if len(params.Client) != 0 && l.conf.AnonymizeClientIP { params.Client = l.getClientIP(params.Client) } // add from file fileEntries, oldest, total := l.searchFiles(params) if params.OlderThan.IsZero() { // In case if the timer is not precise (for instance, on Windows) // We really want to get all records including those added just before the call params.OlderThan = now.Add(time.Millisecond) } // add from memory buffer l.bufferLock.Lock() total += len(l.buffer) memoryEntries := make([]*logEntry, 0) // go through the buffer in the reverse order // from NEWER to OLDER for i := len(l.buffer) - 1; i >= 0; i-- { entry := l.buffer[i] if entry.Time.UnixNano() >= params.OlderThan.UnixNano() { // Ignore entries newer than what was requested continue } if !matchesGetDataParams(entry, params) { continue } memoryEntries = append(memoryEntries, entry) } l.bufferLock.Unlock() // now let's get a unified collection entries := append(memoryEntries, fileEntries...) if len(entries) > getDataLimit { // remove extra records entries = entries[:getDataLimit] } if len(entries) == getDataLimit { // change the "oldest" value here. // we cannot use the "oldest" we got from "searchFiles" anymore // because after adding in-memory records and removing extra records // the situation has changed oldest = entries[len(entries)-1].Time } // init the response object var data = []map[string]interface{}{} // the elements order is already reversed (from newer to older) for i := 0; i < len(entries); i++ { entry := entries[i] jsonEntry := l.logEntryToJSONEntry(entry) data = append(data, jsonEntry) } log.Debug("QueryLog: prepared data (%d/%d) older than %s in %s", len(entries), total, params.OlderThan, time.Since(now)) var result = map[string]interface{}{} result["oldest"] = "" if !oldest.IsZero() { result["oldest"] = oldest.Format(time.RFC3339Nano) } result["data"] = data return result } // Get Client IP address func (l *queryLog) getClientIP(clientIP string) string { if l.conf.AnonymizeClientIP { ip := net.ParseIP(clientIP) if ip != nil { ip4 := ip.To4() const AnonymizeClientIP4Mask = 24 const AnonymizeClientIP6Mask = 112 if ip4 != nil { clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String() } else { clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String() } } } return clientIP } func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} { var msg *dns.Msg if len(entry.Answer) > 0 { msg = new(dns.Msg) if err := msg.Unpack(entry.Answer); err != nil { log.Debug("Failed to unpack dns message answer: %s: %s", err, string(entry.Answer)) msg = nil } } jsonEntry := map[string]interface{}{ "reason": entry.Result.Reason.String(), "elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64), "time": entry.Time.Format(time.RFC3339Nano), "client": l.getClientIP(entry.IP), } jsonEntry["question"] = map[string]interface{}{ "host": entry.QHost, "type": entry.QType, "class": entry.QClass, } if msg != nil { jsonEntry["status"] = dns.RcodeToString[msg.Rcode] opt := msg.IsEdns0() dnssecOk := false if opt != nil { dnssecOk = opt.Do() } jsonEntry["answer_dnssec"] = dnssecOk } if len(entry.Result.Rule) > 0 { jsonEntry["rule"] = entry.Result.Rule jsonEntry["filterId"] = entry.Result.FilterID } if len(entry.Result.ServiceName) != 0 { jsonEntry["service_name"] = entry.Result.ServiceName } answers := answerToMap(msg) if answers != nil { jsonEntry["answer"] = answers } if len(entry.OrigAnswer) != 0 { a := new(dns.Msg) err := a.Unpack(entry.OrigAnswer) if err == nil { answers = answerToMap(a) if answers != nil { jsonEntry["original_answer"] = answers } } else { log.Debug("Querylog: msg.Unpack(entry.OrigAnswer): %s: %s", err, string(entry.OrigAnswer)) } } return jsonEntry } func answerToMap(a *dns.Msg) []map[string]interface{} { if a == nil || len(a.Answer) == 0 { return nil } var answers = []map[string]interface{}{} for _, k := range a.Answer { header := k.Header() answer := map[string]interface{}{ "type": dns.TypeToString[header.Rrtype], "ttl": header.Ttl, } // try most common record types switch v := k.(type) { case *dns.A: answer["value"] = v.A.String() case *dns.AAAA: answer["value"] = v.AAAA.String() case *dns.MX: answer["value"] = fmt.Sprintf("%v %v", v.Preference, v.Mx) case *dns.CNAME: answer["value"] = v.Target case *dns.NS: answer["value"] = v.Ns case *dns.SPF: answer["value"] = v.Txt case *dns.TXT: answer["value"] = v.Txt case *dns.PTR: answer["value"] = v.Ptr case *dns.SOA: answer["value"] = fmt.Sprintf("%v %v %v %v %v %v %v", v.Ns, v.Mbox, v.Serial, v.Refresh, v.Retry, v.Expire, v.Minttl) case *dns.CAA: answer["value"] = fmt.Sprintf("%v %v \"%v\"", v.Flag, v.Tag, v.Value) case *dns.HINFO: answer["value"] = fmt.Sprintf("\"%v\" \"%v\"", v.Cpu, v.Os) case *dns.RRSIG: answer["value"] = fmt.Sprintf("%v %v %v %v %v %v %v %v %v", dns.TypeToString[v.TypeCovered], v.Algorithm, v.Labels, v.OrigTtl, v.Expiration, v.Inception, v.KeyTag, v.SignerName, v.Signature) default: // type unknown, marshall it as-is answer["value"] = v } answers = append(answers, answer) } return answers }