From 2f5d6593f2d9841f9c18b9ac60a554af60a34a6b Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Thu, 24 Oct 2019 20:00:58 +0300 Subject: [PATCH 1/8] * querylog: major refactor: change on-disk format and API speed up decoding speed up search compatible with previous format (when not searching) --- AGHTechDoc.md | 18 ++- querylog/qlog.go | 149 ++++++++---------- querylog/qlog_http.go | 4 +- querylog/querylog_file.go | 311 ++++++++++++++++++++++++++++++++++---- querylog/querylog_test.go | 30 +++- 5 files changed, 387 insertions(+), 125 deletions(-) diff --git a/AGHTechDoc.md b/AGHTechDoc.md index d901a862..dafb44c0 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -1012,17 +1012,20 @@ Response: When a new DNS request is received and processed, we store information about this event in "query log". It is a file on disk in JSON format: { - "Question":"..."," - Answer":"...", + "IP":"127.0.0.1", // client IP + "T":"...", // response time + "QH":"...", // target host name without the last dot + "QT":"...", // question type + "QC":"...", // question class + "Answer":"...", "Result":{ "IsFiltered":true, "Reason":3, "Rule":"...", "FilterID":1 }, - "Time":"...", "Elapsed":12345, - "IP":"127.0.0.1" + "Upstream":"...", } @@ -1052,7 +1055,7 @@ Request: &filter_question_type=A | AAAA &filter_response_status= | filtered -If `older_than` value is set, server returns the next chunk of entries that are older than this time stamp. This setting is used for paging. UI sets the empty value on the first request and gets the latest log entries. To get the older entries, UI sets this value to the timestamp of the last (the oldest) entry from the previous response from Server. +`older_than` setting is used for paging. UI uses an empty value for `older_than` on the first request and gets the latest log entries. To get the older entries, UI sets `older_than` to the `oldest` value from the server's response. If "filter" settings are set, server returns only entries that match the specified request. @@ -1060,7 +1063,9 @@ For `filter.domain` and `filter.client` the server matches substrings by default Response: - [ + { + "oldest":"2006-01-02T15:04:05.999999999Z07:00" + "data":[ { "answer":[ { @@ -1085,6 +1090,7 @@ Response: } ... ] + } The most recent entries are at the top of list. diff --git a/querylog/qlog.go b/querylog/qlog.go index 436216ae..e53d685c 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -20,8 +20,8 @@ const ( queryLogFileName = "querylog.json" // .gz added during compression getDataLimit = 500 // GetData(): maximum log entries to return - // maximum data chunks to parse when filtering entries - maxFilteringChunks = 10 + // maximum entries to parse when searching + maxSearchEntries = 50000 ) // queryLog is a structure that writes and reads the DNS query log @@ -94,12 +94,16 @@ func (l *queryLog) clear() { } type logEntry struct { - Question []byte + IP string `json:"IP"` + Time time.Time `json:"T"` + + QHost string `json:"QH"` + QType string `json:"QT"` + QClass string `json:"QC"` + Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net Result dnsfilter.Result - Time time.Time Elapsed time.Duration - IP string Upstream string `json:",omitempty"` // if empty, means it was cached } @@ -119,21 +123,15 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res return } - var q []byte + if question == nil || len(question.Question) != 1 || len(question.Question[0].Name) == 0 || + ip == nil { + return + } + var a []byte var err error ip := getIPString(addr) - if question == nil { - return - } - - q, err = question.Pack() - if err != nil { - log.Printf("failed to pack question for querylog: %s", err) - return - } - if answer != nil { a, err = answer.Pack() if err != nil { @@ -148,14 +146,18 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res now := time.Now() entry := logEntry{ - Question: q, + IP: ip, + Time: now, + Answer: a, Result: *result, - Time: now, Elapsed: elapsed, - IP: ip, Upstream: upstream, } + q := question.Question[0] + entry.QHost = strings.ToLower(q.Name[:len(q.Name)-1]) // remove the last dot + entry.QType = dns.Type(q.Qtype).String() + entry.QClass = dns.Class(q.Qclass).String() l.bufferLock.Lock() l.buffer = append(l.buffer, &entry) @@ -182,33 +184,22 @@ func isNeeded(entry *logEntry, params getDataParams) bool { return false } - if len(params.Domain) != 0 || params.QuestionType != 0 { - m := dns.Msg{} - _ = m.Unpack(entry.Question) - - if params.QuestionType != 0 { - if m.Question[0].Qtype != params.QuestionType { - return false - } - } - - if len(params.Domain) != 0 && params.StrictMatchDomain { - if m.Question[0].Name != params.Domain { - return false - } - } else if len(params.Domain) != 0 { - if strings.Index(m.Question[0].Name, params.Domain) == -1 { - return false - } + if len(params.QuestionType) != 0 { + if entry.QType != params.QuestionType { + return false } } - if len(params.Client) != 0 && params.StrictMatchClient { - if entry.IP != params.Client { + if len(params.Domain) != 0 { + if (params.StrictMatchDomain && entry.QHost != params.Domain) || + (!params.StrictMatchDomain && strings.Index(entry.QHost, params.Domain) == -1) { return false } - } else if len(params.Client) != 0 { - if strings.Index(entry.IP, params.Client) == -1 { + } + + if len(params.Client) != 0 { + if (params.StrictMatchClient && entry.IP != params.Client) || + (!params.StrictMatchClient && strings.Index(entry.IP, params.Client) == -1) { return false } } @@ -216,31 +207,23 @@ func isNeeded(entry *logEntry, params getDataParams) bool { return true } -func (l *queryLog) readFromFile(params getDataParams) ([]*logEntry, int) { +func (l *queryLog) readFromFile(params getDataParams) ([]*logEntry, time.Time, int) { entries := []*logEntry{} - olderThan := params.OlderThan - totalChunks := 0 - total := 0 + oldest := time.Time{} r := l.OpenReader() if r == nil { - return entries, 0 + return entries, time.Time{}, 0 } - r.BeginRead(olderThan, getDataLimit) - for totalChunks < maxFilteringChunks { - first := true + r.BeginRead(params.OlderThan, getDataLimit, ¶ms) + total := uint64(0) + for total <= maxSearchEntries { newEntries := []*logEntry{} for { entry := r.Next() if entry == nil { break } - total++ - - if first { - first = false - olderThan = entry.Time - } if !isNeeded(entry, params) { continue @@ -251,7 +234,7 @@ func (l *queryLog) readFromFile(params getDataParams) ([]*logEntry, int) { newEntries = append(newEntries, entry) } - log.Debug("entries: +%d (%d) older-than:%s", len(newEntries), len(entries), olderThan) + log.Debug("entries: +%d (%d) [%d]", len(newEntries), len(entries), r.Total()) entries = append(newEntries, entries...) if len(entries) > getDataLimit { @@ -259,15 +242,16 @@ func (l *queryLog) readFromFile(params getDataParams) ([]*logEntry, int) { entries = entries[toremove:] break } - if first || len(entries) == getDataLimit { + if r.Total() == 0 || len(entries) == getDataLimit { break } - totalChunks++ - r.BeginReadPrev(olderThan, getDataLimit) + total += r.Total() + oldest = r.Oldest() + r.BeginReadPrev(getDataLimit) } r.Close() - return entries, total + return entries, oldest, int(total) } // Parameters for getData() @@ -275,7 +259,7 @@ type getDataParams struct { OlderThan time.Time // return entries that are older than this value Domain string // filter by domain name in question Client string // filter by client IP - QuestionType uint16 // filter by question type + QuestionType string // filter by question type ResponseStatus responseStatusType // filter by response status StrictMatchDomain bool // if Domain value must be matched strictly StrictMatchClient bool // if Client value must be matched strictly @@ -291,19 +275,16 @@ const ( ) // Get log entries -func (l *queryLog) getData(params getDataParams) []map[string]interface{} { +func (l *queryLog) getData(params getDataParams) map[string]interface{} { var data = []map[string]interface{}{} - if len(params.Domain) != 0 && params.StrictMatchDomain { - params.Domain = params.Domain + "." - } - + var oldest time.Time now := time.Now() entries := []*logEntry{} total := 0 // add from file - entries, total = l.readFromFile(params) + entries, oldest, total = l.readFromFile(params) if params.OlderThan.IsZero() { params.OlderThan = now @@ -332,26 +313,12 @@ func (l *queryLog) getData(params getDataParams) []map[string]interface{} { // process the elements from latest to oldest for i := len(entries) - 1; i >= 0; i-- { entry := entries[i] - var q *dns.Msg var a *dns.Msg - if len(entry.Question) == 0 { - continue - } - q = new(dns.Msg) - if err := q.Unpack(entry.Question); err != nil { - log.Tracef("q.Unpack(): %s", err) - continue - } - if len(q.Question) != 1 { - log.Tracef("len(q.Question) != 1") - continue - } - if len(entry.Answer) > 0 { a = new(dns.Msg) if err := a.Unpack(entry.Answer); err != nil { - log.Debug("Failed to unpack dns message answer: %s", err) + log.Debug("Failed to unpack dns message answer: %s: %s", err, string(entry.Answer)) a = nil } } @@ -363,9 +330,9 @@ func (l *queryLog) getData(params getDataParams) []map[string]interface{} { "client": entry.IP, } jsonEntry["question"] = map[string]interface{}{ - "host": strings.ToLower(strings.TrimSuffix(q.Question[0].Name, ".")), - "type": dns.Type(q.Question[0].Qtype).String(), - "class": dns.Class(q.Question[0].Qclass).String(), + "host": entry.QHost, + "type": entry.QType, + "class": entry.QClass, } if a != nil { @@ -390,7 +357,17 @@ func (l *queryLog) getData(params getDataParams) []map[string]interface{} { log.Debug("QueryLog: prepared data (%d/%d) older than %s in %s", len(entries), total, params.OlderThan, time.Since(now)) - return data + + var result = map[string]interface{}{} + if len(entries) == getDataLimit { + oldest = entries[0].Time + } + result["oldest"] = "" + if !oldest.IsZero() { + result["oldest"] = oldest.Format(time.RFC3339Nano) + } + result["data"] = data + return result } func answerToMap(a *dns.Msg) []map[string]interface{} { diff --git a/querylog/qlog_http.go b/querylog/qlog_http.go index d48cd053..d9feb9b6 100644 --- a/querylog/qlog_http.go +++ b/querylog/qlog_http.go @@ -67,12 +67,12 @@ func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) { } if len(req.filterQuestionType) != 0 { - qtype, ok := dns.StringToType[req.filterQuestionType] + _, ok := dns.StringToType[req.filterQuestionType] if !ok { httpError(r, w, http.StatusBadRequest, "invalid question_type") return } - params.QuestionType = qtype + params.QuestionType = req.filterQuestionType } if len(req.filterResponseStatus) != 0 { diff --git a/querylog/querylog_file.go b/querylog/querylog_file.go index 3b8f2663..1a466813 100644 --- a/querylog/querylog_file.go +++ b/querylog/querylog_file.go @@ -4,13 +4,17 @@ import ( "bufio" "bytes" "compress/gzip" + "encoding/base64" "encoding/json" "io" "os" + "strconv" "strings" "time" + "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/golibs/log" + "github.com/miekg/dns" ) const enableGzip = false @@ -145,13 +149,15 @@ func (l *queryLog) periodicRotate() { // Reader is the DB reader context type Reader struct { - ql *queryLog + ql *queryLog + search *getDataParams f *os.File reader *bufio.Reader // reads file line by line now time.Time validFrom int64 // UNIX time (ns) olderThan int64 // UNIX time (ns) + oldest time.Time files []string ifile int @@ -161,10 +167,12 @@ type Reader struct { latest bool // return the latest entries filePrepared bool - searching bool // we're seaching for an entry with exact time stamp + seeking bool // we're seaching for an entry with exact time stamp fseeker fileSeeker // file seeker object fpos uint64 // current file offset nSeekRequests uint32 // number of Seek() requests made (finding a new line doesn't count) + + timecnt uint64 } type fileSeeker struct { @@ -197,8 +205,8 @@ func (r *Reader) Close() { if r.count > 0 { perunit = elapsed / time.Duration(r.count) } - log.Debug("querylog: read %d entries in %v, %v/entry, seek-reqs:%d", - r.count, elapsed, perunit, r.nSeekRequests) + log.Debug("querylog: read %d entries in %v, %v/entry, seek-reqs:%d time:%dus (%d%%)", + r.count, elapsed, perunit, r.nSeekRequests, r.timecnt/1000, r.timecnt*100/uint64(elapsed.Nanoseconds())) if r.f != nil { r.f.Close() @@ -208,25 +216,26 @@ func (r *Reader) Close() { // BeginRead - start reading // olderThan: stop returning entries when an entry with this time is reached // count: minimum number of entries to return -func (r *Reader) BeginRead(olderThan time.Time, count uint64) { +func (r *Reader) BeginRead(olderThan time.Time, count uint64, search *getDataParams) { r.olderThan = olderThan.UnixNano() r.latest = olderThan.IsZero() + r.oldest = time.Time{} + r.search = search r.limit = count if r.latest { r.olderThan = r.now.UnixNano() } r.filePrepared = false - r.searching = false + r.seeking = false } // BeginReadPrev - start reading the previous data chunk -func (r *Reader) BeginReadPrev(olderThan time.Time, count uint64) { - r.olderThan = olderThan.UnixNano() - r.latest = olderThan.IsZero() +func (r *Reader) BeginReadPrev(count uint64) { + r.olderThan = r.oldest.UnixNano() + r.oldest = time.Time{} + r.latest = false r.limit = count - if r.latest { - r.olderThan = r.now.UnixNano() - } + r.count = 0 off := r.fpos - maxEntrySize*(r.limit+1) if int64(off) < maxEntrySize { @@ -245,7 +254,7 @@ func (r *Reader) BeginReadPrev(olderThan time.Time, count uint64) { r.fseeker.pos = r.fpos r.filePrepared = true - r.searching = false + r.seeking = false } // Perform binary seek @@ -335,7 +344,7 @@ func (r *Reader) prepareRead() bool { } } else { // start searching in file: we'll read the first chunk of data from the middle of file - r.searching = true + r.seeking = true r.fseeker = fileSeeker{} r.fseeker.target = uint64(r.olderThan) r.fseeker.hi = fsize @@ -358,6 +367,226 @@ func (r *Reader) prepareRead() bool { return true } +// Get bool value from "key":bool +func readJSONBool(s, name string) (bool, bool) { + i := strings.Index(s, "\""+name+"\":") + if i == -1 { + return false, false + } + start := i + 1 + len(name) + 2 + b := false + if strings.HasPrefix(s[start:], "true") { + b = true + } else if !strings.HasPrefix(s[start:], "false") { + return false, false + } + return b, true +} + +// Get value from "key":"value" +func readJSONValue(s, name string) string { + i := strings.Index(s, "\""+name+"\":\"") + if i == -1 { + return "" + } + start := i + 1 + len(name) + 3 + i = strings.IndexByte(s[start:], '"') + if i == -1 { + return "" + } + end := start + i + return s[start:end] +} + +func (r *Reader) applySearch(str string) bool { + if r.search.ResponseStatus == responseStatusFiltered { + boolVal, ok := readJSONBool(str, "IsFiltered") + if !ok || !boolVal { + return false + } + } + + if len(r.search.Domain) != 0 { + val := readJSONValue(str, "QH") + if len(val) == 0 { + return false + } + + if (r.search.StrictMatchDomain && val != r.search.Domain) || + (!r.search.StrictMatchDomain && strings.Index(val, r.search.Domain) == -1) { + return false + } + } + + if len(r.search.QuestionType) != 0 { + val := readJSONValue(str, "QT") + if len(val) == 0 { + return false + } + if val != r.search.QuestionType { + return false + } + } + + if len(r.search.Client) != 0 { + val := readJSONValue(str, "IP") + if len(val) == 0 { + log.Debug("QueryLog: failed to decode") + return false + } + + if (r.search.StrictMatchClient && val != r.search.Client) || + (!r.search.StrictMatchClient && strings.Index(val, r.search.Client) == -1) { + return false + } + } + + return true +} + +const ( + jsonTErr = iota + jsonTObj + jsonTStr + jsonTNum + jsonTBool +) + +// Parse JSON key-value pair +// e.g.: "key":VALUE where VALUE is "string", true|false (boolean), or 123.456 (number) +// Note the limitations: +// . doesn't support whitespace +// . doesn't support "null" +// . doesn't validate boolean or number +// . no proper handling of {} braces +// . no handling of [] brackets +// Return (key, value, type) +func readJSON(ps *string) (string, string, int32) { + s := *ps + k := "" + v := "" + t := int32(jsonTErr) + + q1 := strings.IndexByte(s, '"') + if q1 == -1 { + return k, v, t + } + q2 := strings.IndexByte(s[q1+1:], '"') + if q2 == -1 { + return k, v, t + } + k = s[q1+1 : q1+1+q2] + s = s[q1+1+q2+1:] + + if len(s) < 2 || s[0] != ':' { + return k, v, t + } + + if s[1] == '"' { + q2 = strings.IndexByte(s[2:], '"') + if q2 == -1 { + return k, v, t + } + v = s[2 : 2+q2] + t = jsonTStr + s = s[2+q2+1:] + + } else if s[1] == '{' { + t = jsonTObj + s = s[1+1:] + + } else { + sep := strings.IndexAny(s[1:], ",}") + if sep == -1 { + return k, v, t + } + v = s[1 : 1+sep] + if s[1] == 't' || s[1] == 'f' { + t = jsonTBool + } else if s[1] == '.' || (s[1] >= '0' && s[1] <= '9') { + t = jsonTNum + } + s = s[1+sep+1:] + } + + *ps = s + return k, v, t +} + +// nolint (gocyclo) +func decode(ent *logEntry, str string) { + var b bool + var i int + var err error + for { + k, v, t := readJSON(&str) + if t == jsonTErr { + break + } + switch k { + case "IP": + ent.IP = v + case "T": + ent.Time, err = time.Parse(time.RFC3339, v) + + case "QH": + ent.QHost = v + case "QT": + ent.QType = v + case "QC": + ent.QClass = v + + case "Answer": + ent.Answer, err = base64.StdEncoding.DecodeString(v) + + case "IsFiltered": + b, err = strconv.ParseBool(v) + ent.Result.IsFiltered = b + case "Rule": + ent.Result.Rule = v + case "FilterID": + i, err = strconv.Atoi(v) + ent.Result.FilterID = int64(i) + case "Reason": + i, err = strconv.Atoi(v) + ent.Result.Reason = dnsfilter.Reason(i) + + case "Upstream": + ent.Upstream = v + case "Elapsed": + i, err = strconv.Atoi(v) + ent.Elapsed = time.Duration(i) + + // pre-v0.99.3 compatibility: + case "Question": + var qstr []byte + qstr, err = base64.StdEncoding.DecodeString(v) + if err != nil { + break + } + q := new(dns.Msg) + err = q.Unpack(qstr) + if err != nil { + break + } + ent.QHost = q.Question[0].Name + if len(ent.QHost) == 0 { + break + } + ent.QHost = ent.QHost[:len(ent.QHost)-1] + ent.QType = dns.TypeToString[q.Question[0].Qtype] + ent.QClass = dns.ClassToString[q.Question[0].Qclass] + case "Time": + ent.Time, err = time.Parse(time.RFC3339, v) + } + + if err != nil { + log.Debug("decode err: %s", err) + break + } + } +} + // Next - return the next entry or nil if reading is finished func (r *Reader) Next() *logEntry { // nolint for { @@ -379,24 +608,28 @@ func (r *Reader) Next() *logEntry { // nolint r.filePrepared = true } - // open decoder b, err := r.reader.ReadBytes('\n') if err != nil { return nil } - strReader := strings.NewReader(string(b)) - jd := json.NewDecoder(strReader) + str := string(b) - // read data - var entry logEntry - err = jd.Decode(&entry) - if err != nil { - log.Debug("QueryLog: Failed to decode: %s", err) + val := readJSONValue(str, "T") + if len(val) == 0 { + val = readJSONValue(str, "Time") + } + if len(val) == 0 { + log.Debug("QueryLog: failed to decode") continue } + tm, err := time.Parse(time.RFC3339, val) + if err != nil { + log.Debug("QueryLog: failed to decode") + continue + } + t := tm.UnixNano() - t := entry.Time.UnixNano() - if r.searching { + if r.seeking { r.reader = nil rr := r.fseeker.seekBinary(uint64(t)) @@ -407,7 +640,7 @@ func (r *Reader) Next() *logEntry { // nolint } else if rr == 0 { // We found the target entry. // We'll start reading the previous chunk of data. - r.searching = false + r.seeking = false off := r.fpos - (maxEntrySize * (r.limit + 1)) if int64(off) < maxEntrySize { @@ -430,19 +663,37 @@ func (r *Reader) Next() *logEntry { // nolint continue } + if r.oldest.IsZero() { + r.oldest = tm + } + if t < r.validFrom { continue } if t >= r.olderThan { return nil } - r.count++ - return &entry + + if !r.applySearch(str) { + continue + } + + st := time.Now() + var ent logEntry + decode(&ent, str) + r.timecnt += uint64(time.Now().Sub(st).Nanoseconds()) + + return &ent } } -// Total returns the total number of items -func (r *Reader) Total() int { - return 0 +// Total returns the total number of processed items +func (r *Reader) Total() uint64 { + return r.count +} + +// Oldest returns the time of the oldest processed entry +func (r *Reader) Oldest() time.Time { + return r.oldest } diff --git a/querylog/querylog_test.go b/querylog/querylog_test.go index e300cba1..91dc405b 100644 --- a/querylog/querylog_test.go +++ b/querylog/querylog_test.go @@ -42,7 +42,35 @@ func TestQueryLog(t *testing.T) { OlderThan: time.Now(), } d := l.getData(params) - m := d[0] + mdata := d["data"].([]map[string]interface{}) + m := mdata[0] mq := m["question"].(map[string]interface{}) assert.True(t, mq["host"].(string) == "example.org") } + +func TestJSON(t *testing.T) { + s := ` + {"keystr":"val","obj":{"keybool":true,"keyint":123456}} + ` + k, v, jtype := readJSON(&s) + assert.Equal(t, jtype, int32(jsonTStr)) + assert.Equal(t, "keystr", k) + assert.Equal(t, "val", v) + + k, v, jtype = readJSON(&s) + assert.Equal(t, jtype, int32(jsonTObj)) + assert.Equal(t, "obj", k) + + k, v, jtype = readJSON(&s) + assert.Equal(t, jtype, int32(jsonTBool)) + assert.Equal(t, "keybool", k) + assert.Equal(t, "true", v) + + k, v, jtype = readJSON(&s) + assert.Equal(t, jtype, int32(jsonTNum)) + assert.Equal(t, "keyint", k) + assert.Equal(t, "123456", v) + + k, v, jtype = readJSON(&s) + assert.True(t, jtype == jsonTErr) +} From 0cd6781a9a94bbb0cf1d0bd310fc50d665da4674 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Tue, 19 Nov 2019 14:09:10 +0300 Subject: [PATCH 2/8] * QueryLog.Add() now receives net.IP, not net.Addr --- dnsforward/dnsforward.go | 13 ++++++++++++- querylog/qlog.go | 16 ++-------------- querylog/querylog.go | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index 3ecd80d5..cb86c235 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -462,7 +462,7 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error { if d.Upstream != nil { upstreamAddr = d.Upstream.Address() } - s.queryLog.Add(msg, d.Res, res, elapsed, d.Addr, upstreamAddr) + s.queryLog.Add(msg, d.Res, res, elapsed, getIP(d.Addr), upstreamAddr) } s.updateStats(d, elapsed, *res) @@ -471,6 +471,17 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error { return nil } +// Get IP address from net.Addr +func getIP(addr net.Addr) net.IP { + switch addr := addr.(type) { + case *net.UDPAddr: + return addr.IP + case *net.TCPAddr: + return addr.IP + } + return nil +} + func (s *Server) updateStats(d *proxy.DNSContext, elapsed time.Duration, res dnsfilter.Result) { if s.stats == nil { return diff --git a/querylog/qlog.go b/querylog/qlog.go index e53d685c..151b16ff 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -107,18 +107,7 @@ type logEntry struct { Upstream string `json:",omitempty"` // if empty, means it was cached } -// getIPString is a helper function that extracts IP address from net.Addr -func getIPString(addr net.Addr) string { - switch addr := addr.(type) { - case *net.UDPAddr: - return addr.IP.String() - case *net.TCPAddr: - return addr.IP.String() - } - return "" -} - -func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) { +func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, ip net.IP, upstream string) { if !l.conf.Enabled { return } @@ -130,7 +119,6 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res var a []byte var err error - ip := getIPString(addr) if answer != nil { a, err = answer.Pack() @@ -146,7 +134,7 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res now := time.Now() entry := logEntry{ - IP: ip, + IP: ip.String(), Time: now, Answer: a, diff --git a/querylog/querylog.go b/querylog/querylog.go index 26bd55a0..5158a211 100644 --- a/querylog/querylog.go +++ b/querylog/querylog.go @@ -21,7 +21,7 @@ type QueryLog interface { Close() // Add a log entry - Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) + Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, ip net.IP, upstream string) // WriteDiskConfig - write configuration WriteDiskConfig(dc *DiskConfig) From 68cd7976b72336d15098c1d73eb72f858b23f8be Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Tue, 19 Nov 2019 14:09:40 +0300 Subject: [PATCH 3/8] * querylog: add more tests --- querylog/qlog.go | 4 +- querylog/querylog_test.go | 105 ++++++++++++++++++++++++++++++++++---- 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/querylog/qlog.go b/querylog/qlog.go index 151b16ff..713a2849 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -373,9 +373,9 @@ func answerToMap(a *dns.Msg) []map[string]interface{} { // try most common record types switch v := k.(type) { case *dns.A: - answer["value"] = v.A + answer["value"] = v.A.String() case *dns.AAAA: - answer["value"] = v.AAAA + answer["value"] = v.AAAA.String() case *dns.MX: answer["value"] = fmt.Sprintf("%v %v", v.Preference, v.Mx) case *dns.CNAME: diff --git a/querylog/querylog_test.go b/querylog/querylog_test.go index 91dc405b..63bcbfd8 100644 --- a/querylog/querylog_test.go +++ b/querylog/querylog_test.go @@ -2,6 +2,7 @@ package querylog import ( "net" + "os" "testing" "time" @@ -10,16 +11,94 @@ import ( "github.com/stretchr/testify/assert" ) +func prepareTestDir() string { + const dir = "./agh-test" + _ = os.RemoveAll(dir) + _ = os.MkdirAll(dir, 0755) + return dir +} + +// Check adding and loading (with filtering) entries from disk and memory func TestQueryLog(t *testing.T) { conf := Config{ Enabled: true, Interval: 1, } + conf.BaseDir = prepareTestDir() + defer func() { _ = os.RemoveAll(conf.BaseDir) }() l := newQueryLog(conf) + // add disk entries + addEntry(l, "example.org", "1.2.3.4", "0.1.2.3") + addEntry(l, "example.org", "1.2.3.4", "0.1.2.3") + + // write to disk + l.flushLogBuffer(true) + + // add memory entries + addEntry(l, "test.example.org", "2.2.3.4", "0.1.2.4") + + // get all entries + params := getDataParams{ + OlderThan: time.Time{}, + } + d := l.getData(params) + mdata := d["data"].([]map[string]interface{}) + assert.True(t, len(mdata) == 2) + assert.True(t, checkEntry(t, mdata[0], "test.example.org", "2.2.3.4", "0.1.2.4")) + assert.True(t, checkEntry(t, mdata[1], "example.org", "1.2.3.4", "0.1.2.3")) + + // search by domain (strict) + params = getDataParams{ + OlderThan: time.Time{}, + Domain: "test.example.org", + StrictMatchDomain: true, + } + d = l.getData(params) + mdata = d["data"].([]map[string]interface{}) + assert.True(t, len(mdata) == 1) + assert.True(t, checkEntry(t, mdata[0], "test.example.org", "2.2.3.4", "0.1.2.4")) + + // search by domain + params = getDataParams{ + OlderThan: time.Time{}, + Domain: "example.org", + StrictMatchDomain: false, + } + d = l.getData(params) + mdata = d["data"].([]map[string]interface{}) + assert.True(t, len(mdata) == 2) + assert.True(t, checkEntry(t, mdata[0], "test.example.org", "2.2.3.4", "0.1.2.4")) + assert.True(t, checkEntry(t, mdata[1], "example.org", "1.2.3.4", "0.1.2.3")) + + // search by client IP (strict) + params = getDataParams{ + OlderThan: time.Time{}, + Client: "0.1.2.3", + StrictMatchClient: true, + } + d = l.getData(params) + mdata = d["data"].([]map[string]interface{}) + assert.True(t, len(mdata) == 1) + assert.True(t, checkEntry(t, mdata[0], "example.org", "1.2.3.4", "0.1.2.3")) + + // search by client IP + params = getDataParams{ + OlderThan: time.Time{}, + Client: "0.1.2", + StrictMatchClient: false, + } + d = l.getData(params) + mdata = d["data"].([]map[string]interface{}) + assert.True(t, len(mdata) == 2) + assert.True(t, checkEntry(t, mdata[0], "test.example.org", "2.2.3.4", "0.1.2.4")) + assert.True(t, checkEntry(t, mdata[1], "example.org", "1.2.3.4", "0.1.2.3")) +} + +func addEntry(l *queryLog, host, answerStr, client string) { q := dns.Msg{} q.Question = append(q.Question, dns.Question{ - Name: "example.org.", + Name: host + ".", Qtype: dns.TypeA, Qclass: dns.ClassINET, }) @@ -32,20 +111,24 @@ func TestQueryLog(t *testing.T) { Rrtype: dns.TypeA, Class: dns.ClassINET, } - answer.A = net.IP{1, 2, 3, 4} + answer.A = net.ParseIP(answerStr) a.Answer = append(a.Answer, answer) - res := dnsfilter.Result{} - l.Add(&q, &a, &res, 0, nil, "upstream") + l.Add(&q, &a, &res, 0, net.ParseIP(client), "upstream") +} - params := getDataParams{ - OlderThan: time.Now(), - } - d := l.getData(params) - mdata := d["data"].([]map[string]interface{}) - m := mdata[0] +func checkEntry(t *testing.T, m map[string]interface{}, host, answer, client string) bool { mq := m["question"].(map[string]interface{}) - assert.True(t, mq["host"].(string) == "example.org") + ma := m["answer"].([]map[string]interface{}) + ma0 := ma[0] + if !assert.True(t, mq["host"].(string) == host) || + !assert.True(t, mq["class"].(string) == "IN") || + !assert.True(t, mq["type"].(string) == "A") || + !assert.True(t, ma0["value"].(string) == answer) || + !assert.True(t, m["client"].(string) == client) { + return false + } + return true } func TestJSON(t *testing.T) { From 33093de6aa30eae902e82cb556960a36aa1bd1f1 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Thu, 14 Nov 2019 15:18:43 +0300 Subject: [PATCH 4/8] * openapi: update 'QueryLog' --- openapi/CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ openapi/openapi.yaml | 12 +++++++++--- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index 281d58e5..3172be07 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -1,6 +1,46 @@ # AdGuard Home API Change Log +## v0.99.3: API changes + +### API: Get query log: GET /control/querylog + +The response data is now a JSON object, not an array. + +Response: + + 200 OK + + { + "oldest":"2006-01-02T15:04:05.999999999Z07:00" + "data":[ + { + "answer":[ + { + "ttl":10, + "type":"AAAA", + "value":"::" + } + ... + ], + "client":"127.0.0.1", + "elapsedMs":"0.098403", + "filterId":1, + "question":{ + "class":"IN", + "host":"doubleclick.net", + "type":"AAAA" + }, + "reason":"FilteredBlackList", + "rule":"||doubleclick.net^", + "status":"NOERROR", + "time":"2006-01-02T15:04:05.999999999Z07:00" + } + ... + ] + } + + ## v0.99.1: API changes ### API: Get current user info: GET /control/profile diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index eca972b3..d871fef3 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1417,10 +1417,16 @@ definitions: example: "2018-11-26T00:02:41+03:00" QueryLog: - type: "array" + type: "object" description: "Query log" - items: - $ref: "#/definitions/QueryLogItem" + properties: + oldest: + type: "string" + example: "2018-11-26T00:02:41+03:00" + data: + type: "array" + items: + $ref: "#/definitions/QueryLogItem" QueryLogConfig: type: "object" From 941b6c5976d8b9aad57e054b2ef5d69e210004e9 Mon Sep 17 00:00:00 2001 From: Ildar Kamalov Date: Wed, 13 Nov 2019 19:09:40 +0300 Subject: [PATCH 5/8] + client: use oldest param --- client/src/actions/queryLogs.js | 10 +++++++--- client/src/components/Logs/index.js | 14 ++++++-------- client/src/reducers/queryLogs.js | 7 ++++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index 09e93a60..29f5bc45 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -14,9 +14,13 @@ export const getLogsSuccess = createAction('GET_LOGS_SUCCESS'); export const getLogs = config => async (dispatch) => { dispatch(getLogsRequest()); try { - const { filter, lastRowTime: older_than } = config; - const logs = normalizeLogs(await apiClient.getQueryLog({ ...filter, older_than })); - dispatch(getLogsSuccess({ logs, ...config })); + const { filter, older_than } = config; + const rawLogs = await apiClient.getQueryLog({ ...filter, older_than }); + const { data, oldest } = rawLogs; + const logs = normalizeLogs(data); + dispatch(getLogsSuccess({ + logs, oldest, filter, ...config, + })); } catch (error) { dispatch(addErrorToast({ error })); dispatch(getLogsFailure(error)); diff --git a/client/src/components/Logs/index.js b/client/src/components/Logs/index.js index e61b19d1..fbb2e7df 100644 --- a/client/src/components/Logs/index.js +++ b/client/src/components/Logs/index.js @@ -41,10 +41,10 @@ class Logs extends Component { this.props.getLogsConfig(); } - getLogs = (lastRowTime, filter, page, pageSize, filtered) => { + getLogs = (older_than, filter, page, pageSize, filtered) => { if (this.props.queryLogs.enabled) { this.props.getLogs({ - lastRowTime, filter, page, pageSize, filtered, + older_than, filter, page, pageSize, filtered, }); } }; @@ -53,9 +53,9 @@ class Logs extends Component { window.location.reload(); }; - handleLogsFiltering = debounce((lastRowTime, filter, page, pageSize, filtered) => { + handleLogsFiltering = debounce((older_than, filter, page, pageSize, filtered) => { this.props.getLogs({ - lastRowTime, + older_than, filter, page, pageSize, @@ -264,13 +264,11 @@ class Logs extends Component { fetchData = (state) => { const { pageSize, page, pages } = state; - const { allLogs, filter } = this.props.queryLogs; + const { filter, oldest } = this.props.queryLogs; const isLastPage = pages && (page + 1 === pages); if (isLastPage) { - const lastRow = allLogs[allLogs.length - 1]; - const lastRowTime = (lastRow && lastRow.time) || ''; - this.getLogs(lastRowTime, filter, page, pageSize, true); + this.getLogs(oldest, filter, page, pageSize, false); } else { this.props.setLogsPagination({ page, pageSize }); } diff --git a/client/src/reducers/queryLogs.js b/client/src/reducers/queryLogs.js index 2f04099b..ff59426f 100644 --- a/client/src/reducers/queryLogs.js +++ b/client/src/reducers/queryLogs.js @@ -28,12 +28,12 @@ const queryLogs = handleActions( [actions.getLogsFailure]: state => ({ ...state, processingGetLogs: false }), [actions.getLogsSuccess]: (state, { payload }) => { const { - logs, lastRowTime, page, pageSize, filtered, + logs, oldest, older_than, page, pageSize, filtered, } = payload; let logsWithOffset = state.allLogs.length > 0 ? state.allLogs : logs; let allLogs = logs; - if (lastRowTime) { + if (older_than) { logsWithOffset = [...state.allLogs, ...logs]; allLogs = [...state.allLogs, ...logs]; } else if (filtered) { @@ -49,6 +49,7 @@ const queryLogs = handleActions( return { ...state, + oldest, pages, total, allLogs, @@ -93,7 +94,7 @@ const queryLogs = handleActions( pages: 0, total: 0, enabled: true, - older_than: '', + oldest: '', filter: DEFAULT_LOGS_FILTER, }, ); From 6b64d393bdcf89d5d9aa1a83b062a46b74caccef Mon Sep 17 00:00:00 2001 From: Ildar Kamalov Date: Wed, 13 Nov 2019 19:20:22 +0300 Subject: [PATCH 6/8] + client: hide page size option and page info --- client/src/components/Logs/index.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/src/components/Logs/index.js b/client/src/components/Logs/index.js index fbb2e7df..421f423e 100644 --- a/client/src/components/Logs/index.js +++ b/client/src/components/Logs/index.js @@ -25,7 +25,7 @@ import Tooltip from '../ui/Tooltip'; import './Logs.css'; const TABLE_FIRST_PAGE = 0; -const TABLE_DEFAULT_PAGE_SIZE = 50; +const TABLE_DEFAULT_PAGE_SIZE = 100; const INITIAL_REQUEST_DATA = ['', DEFAULT_LOGS_FILTER, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; const FILTERED_REASON = 'Filtered'; const RESPONSE_FILTER = { @@ -367,20 +367,22 @@ class Logs extends Component { sortable={false} data={logs || []} loading={isLoading} + showPagination={true} showPageJump={false} + showPageSizeOptions={false} onFetchData={this.fetchData} onFilteredChange={this.handleFilterChange} className="logs__table" - showPagination={true} defaultPageSize={TABLE_DEFAULT_PAGE_SIZE} previousText={t('previous_btn')} nextText={t('next_btn')} loadingText={t('loading_table_status')} - pageText={t('page_table_footer_text')} - ofText={t('of_table_footer_text')} rowsText={t('rows_table_footer_text')} noDataText={t('no_logs_found')} - renderTotalPagesCount={this.showTotalPagesCount} + pageText={''} + ofText={''} + renderCurrentPage={() => false} + renderTotalPagesCount={() => false} defaultFilterMethod={(filter, row) => { const id = filter.pivotId || filter.id; return row[id] !== undefined From e243e69a6e9b2fae677ad4b3647fb1074675c95c Mon Sep 17 00:00:00 2001 From: Ildar Kamalov Date: Fri, 15 Nov 2019 10:51:45 +0300 Subject: [PATCH 7/8] + client: separate filters from the table component --- client/src/actions/queryLogs.js | 38 +++++-- client/src/components/Logs/Filters/Form.js | 116 ++++++++++++++++++++ client/src/components/Logs/Filters/index.js | 45 ++++++++ client/src/components/Logs/index.js | 106 +++++------------- client/src/components/ui/Tooltip.css | 1 + client/src/containers/Logs.js | 3 +- client/src/helpers/constants.js | 7 ++ client/src/reducers/queryLogs.js | 41 +++++-- 8 files changed, 263 insertions(+), 94 deletions(-) create mode 100644 client/src/components/Logs/Filters/Form.js create mode 100644 client/src/components/Logs/Filters/index.js diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index 29f5bc45..ce6b5a0c 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -4,8 +4,19 @@ import apiClient from '../api/Api'; import { addErrorToast, addSuccessToast } from './index'; import { normalizeLogs } from '../helpers/helpers'; +const getLogsWithParams = async (config) => { + const { older_than, filter, ...values } = config; + const rawLogs = await apiClient.getQueryLog({ ...filter, older_than }); + const { data, oldest } = rawLogs; + const logs = normalizeLogs(data); + + return { + logs, oldest, older_than, filter, ...values, + }; +}; + export const setLogsPagination = createAction('LOGS_PAGINATION'); -export const setLogsFilter = createAction('LOGS_FILTER'); +export const setLogsPage = createAction('SET_LOG_PAGE'); export const getLogsRequest = createAction('GET_LOGS_REQUEST'); export const getLogsFailure = createAction('GET_LOGS_FAILURE'); @@ -14,19 +25,30 @@ export const getLogsSuccess = createAction('GET_LOGS_SUCCESS'); export const getLogs = config => async (dispatch) => { dispatch(getLogsRequest()); try { - const { filter, older_than } = config; - const rawLogs = await apiClient.getQueryLog({ ...filter, older_than }); - const { data, oldest } = rawLogs; - const logs = normalizeLogs(data); - dispatch(getLogsSuccess({ - logs, oldest, filter, ...config, - })); + const logs = await getLogsWithParams(config); + dispatch(getLogsSuccess(logs)); } catch (error) { dispatch(addErrorToast({ error })); dispatch(getLogsFailure(error)); } }; +export const setLogsFilterRequest = createAction('SET_LOGS_FILTER_REQUEST'); +export const setLogsFilterFailure = createAction('SET_LOGS_FILTER_FAILURE'); +export const setLogsFilterSuccess = createAction('SET_LOGS_FILTER_SUCCESS'); + +export const setLogsFilter = filter => async (dispatch) => { + dispatch(setLogsFilterRequest()); + try { + const logs = await getLogsWithParams({ older_than: '', filter }); + dispatch(setLogsFilterSuccess(logs)); + dispatch(setLogsPage(0)); + } catch (error) { + dispatch(addErrorToast({ error })); + dispatch(setLogsFilterFailure(error)); + } +}; + export const clearLogsRequest = createAction('CLEAR_LOGS_REQUEST'); export const clearLogsFailure = createAction('CLEAR_LOGS_FAILURE'); export const clearLogsSuccess = createAction('CLEAR_LOGS_SUCCESS'); diff --git a/client/src/components/Logs/Filters/Form.js b/client/src/components/Logs/Filters/Form.js new file mode 100644 index 00000000..9b175fc9 --- /dev/null +++ b/client/src/components/Logs/Filters/Form.js @@ -0,0 +1,116 @@ +import React, { Fragment } from 'react'; +import PropTypes from 'prop-types'; +import { Field, reduxForm } from 'redux-form'; +import { withNamespaces, Trans } from 'react-i18next'; +import flow from 'lodash/flow'; + +import { renderField } from '../../../helpers/form'; +import { RESPONSE_FILTER } from '../../../helpers/constants'; +import Tooltip from '../../ui/Tooltip'; + +const renderFilterField = ({ + input, + id, + className, + placeholder, + type, + disabled, + autoComplete, + tooltip, + meta: { touched, error }, +}) => ( + +
+ + + + + {!disabled && + touched && + (error && {error})} +
+ +); + +const Form = (props) => { + const { + t, + handleChange, + } = props; + + return ( +
+
+
+ +
+
+ +
+
+ + + + +
+
+ +
+
+
+ ); +}; + +Form.propTypes = { + handleChange: PropTypes.func, + t: PropTypes.func.isRequired, +}; + +export default flow([ + withNamespaces(), + reduxForm({ + form: 'logsFilterForm', + }), +])(Form); diff --git a/client/src/components/Logs/Filters/index.js b/client/src/components/Logs/Filters/index.js new file mode 100644 index 00000000..20102397 --- /dev/null +++ b/client/src/components/Logs/Filters/index.js @@ -0,0 +1,45 @@ +import React, { Component } from 'react'; +import PropTypes from 'prop-types'; +import debounce from 'lodash/debounce'; + +import { DEBOUNCE_FILTER_TIMEOUT, RESPONSE_FILTER } from '../../../helpers/constants'; +import { isValidQuestionType } from '../../../helpers/helpers'; +import Form from './Form'; + +class Filters extends Component { + getFilters = (filtered) => { + const { + domain, client, type, response, + } = filtered; + + return { + filter_domain: domain || '', + filter_client: client || '', + filter_question_type: isValidQuestionType(type) ? type.toUpperCase() : '', + filter_response_status: response === RESPONSE_FILTER.FILTERED ? response : '', + }; + }; + + handleFormChange = debounce((values) => { + const filter = this.getFilters(values); + this.props.setLogsFilter(filter); + }, DEBOUNCE_FILTER_TIMEOUT); + + render() { + const { filter } = this.props; + + return ( +
+ ); + } +} + +Filters.propTypes = { + filter: PropTypes.object.isRequired, + setLogsFilter: PropTypes.func.isRequired, +}; + +export default Filters; diff --git a/client/src/components/Logs/index.js b/client/src/components/Logs/index.js index 421f423e..d1604525 100644 --- a/client/src/components/Logs/index.js +++ b/client/src/components/Logs/index.js @@ -5,46 +5,40 @@ import escapeRegExp from 'lodash/escapeRegExp'; import endsWith from 'lodash/endsWith'; import { Trans, withNamespaces } from 'react-i18next'; import { HashLink as Link } from 'react-router-hash-link'; -import debounce from 'lodash/debounce'; import { formatTime, formatDateTime, - isValidQuestionType, } from '../../helpers/helpers'; -import { SERVICES, FILTERED_STATUS, DEBOUNCE_TIMEOUT, DEFAULT_LOGS_FILTER } from '../../helpers/constants'; +import { SERVICES, FILTERED_STATUS, DEFAULT_LOGS_FILTER, RESPONSE_FILTER, TABLE_DEFAULT_PAGE_SIZE } from '../../helpers/constants'; import { getTrackerData } from '../../helpers/trackers/trackers'; import { formatClientCell } from '../../helpers/formatClientCell'; +import Filters from './Filters'; import PageTitle from '../ui/PageTitle'; import Card from '../ui/Card'; import Loading from '../ui/Loading'; import PopoverFiltered from '../ui/PopoverFilter'; import Popover from '../ui/Popover'; -import Tooltip from '../ui/Tooltip'; import './Logs.css'; const TABLE_FIRST_PAGE = 0; -const TABLE_DEFAULT_PAGE_SIZE = 100; const INITIAL_REQUEST_DATA = ['', DEFAULT_LOGS_FILTER, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; const FILTERED_REASON = 'Filtered'; -const RESPONSE_FILTER = { - ALL: 'all', - FILTERED: 'filtered', -}; class Logs extends Component { componentDidMount() { + this.props.setLogsPage(TABLE_FIRST_PAGE); this.getLogs(...INITIAL_REQUEST_DATA); this.props.getFilteringStatus(); this.props.getClients(); this.props.getLogsConfig(); } - getLogs = (older_than, filter, page, pageSize, filtered) => { + getLogs = (older_than, filter, page) => { if (this.props.queryLogs.enabled) { this.props.getLogs({ - older_than, filter, page, pageSize, filtered, + older_than, filter, page, pageSize: TABLE_DEFAULT_PAGE_SIZE, }); } }; @@ -53,16 +47,6 @@ class Logs extends Component { window.location.reload(); }; - handleLogsFiltering = debounce((older_than, filter, page, pageSize, filtered) => { - this.props.getLogs({ - older_than, - filter, - page, - pageSize, - filtered, - }); - }, DEBOUNCE_TIMEOUT); - renderTooltip = (isFiltered, rule, filter, service) => isFiltered && ; @@ -232,70 +216,25 @@ class Logs extends Component { ); }; - getFilterInput = ({ filter, onChange }) => ( - -
- onChange(event.target.value)} - value={filter ? filter.value : ''} - /> - - - -
- - ); - - getFilters = (filtered) => { - const filteredObj = filtered.reduce((acc, cur) => ({ ...acc, [cur.id]: cur.value }), {}); - const { - domain, client, type, response, - } = filteredObj; - - return { - filter_domain: domain || '', - filter_client: client || '', - filter_question_type: isValidQuestionType(type) ? type.toUpperCase() : '', - filter_response_status: response === RESPONSE_FILTER.FILTERED ? response : '', - }; - }; - fetchData = (state) => { - const { pageSize, page, pages } = state; - const { filter, oldest } = this.props.queryLogs; + const { pages } = state; + const { + filter, oldest, page, + } = this.props.queryLogs; const isLastPage = pages && (page + 1 === pages); if (isLastPage) { - this.getLogs(oldest, filter, page, pageSize, false); + this.getLogs(oldest, filter, page); } else { - this.props.setLogsPagination({ page, pageSize }); + this.props.setLogsPagination({ page, pageSize: TABLE_DEFAULT_PAGE_SIZE }); } }; - handleFilterChange = (filtered) => { - const filters = this.getFilters(filtered); - this.props.setLogsFilter(filters); - this.handleLogsFiltering('', filters, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE, true); - } - - showTotalPagesCount = (pages) => { - const { total, isEntireLog } = this.props.queryLogs; - const showEllipsis = !isEntireLog && total >= 500; - - return ( - - {pages || 1}{showEllipsis && '…' } - - ); - } - renderLogs() { const { queryLogs, dashboard, t } = this.props; const { processingClients } = dashboard; const { - processingGetLogs, processingGetConfig, logs, pages, + processingGetLogs, processingGetConfig, logs, pages, page, } = queryLogs; const isLoading = processingGetLogs || processingClients || processingGetConfig; @@ -304,7 +243,6 @@ class Logs extends Component { Header: t('time_table_header'), accessor: 'time', maxWidth: 100, - filterable: false, Cell: this.getTimeCell, }, { @@ -360,18 +298,20 @@ class Logs extends Component { return ( this.props.setLogsPage(newPage)} className="logs__table" defaultPageSize={TABLE_DEFAULT_PAGE_SIZE} previousText={t('previous_btn')} @@ -381,7 +321,6 @@ class Logs extends Component { noDataText={t('no_logs_found')} pageText={''} ofText={''} - renderCurrentPage={() => false} renderTotalPagesCount={() => false} defaultFilterMethod={(filter, row) => { const id = filter.pivotId || filter.id; @@ -446,7 +385,17 @@ class Logs extends Component { {refreshButton} {enabled && processingGetConfig && } - {enabled && !processingGetConfig && {this.renderLogs()}} + {enabled && !processingGetConfig && ( + + + + + {this.renderLogs()} + + )} {!enabled && !processingGetConfig && (
@@ -479,6 +428,7 @@ Logs.propTypes = { getLogsConfig: PropTypes.func.isRequired, setLogsPagination: PropTypes.func.isRequired, setLogsFilter: PropTypes.func.isRequired, + setLogsPage: PropTypes.func.isRequired, t: PropTypes.func.isRequired, }; diff --git a/client/src/components/ui/Tooltip.css b/client/src/components/ui/Tooltip.css index 27505a68..9ad8af3b 100644 --- a/client/src/components/ui/Tooltip.css +++ b/client/src/components/ui/Tooltip.css @@ -64,6 +64,7 @@ top: calc(100% + 10px); right: -10px; left: initial; + width: 255px; transform: none; } diff --git a/client/src/containers/Logs.js b/client/src/containers/Logs.js index 3b93ec94..be328fbd 100644 --- a/client/src/containers/Logs.js +++ b/client/src/containers/Logs.js @@ -1,7 +1,7 @@ import { connect } from 'react-redux'; import { addSuccessToast, getClients } from '../actions'; import { getFilteringStatus, setRules } from '../actions/filtering'; -import { getLogs, getLogsConfig, setLogsPagination, setLogsFilter } from '../actions/queryLogs'; +import { getLogs, getLogsConfig, setLogsPagination, setLogsFilter, setLogsPage } from '../actions/queryLogs'; import Logs from '../components/Logs'; const mapStateToProps = (state) => { @@ -19,6 +19,7 @@ const mapDispatchToProps = { getLogsConfig, setLogsPagination, setLogsFilter, + setLogsPage, }; export default connect( diff --git a/client/src/helpers/constants.js b/client/src/helpers/constants.js index fb82c1c9..ca19818e 100644 --- a/client/src/helpers/constants.js +++ b/client/src/helpers/constants.js @@ -141,6 +141,7 @@ export const STANDARD_HTTPS_PORT = 443; export const EMPTY_DATE = '0001-01-01T00:00:00Z'; export const DEBOUNCE_TIMEOUT = 300; +export const DEBOUNCE_FILTER_TIMEOUT = 500; export const CHECK_TIMEOUT = 1000; export const STOP_TIMEOUT = 10000; @@ -379,3 +380,9 @@ export const DEFAULT_LOGS_FILTER = { }; export const DEFAULT_LANGUAGE = 'en'; + +export const TABLE_DEFAULT_PAGE_SIZE = 100; +export const RESPONSE_FILTER = { + ALL: 'all', + FILTERED: 'filtered', +}; diff --git a/client/src/reducers/queryLogs.js b/client/src/reducers/queryLogs.js index ff59426f..4f883353 100644 --- a/client/src/reducers/queryLogs.js +++ b/client/src/reducers/queryLogs.js @@ -20,15 +20,43 @@ const queryLogs = handleActions( }; }, - [actions.setLogsFilter]: (state, { payload }) => ( - { ...state, filter: payload } - ), + [actions.setLogsPage]: (state, { payload }) => ({ + ...state, + page: payload, + }), + + [actions.setLogsFilterRequest]: state => ({ ...state, processingGetLogs: true }), + [actions.setLogsFilterFailure]: state => ({ ...state, processingGetLogs: false }), + [actions.setLogsFilterSuccess]: (state, { payload }) => { + const { logs, oldest, filter } = payload; + const pageSize = 100; + const page = 0; + + const pages = Math.ceil(logs.length / pageSize); + const total = logs.length; + const rowsStart = pageSize * page; + const rowsEnd = (pageSize * page) + pageSize; + const logsSlice = logs.slice(rowsStart, rowsEnd); + const isFiltered = Object.keys(filter).some(key => filter[key]); + + return { + ...state, + oldest, + filter, + isFiltered, + pages, + total, + logs: logsSlice, + allLogs: logs, + processingGetLogs: false, + }; + }, [actions.getLogsRequest]: state => ({ ...state, processingGetLogs: true }), [actions.getLogsFailure]: state => ({ ...state, processingGetLogs: false }), [actions.getLogsSuccess]: (state, { payload }) => { const { - logs, oldest, older_than, page, pageSize, filtered, + logs, oldest, older_than, page, pageSize, } = payload; let logsWithOffset = state.allLogs.length > 0 ? state.allLogs : logs; let allLogs = logs; @@ -36,9 +64,6 @@ const queryLogs = handleActions( if (older_than) { logsWithOffset = [...state.allLogs, ...logs]; allLogs = [...state.allLogs, ...logs]; - } else if (filtered) { - logsWithOffset = logs; - allLogs = logs; } const pages = Math.ceil(logsWithOffset.length / pageSize); @@ -91,11 +116,13 @@ const queryLogs = handleActions( logs: [], interval: 1, allLogs: [], + page: 0, pages: 0, total: 0, enabled: true, oldest: '', filter: DEFAULT_LOGS_FILTER, + isFiltered: false, }, ); From 62c8664fd75439b7597d935992c380f9c0675660 Mon Sep 17 00:00:00 2001 From: Ildar Kamalov Date: Mon, 18 Nov 2019 14:51:09 +0300 Subject: [PATCH 8/8] + client: load additional search results --- client/src/__locales/en.json | 2 +- client/src/actions/queryLogs.js | 58 +++++++++++++++++-- client/src/components/Logs/Filters/Form.js | 22 +++---- client/src/components/Logs/Filters/index.js | 41 ++++++++------ client/src/components/Logs/index.js | 63 +++++++-------------- client/src/components/ui/Card.css | 30 ++++++++++ client/src/reducers/queryLogs.js | 11 ++++ 7 files changed, 151 insertions(+), 76 deletions(-) diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 53f269a2..4365b6ec 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -401,4 +401,4 @@ "descr": "Description", "whois": "Whois", "filtering_rules_learn_more": "<0>Learn more about creating your own hosts blocklists." -} \ No newline at end of file +} diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index ce6b5a0c..35b4f7af 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -3,6 +3,7 @@ import { createAction } from 'redux-actions'; import apiClient from '../api/Api'; import { addErrorToast, addSuccessToast } from './index'; import { normalizeLogs } from '../helpers/helpers'; +import { TABLE_DEFAULT_PAGE_SIZE } from '../helpers/constants'; const getLogsWithParams = async (config) => { const { older_than, filter, ...values } = config; @@ -15,6 +16,41 @@ const getLogsWithParams = async (config) => { }; }; +export const getAdditionalLogsRequest = createAction('GET_ADDITIONAL_LOGS_REQUEST'); +export const getAdditionalLogsFailure = createAction('GET_ADDITIONAL_LOGS_FAILURE'); +export const getAdditionalLogsSuccess = createAction('GET_ADDITIONAL_LOGS_SUCCESS'); + +const checkFilteredLogs = async (data, filter, dispatch, total) => { + const { logs, oldest } = data; + const totalData = total || { logs }; + + const needToGetAdditionalLogs = (logs.length < TABLE_DEFAULT_PAGE_SIZE || + totalData.logs.length < TABLE_DEFAULT_PAGE_SIZE) && + oldest !== ''; + + if (needToGetAdditionalLogs) { + dispatch(getAdditionalLogsRequest()); + + try { + const additionalLogs = await getLogsWithParams({ older_than: oldest, filter }); + if (additionalLogs.logs.length > 0) { + return await checkFilteredLogs(additionalLogs, filter, dispatch, { + logs: [...totalData.logs, ...additionalLogs.logs], + oldest: additionalLogs.oldest, + }); + } + dispatch(getAdditionalLogsSuccess()); + return totalData; + } catch (error) { + dispatch(addErrorToast({ error })); + dispatch(getAdditionalLogsFailure(error)); + } + } + + dispatch(getAdditionalLogsSuccess()); + return totalData; +}; + export const setLogsPagination = createAction('LOGS_PAGINATION'); export const setLogsPage = createAction('SET_LOG_PAGE'); @@ -22,11 +58,20 @@ export const getLogsRequest = createAction('GET_LOGS_REQUEST'); export const getLogsFailure = createAction('GET_LOGS_FAILURE'); export const getLogsSuccess = createAction('GET_LOGS_SUCCESS'); -export const getLogs = config => async (dispatch) => { +export const getLogs = config => async (dispatch, getState) => { dispatch(getLogsRequest()); try { - const logs = await getLogsWithParams(config); - dispatch(getLogsSuccess(logs)); + const { isFiltered, filter, page } = getState().queryLogs; + const data = await getLogsWithParams({ ...config, filter }); + + if (isFiltered) { + const additionalData = await checkFilteredLogs(data, filter, dispatch); + const updatedData = additionalData.logs ? { ...data, ...additionalData } : data; + dispatch(getLogsSuccess(updatedData)); + dispatch(setLogsPagination({ page, pageSize: TABLE_DEFAULT_PAGE_SIZE })); + } else { + dispatch(getLogsSuccess(data)); + } } catch (error) { dispatch(addErrorToast({ error })); dispatch(getLogsFailure(error)); @@ -40,8 +85,11 @@ export const setLogsFilterSuccess = createAction('SET_LOGS_FILTER_SUCCESS'); export const setLogsFilter = filter => async (dispatch) => { dispatch(setLogsFilterRequest()); try { - const logs = await getLogsWithParams({ older_than: '', filter }); - dispatch(setLogsFilterSuccess(logs)); + const data = await getLogsWithParams({ older_than: '', filter }); + const additionalData = await checkFilteredLogs(data, filter, dispatch); + const updatedData = additionalData.logs ? { ...data, ...additionalData } : data; + + dispatch(setLogsFilterSuccess({ ...updatedData, filter })); dispatch(setLogsPage(0)); } catch (error) { dispatch(addErrorToast({ error })); diff --git a/client/src/components/Logs/Filters/Form.js b/client/src/components/Logs/Filters/Form.js index 9b175fc9..cedc6f5e 100644 --- a/client/src/components/Logs/Filters/Form.js +++ b/client/src/components/Logs/Filters/Form.js @@ -49,10 +49,10 @@ const Form = (props) => { return (
-
+
{ onChange={handleChange} />
-
+
{ onChange={handleChange} />
-
+
@@ -86,10 +86,10 @@ const Form = (props) => {
-
+
{ - const { - domain, client, type, response, - } = filtered; - - return { - filter_domain: domain || '', - filter_client: client || '', - filter_question_type: isValidQuestionType(type) ? type.toUpperCase() : '', - filter_response_status: response === RESPONSE_FILTER.FILTERED ? response : '', - }; - }; + getFilters = ({ + filter_domain, filter_question_type, filter_response_status, filter_client, + }) => ({ + filter_domain: filter_domain || '', + filter_question_type: isValidQuestionType(filter_question_type) ? filter_question_type.toUpperCase() : '', + filter_response_status: filter_response_status === RESPONSE_FILTER.FILTERED ? filter_response_status : '', + filter_client: filter_client || '', + }); handleFormChange = debounce((values) => { const filter = this.getFilters(values); @@ -26,13 +24,20 @@ class Filters extends Component { }, DEBOUNCE_FILTER_TIMEOUT); render() { - const { filter } = this.props; + const { filter, processingAdditionalLogs } = this.props; + + const cardBodyClass = classnames({ + 'card-body': true, + 'card-body--loading': processingAdditionalLogs, + }); return ( - + + + ); } } @@ -40,6 +45,8 @@ class Filters extends Component { Filters.propTypes = { filter: PropTypes.object.isRequired, setLogsFilter: PropTypes.func.isRequired, + processingGetLogs: PropTypes.bool.isRequired, + processingAdditionalLogs: PropTypes.bool.isRequired, }; export default Filters; diff --git a/client/src/components/Logs/index.js b/client/src/components/Logs/index.js index d1604525..68b9cc61 100644 --- a/client/src/components/Logs/index.js +++ b/client/src/components/Logs/index.js @@ -10,7 +10,7 @@ import { formatTime, formatDateTime, } from '../../helpers/helpers'; -import { SERVICES, FILTERED_STATUS, DEFAULT_LOGS_FILTER, RESPONSE_FILTER, TABLE_DEFAULT_PAGE_SIZE } from '../../helpers/constants'; +import { SERVICES, FILTERED_STATUS, TABLE_DEFAULT_PAGE_SIZE } from '../../helpers/constants'; import { getTrackerData } from '../../helpers/trackers/trackers'; import { formatClientCell } from '../../helpers/formatClientCell'; @@ -23,7 +23,7 @@ import Popover from '../ui/Popover'; import './Logs.css'; const TABLE_FIRST_PAGE = 0; -const INITIAL_REQUEST_DATA = ['', DEFAULT_LOGS_FILTER, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; +const INITIAL_REQUEST_DATA = ['', TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; const FILTERED_REASON = 'Filtered'; class Logs extends Component { @@ -35,10 +35,10 @@ class Logs extends Component { this.props.getLogsConfig(); } - getLogs = (older_than, filter, page) => { + getLogs = (older_than, page) => { if (this.props.queryLogs.enabled) { this.props.getLogs({ - older_than, filter, page, pageSize: TABLE_DEFAULT_PAGE_SIZE, + older_than, page, pageSize: TABLE_DEFAULT_PAGE_SIZE, }); } }; @@ -218,18 +218,19 @@ class Logs extends Component { fetchData = (state) => { const { pages } = state; - const { - filter, oldest, page, - } = this.props.queryLogs; + const { oldest, page } = this.props.queryLogs; const isLastPage = pages && (page + 1 === pages); if (isLastPage) { - this.getLogs(oldest, filter, page); - } else { - this.props.setLogsPagination({ page, pageSize: TABLE_DEFAULT_PAGE_SIZE }); + this.getLogs(oldest, page); } }; + changePage = (page) => { + this.props.setLogsPage(page); + this.props.setLogsPagination({ page, pageSize: TABLE_DEFAULT_PAGE_SIZE }); + }; + renderLogs() { const { queryLogs, dashboard, t } = this.props; const { processingClients } = dashboard; @@ -250,7 +251,6 @@ class Logs extends Component { accessor: 'domain', minWidth: 180, Cell: this.getDomainCell, - Filter: this.getFilterInput, }, { Header: t('type_table_header'), @@ -262,28 +262,6 @@ class Logs extends Component { accessor: 'response', minWidth: 250, Cell: this.getResponseCell, - filterMethod: (filter, row) => { - if (filter.value === RESPONSE_FILTER.FILTERED) { - // eslint-disable-next-line no-underscore-dangle - const { reason } = row._original; - return this.checkFiltered(reason) || this.checkWhiteList(reason); - } - return true; - }, - Filter: ({ filter, onChange }) => ( - - ), }, { Header: t('client_table_header'), @@ -291,7 +269,6 @@ class Logs extends Component { maxWidth: 240, minWidth: 240, Cell: this.getClientCell, - Filter: this.getFilterInput, }, ]; @@ -311,7 +288,7 @@ class Logs extends Component { showPageJump={false} showPageSizeOptions={false} onFetchData={this.fetchData} - onPageChange={newPage => this.props.setLogsPage(newPage)} + onPageChange={this.changePage} className="logs__table" defaultPageSize={TABLE_DEFAULT_PAGE_SIZE} previousText={t('previous_btn')} @@ -365,7 +342,9 @@ class Logs extends Component { render() { const { queryLogs, t } = this.props; - const { enabled, processingGetConfig } = queryLogs; + const { + enabled, processingGetConfig, processingAdditionalLogs, processingGetLogs, + } = queryLogs; const refreshButton = enabled ? (