AdGuardHome/querylog/querylog_file.go

140 lines
2.9 KiB
Go
Raw Normal View History

package querylog
import (
"bytes"
"encoding/json"
"os"
"time"
"github.com/AdguardTeam/golibs/log"
)
2019-02-11 11:22:36 +00:00
// flushLogBuffer flushes the current buffer to file and resets the current buffer
func (l *queryLog) flushLogBuffer(fullFlush bool) error {
if !l.conf.FileEnabled {
return nil
}
l.fileFlushLock.Lock()
defer l.fileFlushLock.Unlock()
// flush remainder to file
l.bufferLock.Lock()
needFlush := len(l.buffer) >= int(l.conf.MemSize)
if !needFlush && !fullFlush {
l.bufferLock.Unlock()
return nil
}
flushBuffer := l.buffer
l.buffer = nil
l.flushPending = false
l.bufferLock.Unlock()
err := l.flushToFile(flushBuffer)
if err != nil {
log.Error("Saving querylog to file failed: %s", err)
return err
}
return nil
}
// flushToFile saves the specified log entries to the query log file
func (l *queryLog) flushToFile(buffer []*logEntry) error {
if len(buffer) == 0 {
2019-05-08 08:43:47 +01:00
log.Debug("querylog: there's nothing to write to a file")
return nil
}
start := time.Now()
var b bytes.Buffer
e := json.NewEncoder(&b)
for _, entry := range buffer {
err := e.Encode(entry)
if err != nil {
log.Error("Failed to marshal entry: %s", err)
return err
}
}
elapsed := time.Since(start)
log.Debug("%d elements serialized via json in %v: %d kB, %v/entry, %v/entry", len(buffer), elapsed, b.Len()/1024, float64(b.Len())/float64(len(buffer)), elapsed/time.Duration(len(buffer)))
var err error
var zb bytes.Buffer
filename := l.logFile
zb = b
l.fileWriteLock.Lock()
defer l.fileWriteLock.Unlock()
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Error("failed to create file \"%s\": %s", filename, err)
return err
}
defer f.Close()
n, err := f.Write(zb.Bytes())
if err != nil {
log.Error("Couldn't write to file: %s", err)
return err
}
- querylog: file rotation didn't work properly; fix entry searching algorithm If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started. Squashed commit of the following: commit 427ae91a512cd146ebfffad06ed24eb723cb9e7d Merge: 067fac65 e56c746b Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 18:18:46 2020 +0300 Merge remote-tracking branch 'origin/master' into qlogs-rotate commit 067fac65b1a87d499900f4860ffa96ed8208967c Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 15:30:48 2020 +0300 minor commit c2059a15700e5696cb1bb5cd49129c6020d986f4 Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 14:53:07 2020 +0300 improve commit a279438eaf1cf40b820652093fb56d56784de7d8 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 18:49:14 2020 +0300 minor commit 26ac130f139f565de39200e484b3bd4a04afcfcc Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:54:27 2020 +0300 rename commit 0fad7b88dbeadcddd4d77536a18da72f3203ea80 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:36 2020 +0300 + TestQLogSeek commit fa6afc6d4dc592b1fef67c4a069ea50fae600a58 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:34 2020 +0300 minor commit 11e6ab9131e5c37467e8530a2db95a82bbb0603b Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:45:47 2020 +0300 fix tests commit 7cbb89948df0e69b1bae8f8cde1879b5b1c4b1d6 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:29:43 2020 +0300 - querylog: fix entry searching algorithm commit 745d44863d88b321bd7001f24a68620f7ef05819 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 18:34:14 2020 +0300 - querylog: file rotation didn't work properly If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started.
2020-09-02 17:42:26 +01:00
log.Debug("querylog: ok \"%s\": %v bytes written", filename, n)
return nil
}
func (l *queryLog) rotate() error {
from := l.logFile
to := l.logFile + ".1"
if _, err := os.Stat(from); os.IsNotExist(err) {
// do nothing, file doesn't exist
return nil
}
err := os.Rename(from, to)
if err != nil {
- querylog: file rotation didn't work properly; fix entry searching algorithm If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started. Squashed commit of the following: commit 427ae91a512cd146ebfffad06ed24eb723cb9e7d Merge: 067fac65 e56c746b Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 18:18:46 2020 +0300 Merge remote-tracking branch 'origin/master' into qlogs-rotate commit 067fac65b1a87d499900f4860ffa96ed8208967c Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 15:30:48 2020 +0300 minor commit c2059a15700e5696cb1bb5cd49129c6020d986f4 Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 14:53:07 2020 +0300 improve commit a279438eaf1cf40b820652093fb56d56784de7d8 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 18:49:14 2020 +0300 minor commit 26ac130f139f565de39200e484b3bd4a04afcfcc Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:54:27 2020 +0300 rename commit 0fad7b88dbeadcddd4d77536a18da72f3203ea80 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:36 2020 +0300 + TestQLogSeek commit fa6afc6d4dc592b1fef67c4a069ea50fae600a58 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:34 2020 +0300 minor commit 11e6ab9131e5c37467e8530a2db95a82bbb0603b Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:45:47 2020 +0300 fix tests commit 7cbb89948df0e69b1bae8f8cde1879b5b1c4b1d6 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:29:43 2020 +0300 - querylog: fix entry searching algorithm commit 745d44863d88b321bd7001f24a68620f7ef05819 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 18:34:14 2020 +0300 - querylog: file rotation didn't work properly If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started.
2020-09-02 17:42:26 +01:00
log.Error("querylog: failed to rename file: %s", err)
return err
}
- querylog: file rotation didn't work properly; fix entry searching algorithm If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started. Squashed commit of the following: commit 427ae91a512cd146ebfffad06ed24eb723cb9e7d Merge: 067fac65 e56c746b Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 18:18:46 2020 +0300 Merge remote-tracking branch 'origin/master' into qlogs-rotate commit 067fac65b1a87d499900f4860ffa96ed8208967c Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 15:30:48 2020 +0300 minor commit c2059a15700e5696cb1bb5cd49129c6020d986f4 Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 14:53:07 2020 +0300 improve commit a279438eaf1cf40b820652093fb56d56784de7d8 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 18:49:14 2020 +0300 minor commit 26ac130f139f565de39200e484b3bd4a04afcfcc Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:54:27 2020 +0300 rename commit 0fad7b88dbeadcddd4d77536a18da72f3203ea80 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:36 2020 +0300 + TestQLogSeek commit fa6afc6d4dc592b1fef67c4a069ea50fae600a58 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:34 2020 +0300 minor commit 11e6ab9131e5c37467e8530a2db95a82bbb0603b Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:45:47 2020 +0300 fix tests commit 7cbb89948df0e69b1bae8f8cde1879b5b1c4b1d6 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:29:43 2020 +0300 - querylog: fix entry searching algorithm commit 745d44863d88b321bd7001f24a68620f7ef05819 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 18:34:14 2020 +0300 - querylog: file rotation didn't work properly If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started.
2020-09-02 17:42:26 +01:00
log.Debug("querylog: renamed %s -> %s", from, to)
return nil
}
- querylog: file rotation didn't work properly; fix entry searching algorithm If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started. Squashed commit of the following: commit 427ae91a512cd146ebfffad06ed24eb723cb9e7d Merge: 067fac65 e56c746b Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 18:18:46 2020 +0300 Merge remote-tracking branch 'origin/master' into qlogs-rotate commit 067fac65b1a87d499900f4860ffa96ed8208967c Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 15:30:48 2020 +0300 minor commit c2059a15700e5696cb1bb5cd49129c6020d986f4 Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 14:53:07 2020 +0300 improve commit a279438eaf1cf40b820652093fb56d56784de7d8 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 18:49:14 2020 +0300 minor commit 26ac130f139f565de39200e484b3bd4a04afcfcc Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:54:27 2020 +0300 rename commit 0fad7b88dbeadcddd4d77536a18da72f3203ea80 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:36 2020 +0300 + TestQLogSeek commit fa6afc6d4dc592b1fef67c4a069ea50fae600a58 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:34 2020 +0300 minor commit 11e6ab9131e5c37467e8530a2db95a82bbb0603b Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:45:47 2020 +0300 fix tests commit 7cbb89948df0e69b1bae8f8cde1879b5b1c4b1d6 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:29:43 2020 +0300 - querylog: fix entry searching algorithm commit 745d44863d88b321bd7001f24a68620f7ef05819 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 18:34:14 2020 +0300 - querylog: file rotation didn't work properly If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started.
2020-09-02 17:42:26 +01:00
func (l *queryLog) readFileFirstTimeValue() int64 {
f, err := os.Open(l.logFile)
if err != nil {
return -1
}
defer f.Close()
buf := make([]byte, 500)
r, err := f.Read(buf)
if err != nil {
return -1
}
buf = buf[:r]
val := readJSONValue(string(buf), "T")
t, err := time.Parse(time.RFC3339Nano, val)
if err != nil {
return -1
}
log.Debug("querylog: the oldest log entry: %s", val)
return t.Unix()
}
func (l *queryLog) periodicRotate() {
- querylog: file rotation didn't work properly; fix entry searching algorithm If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started. Squashed commit of the following: commit 427ae91a512cd146ebfffad06ed24eb723cb9e7d Merge: 067fac65 e56c746b Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 18:18:46 2020 +0300 Merge remote-tracking branch 'origin/master' into qlogs-rotate commit 067fac65b1a87d499900f4860ffa96ed8208967c Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 15:30:48 2020 +0300 minor commit c2059a15700e5696cb1bb5cd49129c6020d986f4 Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 14:53:07 2020 +0300 improve commit a279438eaf1cf40b820652093fb56d56784de7d8 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 18:49:14 2020 +0300 minor commit 26ac130f139f565de39200e484b3bd4a04afcfcc Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:54:27 2020 +0300 rename commit 0fad7b88dbeadcddd4d77536a18da72f3203ea80 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:36 2020 +0300 + TestQLogSeek commit fa6afc6d4dc592b1fef67c4a069ea50fae600a58 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:34 2020 +0300 minor commit 11e6ab9131e5c37467e8530a2db95a82bbb0603b Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:45:47 2020 +0300 fix tests commit 7cbb89948df0e69b1bae8f8cde1879b5b1c4b1d6 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:29:43 2020 +0300 - querylog: fix entry searching algorithm commit 745d44863d88b321bd7001f24a68620f7ef05819 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 18:34:14 2020 +0300 - querylog: file rotation didn't work properly If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started.
2020-09-02 17:42:26 +01:00
intervalSeconds := uint64(l.conf.Interval) * 24 * 60 * 60
for {
oldest := l.readFileFirstTimeValue()
if uint64(oldest)+intervalSeconds <= uint64(time.Now().Unix()) {
_ = l.rotate()
}
- querylog: file rotation didn't work properly; fix entry searching algorithm If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started. Squashed commit of the following: commit 427ae91a512cd146ebfffad06ed24eb723cb9e7d Merge: 067fac65 e56c746b Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 18:18:46 2020 +0300 Merge remote-tracking branch 'origin/master' into qlogs-rotate commit 067fac65b1a87d499900f4860ffa96ed8208967c Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 15:30:48 2020 +0300 minor commit c2059a15700e5696cb1bb5cd49129c6020d986f4 Author: Simon Zolin <s.zolin@adguard.com> Date: Wed Sep 2 14:53:07 2020 +0300 improve commit a279438eaf1cf40b820652093fb56d56784de7d8 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 18:49:14 2020 +0300 minor commit 26ac130f139f565de39200e484b3bd4a04afcfcc Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:54:27 2020 +0300 rename commit 0fad7b88dbeadcddd4d77536a18da72f3203ea80 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:36 2020 +0300 + TestQLogSeek commit fa6afc6d4dc592b1fef67c4a069ea50fae600a58 Author: Simon Zolin <s.zolin@adguard.com> Date: Tue Sep 1 13:05:34 2020 +0300 minor commit 11e6ab9131e5c37467e8530a2db95a82bbb0603b Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:45:47 2020 +0300 fix tests commit 7cbb89948df0e69b1bae8f8cde1879b5b1c4b1d6 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 19:29:43 2020 +0300 - querylog: fix entry searching algorithm commit 745d44863d88b321bd7001f24a68620f7ef05819 Author: Simon Zolin <s.zolin@adguard.com> Date: Mon Aug 31 18:34:14 2020 +0300 - querylog: file rotation didn't work properly If AGH is restarted, file rotation timer is reset which can lead to the situation when file rotation procedure is never started.
2020-09-02 17:42:26 +01:00
time.Sleep(24 * time.Hour)
}
}