querylog file -- disable gzip compression

This commit is contained in:
Eugene Bujak 2018-10-09 05:02:16 +03:00
parent 37f6d38c49
commit ca794aed63
1 changed files with 59 additions and 32 deletions

View File

@ -17,6 +17,8 @@ var (
fileWriteLock sync.Mutex fileWriteLock sync.Mutex
) )
const enableGzip = false
func flushToFile(buffer []logEntry) error { func flushToFile(buffer []logEntry) error {
if len(buffer) == 0 { if len(buffer) == 0 {
return nil return nil
@ -42,31 +44,37 @@ func flushToFile(buffer []logEntry) error {
return err return err
} }
filenamegz := queryLogFileName + ".gz"
var zb bytes.Buffer var zb bytes.Buffer
filename := queryLogFileName
zw := gzip.NewWriter(&zb) // gzip enabled?
zw.Name = queryLogFileName if enableGzip {
zw.ModTime = time.Now() filename += ".gz"
_, err = zw.Write(b.Bytes()) zw := gzip.NewWriter(&zb)
if err != nil { zw.Name = queryLogFileName
log.Printf("Couldn't compress to gzip: %s", err) zw.ModTime = time.Now()
zw.Close()
return err
}
if err = zw.Close(); err != nil { _, err = zw.Write(b.Bytes())
log.Printf("Couldn't close gzip writer: %s", err) if err != nil {
return err log.Printf("Couldn't compress to gzip: %s", err)
zw.Close()
return err
}
if err = zw.Close(); err != nil {
log.Printf("Couldn't close gzip writer: %s", err)
return err
}
} else {
zb = b
} }
fileWriteLock.Lock() fileWriteLock.Lock()
defer fileWriteLock.Unlock() defer fileWriteLock.Unlock()
f, err := os.OpenFile(filenamegz, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil { if err != nil {
log.Printf("failed to create file \"%s\": %s", filenamegz, err) log.Printf("failed to create file \"%s\": %s", filename, err)
return err return err
} }
defer f.Close() defer f.Close()
@ -77,7 +85,7 @@ func flushToFile(buffer []logEntry) error {
return err return err
} }
log.Printf("ok \"%s\": %v bytes written", filenamegz, n) log.Printf("ok \"%s\": %v bytes written", filename, n)
return nil return nil
} }
@ -111,8 +119,13 @@ func checkBuffer(buffer []logEntry, b bytes.Buffer) error {
} }
func rotateQueryLog() error { func rotateQueryLog() error {
from := queryLogFileName + ".gz" from := queryLogFileName
to := queryLogFileName + ".gz.1" to := queryLogFileName + ".1"
if enableGzip {
from = queryLogFileName + ".gz"
to = queryLogFileName + ".gz.1"
}
if _, err := os.Stat(from); os.IsNotExist(err) { if _, err := os.Stat(from); os.IsNotExist(err) {
// do nothing, file doesn't exist // do nothing, file doesn't exist
@ -143,9 +156,18 @@ func periodicQueryLogRotate() {
func genericLoader(onEntry func(entry *logEntry) error, needMore func() bool, timeWindow time.Duration) error { func genericLoader(onEntry func(entry *logEntry) error, needMore func() bool, timeWindow time.Duration) error {
now := time.Now() now := time.Now()
// read from querylog files, try newest file first // read from querylog files, try newest file first
files := []string{ files := []string{}
queryLogFileName + ".gz",
queryLogFileName + ".gz.1", if enableGzip {
files = []string{
queryLogFileName + ".gz",
queryLogFileName + ".gz.1",
}
} else {
files = []string{
queryLogFileName,
queryLogFileName + ".1",
}
} }
// read from all files // read from all files
@ -158,7 +180,6 @@ func genericLoader(onEntry func(entry *logEntry) error, needMore func() bool, ti
continue continue
} }
trace("Opening file %s", file)
f, err := os.Open(file) f, err := os.Open(file)
if err != nil { if err != nil {
log.Printf("Failed to open file \"%s\": %s", file, err) log.Printf("Failed to open file \"%s\": %s", file, err)
@ -167,16 +188,22 @@ func genericLoader(onEntry func(entry *logEntry) error, needMore func() bool, ti
} }
defer f.Close() defer f.Close()
trace("Creating gzip reader") var d *json.Decoder
zr, err := gzip.NewReader(f)
if err != nil {
log.Printf("Failed to create gzip reader: %s", err)
continue
}
defer zr.Close()
trace("Creating json decoder") if enableGzip {
d := json.NewDecoder(zr) trace("Creating gzip reader")
zr, err := gzip.NewReader(f)
if err != nil {
log.Printf("Failed to create gzip reader: %s", err)
continue
}
defer zr.Close()
trace("Creating json decoder")
d = json.NewDecoder(zr)
} else {
d = json.NewDecoder(f)
}
i := 0 i := 0
// entries on file are in oldest->newest order // entries on file are in oldest->newest order