AdGuardHome/internal/stats/stats_internal_test.go

171 lines
3.6 KiB
Go
Raw Permalink Normal View History

cherry-pick: 4358 stats races Merge in DNS/adguard-home from 4358-stats-races to master Updates #4358 Squashed commit of the following: commit 162d17b04d95adad21fb9b3c5a6fb64df2e037ec Merge: 17732cfa d4c3a43b Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 14:04:20 2022 +0300 Merge branch 'master' into 4358-stats-races commit 17732cfa0f3b2589bf2c252697eee1d6b358a66c Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 13:53:42 2022 +0300 stats: imp docs, locking commit 4ee090869af0fa2b777c12027c3b77d5acd6e4de Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:26:19 2022 +0300 stats: revert const commit a7681a1b882cef04511fcd5d569f5abe2f955239 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:23:00 2022 +0300 stats: imp concurrency commit a6c6c1a0572e4201cd24644fd3f86f51fc27f633 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 19:51:30 2022 +0300 stats: imp code, tests, docs commit 954196b49f5ad91d91f445ff656e63c318e4124c Merge: 281e00da 6e63757f Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 13:07:32 2022 +0300 Merge branch 'master' into 4358-stats-races commit 281e00daf781d045269584ce0158eed1d77918df Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:22:18 2022 +0300 stats: imp closing commit ed036d9aa7e25498869edfb866b6e923538970eb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:11:12 2022 +0300 stats: imp tests more commit f848a12487ecd2afc8416e800510090cc1be7330 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 13:54:19 2022 +0300 stats: imp tests, code commit 60e11f042d51ec68850143129e61c701c5e4f3a4 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 16:36:07 2022 +0300 stats: fix test commit 6d97f1db093b5ce0d37984ff96a9ef6f4e02dba1 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 14:53:21 2022 +0300 stats: imp code, docs commit 20c70c2847b0de6c7f9271a8d9a831175ed0c499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 20:53:36 2022 +0300 stats: imp shared memory safety commit 8b3945670a190bab070171e6b4976edab1e3e2a2 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 17:22:55 2022 +0300 stats: imp code
2022-08-17 12:09:13 +01:00
package stats
import (
2022-09-07 16:03:18 +01:00
"fmt"
"path/filepath"
"sync"
"sync/atomic"
cherry-pick: 4358 stats races Merge in DNS/adguard-home from 4358-stats-races to master Updates #4358 Squashed commit of the following: commit 162d17b04d95adad21fb9b3c5a6fb64df2e037ec Merge: 17732cfa d4c3a43b Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 14:04:20 2022 +0300 Merge branch 'master' into 4358-stats-races commit 17732cfa0f3b2589bf2c252697eee1d6b358a66c Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 13:53:42 2022 +0300 stats: imp docs, locking commit 4ee090869af0fa2b777c12027c3b77d5acd6e4de Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:26:19 2022 +0300 stats: revert const commit a7681a1b882cef04511fcd5d569f5abe2f955239 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:23:00 2022 +0300 stats: imp concurrency commit a6c6c1a0572e4201cd24644fd3f86f51fc27f633 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 19:51:30 2022 +0300 stats: imp code, tests, docs commit 954196b49f5ad91d91f445ff656e63c318e4124c Merge: 281e00da 6e63757f Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 13:07:32 2022 +0300 Merge branch 'master' into 4358-stats-races commit 281e00daf781d045269584ce0158eed1d77918df Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:22:18 2022 +0300 stats: imp closing commit ed036d9aa7e25498869edfb866b6e923538970eb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:11:12 2022 +0300 stats: imp tests more commit f848a12487ecd2afc8416e800510090cc1be7330 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 13:54:19 2022 +0300 stats: imp tests, code commit 60e11f042d51ec68850143129e61c701c5e4f3a4 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 16:36:07 2022 +0300 stats: fix test commit 6d97f1db093b5ce0d37984ff96a9ef6f4e02dba1 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 14:53:21 2022 +0300 stats: imp code, docs commit 20c70c2847b0de6c7f9271a8d9a831175ed0c499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 20:53:36 2022 +0300 stats: imp shared memory safety commit 8b3945670a190bab070171e6b4976edab1e3e2a2 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 17:22:55 2022 +0300 stats: imp code
2022-08-17 12:09:13 +01:00
"testing"
2022-09-07 16:03:18 +01:00
"time"
cherry-pick: 4358 stats races Merge in DNS/adguard-home from 4358-stats-races to master Updates #4358 Squashed commit of the following: commit 162d17b04d95adad21fb9b3c5a6fb64df2e037ec Merge: 17732cfa d4c3a43b Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 14:04:20 2022 +0300 Merge branch 'master' into 4358-stats-races commit 17732cfa0f3b2589bf2c252697eee1d6b358a66c Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 13:53:42 2022 +0300 stats: imp docs, locking commit 4ee090869af0fa2b777c12027c3b77d5acd6e4de Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:26:19 2022 +0300 stats: revert const commit a7681a1b882cef04511fcd5d569f5abe2f955239 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:23:00 2022 +0300 stats: imp concurrency commit a6c6c1a0572e4201cd24644fd3f86f51fc27f633 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 19:51:30 2022 +0300 stats: imp code, tests, docs commit 954196b49f5ad91d91f445ff656e63c318e4124c Merge: 281e00da 6e63757f Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 13:07:32 2022 +0300 Merge branch 'master' into 4358-stats-races commit 281e00daf781d045269584ce0158eed1d77918df Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:22:18 2022 +0300 stats: imp closing commit ed036d9aa7e25498869edfb866b6e923538970eb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:11:12 2022 +0300 stats: imp tests more commit f848a12487ecd2afc8416e800510090cc1be7330 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 13:54:19 2022 +0300 stats: imp tests, code commit 60e11f042d51ec68850143129e61c701c5e4f3a4 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 16:36:07 2022 +0300 stats: fix test commit 6d97f1db093b5ce0d37984ff96a9ef6f4e02dba1 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 14:53:21 2022 +0300 stats: imp code, docs commit 20c70c2847b0de6c7f9271a8d9a831175ed0c499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 20:53:36 2022 +0300 stats: imp shared memory safety commit 8b3945670a190bab070171e6b4976edab1e3e2a2 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 17:22:55 2022 +0300 stats: imp code
2022-08-17 12:09:13 +01:00
2022-09-07 16:03:18 +01:00
"github.com/AdguardTeam/golibs/testutil"
2023-04-12 12:48:42 +01:00
"github.com/AdguardTeam/golibs/timeutil"
cherry-pick: 4358 stats races Merge in DNS/adguard-home from 4358-stats-races to master Updates #4358 Squashed commit of the following: commit 162d17b04d95adad21fb9b3c5a6fb64df2e037ec Merge: 17732cfa d4c3a43b Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 14:04:20 2022 +0300 Merge branch 'master' into 4358-stats-races commit 17732cfa0f3b2589bf2c252697eee1d6b358a66c Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 17 13:53:42 2022 +0300 stats: imp docs, locking commit 4ee090869af0fa2b777c12027c3b77d5acd6e4de Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:26:19 2022 +0300 stats: revert const commit a7681a1b882cef04511fcd5d569f5abe2f955239 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 20:23:00 2022 +0300 stats: imp concurrency commit a6c6c1a0572e4201cd24644fd3f86f51fc27f633 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 19:51:30 2022 +0300 stats: imp code, tests, docs commit 954196b49f5ad91d91f445ff656e63c318e4124c Merge: 281e00da 6e63757f Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Aug 16 13:07:32 2022 +0300 Merge branch 'master' into 4358-stats-races commit 281e00daf781d045269584ce0158eed1d77918df Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:22:18 2022 +0300 stats: imp closing commit ed036d9aa7e25498869edfb866b6e923538970eb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 16:11:12 2022 +0300 stats: imp tests more commit f848a12487ecd2afc8416e800510090cc1be7330 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Aug 12 13:54:19 2022 +0300 stats: imp tests, code commit 60e11f042d51ec68850143129e61c701c5e4f3a4 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 16:36:07 2022 +0300 stats: fix test commit 6d97f1db093b5ce0d37984ff96a9ef6f4e02dba1 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Aug 11 14:53:21 2022 +0300 stats: imp code, docs commit 20c70c2847b0de6c7f9271a8d9a831175ed0c499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 20:53:36 2022 +0300 stats: imp shared memory safety commit 8b3945670a190bab070171e6b4976edab1e3e2a2 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Aug 10 17:22:55 2022 +0300 stats: imp code
2022-08-17 12:09:13 +01:00
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
2022-09-07 16:03:18 +01:00
func TestStats_races(t *testing.T) {
var r uint32
idGen := func() (id uint32) { return atomic.LoadUint32(&r) }
conf := Config{
2023-04-12 12:48:42 +01:00
ShouldCountClient: func([]string) bool { return true },
UnitID: idGen,
Filename: filepath.Join(t.TempDir(), "./stats.db"),
Limit: timeutil.Day,
2022-09-07 16:03:18 +01:00
}
s, err := New(conf)
require.NoError(t, err)
s.Start()
startTime := time.Now()
testutil.CleanupAndRequireSuccess(t, s.Close)
writeFunc := func(start, fin *sync.WaitGroup, waitCh <-chan unit, i int) {
2023-09-07 15:13:48 +01:00
e := &Entry{
2023-11-13 14:39:48 +00:00
Domain: fmt.Sprintf("example-%d.org", i),
Client: fmt.Sprintf("client_%d", i),
Result: Result(i)%(resultLast-1) + 1,
ProcessingTime: time.Since(startTime),
2022-09-07 16:03:18 +01:00
}
start.Done()
defer fin.Done()
<-waitCh
s.Update(e)
}
readFunc := func(start, fin *sync.WaitGroup, waitCh <-chan unit) {
start.Done()
defer fin.Done()
<-waitCh
_, _ = s.getData(24)
}
const (
roundsNum = 3
writersNum = 10
readersNum = 5
)
for round := 0; round < roundsNum; round++ {
atomic.StoreUint32(&r, uint32(round))
startWG, finWG := &sync.WaitGroup{}, &sync.WaitGroup{}
waitCh := make(chan unit)
2024-05-15 11:34:12 +01:00
for i := range writersNum {
2022-09-07 16:03:18 +01:00
startWG.Add(1)
finWG.Add(1)
go writeFunc(startWG, finWG, waitCh, i)
}
2024-05-15 11:34:12 +01:00
for range readersNum {
2022-09-07 16:03:18 +01:00
startWG.Add(1)
finWG.Add(1)
go readFunc(startWG, finWG, waitCh)
}
startWG.Wait()
close(waitCh)
finWG.Wait()
}
}
2023-09-07 15:13:48 +01:00
func TestStatsCtx_FillCollectedStats_daily(t *testing.T) {
const (
daysCount = 10
timeUnits = "days"
)
s, err := New(Config{
ShouldCountClient: func([]string) bool { return true },
Filename: filepath.Join(t.TempDir(), "./stats.db"),
Limit: time.Hour,
})
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, s.Close)
sum := make([][]uint64, resultLast)
sum[RFiltered] = make([]uint64, daysCount)
sum[RSafeBrowsing] = make([]uint64, daysCount)
sum[RParental] = make([]uint64, daysCount)
total := make([]uint64, daysCount)
dailyData := []*unitDB{}
2024-05-15 11:34:12 +01:00
for i := range daysCount * 24 {
2023-09-07 15:13:48 +01:00
n := uint64(i)
nResult := make([]uint64, resultLast)
nResult[RFiltered] = n
nResult[RSafeBrowsing] = n
nResult[RParental] = n
day := i / 24
sum[RFiltered][day] += n
sum[RSafeBrowsing][day] += n
sum[RParental][day] += n
t := n * 3
total[day] += t
dailyData = append(dailyData, &unitDB{
NTotal: t,
NResult: nResult,
})
}
data := &StatsResp{}
// In this way we will not skip first hours.
curID := uint32(daysCount * 24)
s.fillCollectedStats(data, dailyData, curID)
assert.Equal(t, timeUnits, data.TimeUnits)
assert.Equal(t, sum[RFiltered], data.BlockedFiltering)
assert.Equal(t, sum[RSafeBrowsing], data.ReplacedSafebrowsing)
assert.Equal(t, sum[RParental], data.ReplacedParental)
assert.Equal(t, total, data.DNSQueries)
}
func TestStatsCtx_DataFromUnits_month(t *testing.T) {
const hoursInMonth = 720
s, err := New(Config{
ShouldCountClient: func([]string) bool { return true },
Filename: filepath.Join(t.TempDir(), "./stats.db"),
Limit: time.Hour,
})
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, s.Close)
units, curID := s.loadUnits(hoursInMonth)
require.Len(t, units, hoursInMonth)
var h uint32
for h = 1; h <= hoursInMonth; h++ {
data := s.dataFromUnits(units[:h], curID)
require.NotNil(t, data)
}
}