tka: implement compaction logic
Signed-off-by: Tom DNetto <tom@tailscale.com>
This commit is contained in:
parent
bb7033174c
commit
ff168a806e
319
tka/tailchonk.go
319
tka/tailchonk.go
|
@ -5,10 +5,12 @@ package tka
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fxamacker/cbor/v2"
|
||||
"tailscale.com/atomicfile"
|
||||
|
@ -53,6 +55,24 @@ type Chonk interface {
|
|||
LastActiveAncestor() (*AUMHash, error)
|
||||
}
|
||||
|
||||
// CompactableChonk implementation are extensions of Chonk, which are
|
||||
// able to be operated by compaction logic to deleted old AUMs.
|
||||
type CompactableChonk interface {
|
||||
Chonk
|
||||
|
||||
// AllAUMs returns all AUMs stored in the chonk.
|
||||
AllAUMs() ([]AUMHash, error)
|
||||
|
||||
// CommitTime returns the time at which the AUM was committed.
|
||||
//
|
||||
// If the AUM does not exist, then os.ErrNotExist is returned.
|
||||
CommitTime(hash AUMHash) (time.Time, error)
|
||||
|
||||
// PurgeAUMs permanently and irrevocably deletes the specified
|
||||
// AUMs from storage.
|
||||
PurgeAUMs(hashes []AUMHash) error
|
||||
}
|
||||
|
||||
// Mem implements in-memory storage of TKA state, suitable for
|
||||
// tests.
|
||||
//
|
||||
|
@ -437,3 +457,302 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error {
|
|||
}
|
||||
return atomicfile.WriteFile(filepath.Join(dir, base), buff.Bytes(), 0644)
|
||||
}
|
||||
|
||||
// CompactionOptions describes tuneables to use when compacting a Chonk.
|
||||
type CompactionOptions struct {
|
||||
// The minimum number of ancestor AUMs to remember. The actual length
|
||||
// of the chain post-compaction may be longer to reach a Checkpoint AUM.
|
||||
MinChain int
|
||||
// The minimum duration to store an AUM before it is a candidate for deletion.
|
||||
MinAge time.Duration
|
||||
}
|
||||
|
||||
// retainState tracks the state of an AUM hash as it is being considered for
|
||||
// deletion.
|
||||
type retainState uint8
|
||||
|
||||
// Valid retainState flags.
|
||||
const (
|
||||
retainStateActive retainState = 1 << iota // The AUM is part of the active chain and less than MinChain hops from HEAD.
|
||||
retainStateYoung // The AUM is younger than MinAge.
|
||||
retainStateLeaf // The AUM is a descendant of an AUM to be retained.
|
||||
retainStateAncestor // The AUM is part of a chain between a retained AUM and the new lastActiveAncestor.
|
||||
retainStateCandidate // The AUM is part of the active chain.
|
||||
|
||||
// retainAUMMask is a bit mask of any bit which should prevent
|
||||
// the deletion of an AUM.
|
||||
retainAUMMask retainState = retainStateActive | retainStateYoung | retainStateLeaf | retainStateAncestor
|
||||
)
|
||||
|
||||
// markActiveChain marks AUMs in the active chain.
|
||||
// All AUMs that are within minChain ancestors of head are
|
||||
// marked retainStateActive, and all remaining ancestors are
|
||||
// marked retainStateCandidate.
|
||||
//
|
||||
// markActiveChain returns the next ancestor AUM which is a checkpoint AUM.
|
||||
func markActiveChain(storage Chonk, verdict map[AUMHash]retainState, minChain int, head AUMHash) (lastActiveAncestor AUMHash, err error) {
|
||||
next, err := storage.AUM(head)
|
||||
if err != nil {
|
||||
return AUMHash{}, err
|
||||
}
|
||||
|
||||
for i := 0; i < minChain; i++ {
|
||||
h := next.Hash()
|
||||
verdict[h] |= retainStateActive
|
||||
|
||||
parent, hasParent := next.Parent()
|
||||
if !hasParent {
|
||||
// Genesis AUM (beginning of time). The chain isnt long enough to need truncating.
|
||||
return h, nil
|
||||
}
|
||||
|
||||
if next, err = storage.AUM(parent); err != nil {
|
||||
if err == os.ErrNotExist {
|
||||
// We've reached the end of the chain we have stored.
|
||||
return h, nil
|
||||
}
|
||||
return AUMHash{}, fmt.Errorf("reading active chain (retainStateActive) (%d): %w", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// If we got this far, we have at least minChain AUMs stored, and minChain number
|
||||
// of ancestors have been marked for retention. We now continue to iterate backwards
|
||||
// till we find an AUM which we can compact to (a Checkpoint AUM).
|
||||
for {
|
||||
h := next.Hash()
|
||||
verdict[h] |= retainStateActive
|
||||
if next.MessageKind == AUMCheckpoint {
|
||||
lastActiveAncestor = h
|
||||
break
|
||||
}
|
||||
|
||||
parent, hasParent := next.Parent()
|
||||
if !hasParent {
|
||||
return AUMHash{}, errors.New("reached genesis AUM without finding an appropriate lastActiveAncestor")
|
||||
}
|
||||
if next, err = storage.AUM(parent); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("searching for compaction target: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Mark remaining known ancestors as retainStateCandidate.
|
||||
for {
|
||||
parent, hasParent := next.Parent()
|
||||
if !hasParent {
|
||||
break
|
||||
}
|
||||
verdict[parent] |= retainStateCandidate
|
||||
if next, err = storage.AUM(parent); err != nil {
|
||||
if err == os.ErrNotExist {
|
||||
// We've reached the end of the chain we have stored.
|
||||
break
|
||||
}
|
||||
return AUMHash{}, fmt.Errorf("reading active chain (retainStateCandidate): %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return lastActiveAncestor, nil
|
||||
}
|
||||
|
||||
// markYoungAUMs marks all AUMs younger than minAge for retention. All
|
||||
// candidate AUMs must exist in verdict.
|
||||
func markYoungAUMs(storage CompactableChonk, verdict map[AUMHash]retainState, minAge time.Duration) error {
|
||||
minTime := time.Now().Add(-minAge)
|
||||
for h, _ := range verdict {
|
||||
commitTime, err := storage.CommitTime(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if commitTime.After(minTime) {
|
||||
verdict[h] |= retainStateYoung
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// markAncestorIntersectionAUMs walks backwards from all AUMs to be retained,
|
||||
// ensuring they intersect with candidateAncestor. All AUMs between a retained
|
||||
// AUM and candidateAncestor are marked for retention.
|
||||
//
|
||||
// If there is no intersection between candidateAncestor and the ancestors of
|
||||
// a retained AUM (this can happen if a retained AUM intersects the main chain
|
||||
// before candidateAncestor) then candidate ancestor is recomputed based on
|
||||
// the new oldest intersection.
|
||||
//
|
||||
// The final value for lastActiveAncestor is returned.
|
||||
func markAncestorIntersectionAUMs(storage Chonk, verdict map[AUMHash]retainState, candidateAncestor AUMHash) (lastActiveAncestor AUMHash, err error) {
|
||||
toScan := make([]AUMHash, 0, len(verdict))
|
||||
for h, v := range verdict {
|
||||
if (v & retainAUMMask) == 0 {
|
||||
continue // not marked for retention, so dont need to consider it
|
||||
}
|
||||
if h == candidateAncestor {
|
||||
continue
|
||||
}
|
||||
toScan = append(toScan, h)
|
||||
}
|
||||
|
||||
var didAdjustCandidateAncestor bool
|
||||
for len(toScan) > 0 {
|
||||
nextIterScan := make([]AUMHash, 0, len(verdict))
|
||||
for _, h := range toScan {
|
||||
if verdict[h]&retainStateAncestor != 0 {
|
||||
// This AUM and its ancestors have already been iterated.
|
||||
continue
|
||||
}
|
||||
verdict[h] |= retainStateAncestor
|
||||
|
||||
a, err := storage.AUM(h)
|
||||
if err != nil {
|
||||
return AUMHash{}, fmt.Errorf("reading %v: %w", h, err)
|
||||
}
|
||||
parent, hasParent := a.Parent()
|
||||
if !hasParent {
|
||||
return AUMHash{}, errors.New("reached genesis AUM without intersecting with candidate ancestor")
|
||||
}
|
||||
|
||||
if verdict[parent]&retainAUMMask != 0 {
|
||||
// Includes candidateAncestor (has retainStateActive set)
|
||||
continue
|
||||
}
|
||||
if verdict[parent]&retainStateCandidate != 0 {
|
||||
// We've intersected with the active chain but haven't done so through
|
||||
// candidateAncestor. That means that we intersect the active chain
|
||||
// before candidateAncestor, hence candidateAncestor actually needs
|
||||
// to be earlier than it is now.
|
||||
candidateAncestor = parent
|
||||
didAdjustCandidateAncestor = true
|
||||
verdict[parent] |= retainStateAncestor
|
||||
|
||||
// There could be AUMs on the active chain between our new candidateAncestor
|
||||
// and the old one, make sure they are marked as retained.
|
||||
next := parent
|
||||
childLoop:
|
||||
for {
|
||||
children, err := storage.ChildAUMs(next)
|
||||
if err != nil {
|
||||
return AUMHash{}, fmt.Errorf("reading children %v: %w", next, err)
|
||||
}
|
||||
// While there can be many children of an AUM, there can only be
|
||||
// one child on the active chain (it will have retainStateCandidate set).
|
||||
for _, a := range children {
|
||||
h := a.Hash()
|
||||
if v := verdict[h]; v&retainStateCandidate != 0 && v&retainStateActive == 0 {
|
||||
verdict[h] |= retainStateAncestor
|
||||
next = h
|
||||
continue childLoop
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
nextIterScan = append(nextIterScan, parent)
|
||||
}
|
||||
toScan = nextIterScan
|
||||
}
|
||||
|
||||
// If candidateAncestor was adjusted backwards, then it may not be a checkpoint
|
||||
// (and hence a valid compaction candidate). If so, iterate backwards and adjust
|
||||
// the candidateAncestor till we find a checkpoint.
|
||||
if didAdjustCandidateAncestor {
|
||||
var next AUM
|
||||
if next, err = storage.AUM(candidateAncestor); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("searching for compaction target: %w", err)
|
||||
}
|
||||
|
||||
for {
|
||||
h := next.Hash()
|
||||
verdict[h] |= retainStateActive
|
||||
if next.MessageKind == AUMCheckpoint {
|
||||
candidateAncestor = h
|
||||
break
|
||||
}
|
||||
|
||||
parent, hasParent := next.Parent()
|
||||
if !hasParent {
|
||||
return AUMHash{}, errors.New("reached genesis AUM without finding an appropriate candidateAncestor")
|
||||
}
|
||||
if next, err = storage.AUM(parent); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("searching for compaction target: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidateAncestor, nil
|
||||
}
|
||||
|
||||
// markDescendantAUMs marks all children of a retained AUM as retained.
|
||||
func markDescendantAUMs(storage Chonk, verdict map[AUMHash]retainState) error {
|
||||
toScan := make([]AUMHash, 0, len(verdict))
|
||||
for h, v := range verdict {
|
||||
if v&retainAUMMask == 0 {
|
||||
continue // not marked, so dont need to mark descendants
|
||||
}
|
||||
toScan = append(toScan, h)
|
||||
}
|
||||
|
||||
for len(toScan) > 0 {
|
||||
nextIterScan := make([]AUMHash, 0, len(verdict))
|
||||
for _, h := range toScan {
|
||||
if verdict[h]&retainStateLeaf != 0 {
|
||||
// This AUM and its decendants have already been marked.
|
||||
continue
|
||||
}
|
||||
verdict[h] |= retainStateLeaf
|
||||
|
||||
children, err := storage.ChildAUMs(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range children {
|
||||
nextIterScan = append(nextIterScan, a.Hash())
|
||||
}
|
||||
}
|
||||
toScan = nextIterScan
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compact deletes old AUMs from storage, based on the parameters given in opts.
|
||||
func Compact(storage CompactableChonk, head AUMHash, opts CompactionOptions) (lastActiveAncestor AUMHash, err error) {
|
||||
if opts.MinChain == 0 {
|
||||
return AUMHash{}, errors.New("opts.MinChain must be set")
|
||||
}
|
||||
if opts.MinAge == 0 {
|
||||
return AUMHash{}, errors.New("opts.MinAge must be set")
|
||||
}
|
||||
|
||||
all, err := storage.AllAUMs()
|
||||
if err != nil {
|
||||
return AUMHash{}, fmt.Errorf("AllAUMs: %w", err)
|
||||
}
|
||||
verdict := make(map[AUMHash]retainState, len(all))
|
||||
for _, h := range all {
|
||||
verdict[h] = 0
|
||||
}
|
||||
|
||||
if lastActiveAncestor, err = markActiveChain(storage, verdict, opts.MinChain, head); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("marking active chain: %w", err)
|
||||
}
|
||||
if err := markYoungAUMs(storage, verdict, opts.MinAge); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("marking young AUMs: %w", err)
|
||||
}
|
||||
if err := markDescendantAUMs(storage, verdict); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("marking decendant AUMs: %w", err)
|
||||
}
|
||||
if lastActiveAncestor, err = markAncestorIntersectionAUMs(storage, verdict, lastActiveAncestor); err != nil {
|
||||
return AUMHash{}, fmt.Errorf("marking ancestor intersection: %w", err)
|
||||
}
|
||||
|
||||
toDelete := make([]AUMHash, 0, len(verdict))
|
||||
for h, v := range verdict {
|
||||
if v&retainAUMMask == 0 { // no retention set
|
||||
toDelete = append(toDelete, h)
|
||||
}
|
||||
}
|
||||
|
||||
return lastActiveAncestor, storage.PurgeAUMs(toDelete)
|
||||
}
|
||||
|
|
|
@ -4,12 +4,15 @@
|
|||
package tka
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"golang.org/x/crypto/blake2s"
|
||||
)
|
||||
|
||||
|
@ -171,3 +174,431 @@ func TestTailchonkFS_Commit(t *testing.T) {
|
|||
t.Errorf("stat of AUM parent failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkActiveChain(t *testing.T) {
|
||||
type aumTemplate struct {
|
||||
AUM AUM
|
||||
}
|
||||
|
||||
tcs := []struct {
|
||||
name string
|
||||
minChain int
|
||||
chain []aumTemplate
|
||||
expectLastActiveIdx int // expected lastActiveAncestor, corresponds to an index on chain.
|
||||
}{
|
||||
{
|
||||
name: "genesis",
|
||||
minChain: 2,
|
||||
chain: []aumTemplate{
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
},
|
||||
expectLastActiveIdx: 0,
|
||||
},
|
||||
{
|
||||
name: "simple truncate",
|
||||
minChain: 2,
|
||||
chain: []aumTemplate{
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
},
|
||||
expectLastActiveIdx: 1,
|
||||
},
|
||||
{
|
||||
name: "long truncate",
|
||||
minChain: 5,
|
||||
chain: []aumTemplate{
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
},
|
||||
expectLastActiveIdx: 2,
|
||||
},
|
||||
{
|
||||
name: "truncate finding checkpoint",
|
||||
minChain: 2,
|
||||
chain: []aumTemplate{
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMAddKey, Key: &Key{}}}, // Should keep searching upwards for a checkpoint
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
|
||||
},
|
||||
expectLastActiveIdx: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
verdict := make(map[AUMHash]retainState, len(tc.chain))
|
||||
|
||||
// Build the state of the tailchonk for tests.
|
||||
storage := &Mem{}
|
||||
var prev AUMHash
|
||||
for i := range tc.chain {
|
||||
if !prev.IsZero() {
|
||||
tc.chain[i].AUM.PrevAUMHash = make([]byte, len(prev[:]))
|
||||
copy(tc.chain[i].AUM.PrevAUMHash, prev[:])
|
||||
}
|
||||
if err := storage.CommitVerifiedAUMs([]AUM{tc.chain[i].AUM}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := tc.chain[i].AUM.Hash()
|
||||
prev = h
|
||||
verdict[h] = 0
|
||||
}
|
||||
|
||||
got, err := markActiveChain(storage, verdict, tc.minChain, prev)
|
||||
if err != nil {
|
||||
t.Logf("state = %+v", verdict)
|
||||
t.Fatalf("markActiveChain() failed: %v", err)
|
||||
}
|
||||
want := tc.chain[tc.expectLastActiveIdx].AUM.Hash()
|
||||
if got != want {
|
||||
t.Logf("state = %+v", verdict)
|
||||
t.Errorf("lastActiveAncestor = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// Make sure the verdict array was marked correctly.
|
||||
for i := range tc.chain {
|
||||
h := tc.chain[i].AUM.Hash()
|
||||
if i >= tc.expectLastActiveIdx {
|
||||
if (verdict[h] & retainStateActive) == 0 {
|
||||
t.Errorf("verdict[%v] = %v, want %v set", h, verdict[h], retainStateActive)
|
||||
}
|
||||
} else {
|
||||
if (verdict[h] & retainStateCandidate) == 0 {
|
||||
t.Errorf("verdict[%v] = %v, want %v set", h, verdict[h], retainStateCandidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkDescendantAUMs(t *testing.T) {
|
||||
c := newTestchain(t, `
|
||||
genesis -> B -> C -> C2
|
||||
| -> D
|
||||
| -> E -> F -> G -> H
|
||||
| -> E2
|
||||
|
||||
// tweak seeds so hashes arent identical
|
||||
C.hashSeed = 1
|
||||
D.hashSeed = 2
|
||||
E.hashSeed = 3
|
||||
E2.hashSeed = 4
|
||||
`)
|
||||
|
||||
verdict := make(map[AUMHash]retainState, len(c.AUMs))
|
||||
for _, a := range c.AUMs {
|
||||
verdict[a.Hash()] = 0
|
||||
}
|
||||
|
||||
// Mark E & C.
|
||||
verdict[c.AUMHashes["C"]] = retainStateActive
|
||||
verdict[c.AUMHashes["E"]] = retainStateActive
|
||||
|
||||
if err := markDescendantAUMs(c.Chonk(), verdict); err != nil {
|
||||
t.Errorf("markDescendantAUMs() failed: %v", err)
|
||||
}
|
||||
|
||||
// Make sure the descendants got marked.
|
||||
hs := c.AUMHashes
|
||||
for _, h := range []AUMHash{hs["C2"], hs["F"], hs["G"], hs["H"], hs["E2"]} {
|
||||
if (verdict[h] & retainStateLeaf) == 0 {
|
||||
t.Errorf("%v was not marked as a descendant", h)
|
||||
}
|
||||
}
|
||||
for _, h := range []AUMHash{hs["genesis"], hs["B"], hs["D"]} {
|
||||
if (verdict[h] & retainStateLeaf) != 0 {
|
||||
t.Errorf("%v was marked as a descendant and shouldnt be", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkAncestorIntersectionAUMs(t *testing.T) {
|
||||
fakeState := &State{
|
||||
Keys: []Key{{Kind: Key25519, Votes: 1}},
|
||||
DisablementSecrets: [][]byte{bytes.Repeat([]byte{1}, 32)},
|
||||
}
|
||||
|
||||
tcs := []struct {
|
||||
name string
|
||||
chain *testChain
|
||||
verdicts map[string]retainState
|
||||
initialAncestor string
|
||||
wantAncestor string
|
||||
wantRetained []string
|
||||
wantDeleted []string
|
||||
}{
|
||||
{
|
||||
name: "genesis",
|
||||
chain: newTestchain(t, `
|
||||
A
|
||||
A.template = checkpoint`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
|
||||
initialAncestor: "A",
|
||||
wantAncestor: "A",
|
||||
verdicts: map[string]retainState{
|
||||
"A": retainStateActive,
|
||||
},
|
||||
wantRetained: []string{"A"},
|
||||
},
|
||||
{
|
||||
name: "no adjustment",
|
||||
chain: newTestchain(t, `
|
||||
DEAD -> A -> B -> C
|
||||
A.template = checkpoint
|
||||
B.template = checkpoint`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
|
||||
initialAncestor: "A",
|
||||
wantAncestor: "A",
|
||||
verdicts: map[string]retainState{
|
||||
"A": retainStateActive,
|
||||
"B": retainStateActive,
|
||||
"C": retainStateActive,
|
||||
"DEAD": retainStateCandidate,
|
||||
},
|
||||
wantRetained: []string{"A", "B", "C"},
|
||||
wantDeleted: []string{"DEAD"},
|
||||
},
|
||||
{
|
||||
name: "fork",
|
||||
chain: newTestchain(t, `
|
||||
A -> B -> C -> D
|
||||
| -> FORK
|
||||
A.template = checkpoint
|
||||
C.template = checkpoint
|
||||
D.template = checkpoint
|
||||
FORK.hashSeed = 2`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
|
||||
initialAncestor: "D",
|
||||
wantAncestor: "C",
|
||||
verdicts: map[string]retainState{
|
||||
"A": retainStateCandidate,
|
||||
"B": retainStateCandidate,
|
||||
"C": retainStateCandidate,
|
||||
"D": retainStateActive,
|
||||
"FORK": retainStateYoung,
|
||||
},
|
||||
wantRetained: []string{"C", "D", "FORK"},
|
||||
wantDeleted: []string{"A", "B"},
|
||||
},
|
||||
{
|
||||
name: "fork finding earlier checkpoint",
|
||||
chain: newTestchain(t, `
|
||||
A -> B -> C -> D -> E -> F
|
||||
| -> FORK
|
||||
A.template = checkpoint
|
||||
B.template = checkpoint
|
||||
E.template = checkpoint
|
||||
FORK.hashSeed = 2`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
|
||||
initialAncestor: "E",
|
||||
wantAncestor: "B",
|
||||
verdicts: map[string]retainState{
|
||||
"A": retainStateCandidate,
|
||||
"B": retainStateCandidate,
|
||||
"C": retainStateCandidate,
|
||||
"D": retainStateCandidate,
|
||||
"E": retainStateActive,
|
||||
"F": retainStateActive,
|
||||
"FORK": retainStateYoung,
|
||||
},
|
||||
wantRetained: []string{"B", "C", "D", "E", "F", "FORK"},
|
||||
wantDeleted: []string{"A"},
|
||||
},
|
||||
{
|
||||
name: "fork multi",
|
||||
chain: newTestchain(t, `
|
||||
A -> B -> C -> D -> E
|
||||
| -> DEADFORK
|
||||
C -> FORK
|
||||
A.template = checkpoint
|
||||
C.template = checkpoint
|
||||
D.template = checkpoint
|
||||
E.template = checkpoint
|
||||
FORK.hashSeed = 2
|
||||
DEADFORK.hashSeed = 3`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
|
||||
initialAncestor: "D",
|
||||
wantAncestor: "C",
|
||||
verdicts: map[string]retainState{
|
||||
"A": retainStateCandidate,
|
||||
"B": retainStateCandidate,
|
||||
"C": retainStateCandidate,
|
||||
"D": retainStateActive,
|
||||
"E": retainStateActive,
|
||||
"FORK": retainStateYoung,
|
||||
"DEADFORK": 0,
|
||||
},
|
||||
wantRetained: []string{"C", "D", "E", "FORK"},
|
||||
wantDeleted: []string{"A", "B", "DEADFORK"},
|
||||
},
|
||||
{
|
||||
name: "fork multi 2",
|
||||
chain: newTestchain(t, `
|
||||
A -> B -> C -> D -> E -> F -> G
|
||||
|
||||
F -> F1
|
||||
D -> F2
|
||||
B -> F3
|
||||
|
||||
A.template = checkpoint
|
||||
B.template = checkpoint
|
||||
D.template = checkpoint
|
||||
F.template = checkpoint
|
||||
F1.hashSeed = 2
|
||||
F2.hashSeed = 3
|
||||
F3.hashSeed = 4`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
|
||||
initialAncestor: "F",
|
||||
wantAncestor: "B",
|
||||
verdicts: map[string]retainState{
|
||||
"A": retainStateCandidate,
|
||||
"B": retainStateCandidate,
|
||||
"C": retainStateCandidate,
|
||||
"D": retainStateCandidate,
|
||||
"E": retainStateCandidate,
|
||||
"F": retainStateActive,
|
||||
"G": retainStateActive,
|
||||
"F1": retainStateYoung,
|
||||
"F2": retainStateYoung,
|
||||
"F3": retainStateYoung,
|
||||
},
|
||||
wantRetained: []string{"B", "C", "D", "E", "F", "G", "F1", "F2", "F3"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
verdict := make(map[AUMHash]retainState, len(tc.verdicts))
|
||||
for name, v := range tc.verdicts {
|
||||
verdict[tc.chain.AUMHashes[name]] = v
|
||||
}
|
||||
|
||||
got, err := markAncestorIntersectionAUMs(tc.chain.Chonk(), verdict, tc.chain.AUMHashes[tc.initialAncestor])
|
||||
if err != nil {
|
||||
t.Logf("state = %+v", verdict)
|
||||
t.Fatalf("markAncestorIntersectionAUMs() failed: %v", err)
|
||||
}
|
||||
if want := tc.chain.AUMHashes[tc.wantAncestor]; got != want {
|
||||
t.Logf("state = %+v", verdict)
|
||||
t.Errorf("lastActiveAncestor = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
for _, name := range tc.wantRetained {
|
||||
h := tc.chain.AUMHashes[name]
|
||||
if v := verdict[h]; v&retainAUMMask == 0 {
|
||||
t.Errorf("AUM %q was not retained: verdict = %v", name, v)
|
||||
}
|
||||
}
|
||||
for _, name := range tc.wantDeleted {
|
||||
h := tc.chain.AUMHashes[name]
|
||||
if v := verdict[h]; v&retainAUMMask != 0 {
|
||||
t.Errorf("AUM %q was retained: verdict = %v", name, v)
|
||||
}
|
||||
}
|
||||
|
||||
if t.Failed() {
|
||||
for name, hash := range tc.chain.AUMHashes {
|
||||
t.Logf("AUM[%q] = %v", name, hash)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type compactingChonkFake struct {
|
||||
Mem
|
||||
|
||||
aumAge map[AUMHash]time.Time
|
||||
t *testing.T
|
||||
wantDelete []AUMHash
|
||||
}
|
||||
|
||||
func (c *compactingChonkFake) AllAUMs() ([]AUMHash, error) {
|
||||
out := make([]AUMHash, 0, len(c.Mem.aums))
|
||||
for h, _ := range c.Mem.aums {
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *compactingChonkFake) CommitTime(hash AUMHash) (time.Time, error) {
|
||||
return c.aumAge[hash], nil
|
||||
}
|
||||
|
||||
func (c *compactingChonkFake) PurgeAUMs(hashes []AUMHash) error {
|
||||
lessHashes := func(a, b AUMHash) bool {
|
||||
return bytes.Compare(a[:], b[:]) < 0
|
||||
}
|
||||
if diff := cmp.Diff(c.wantDelete, hashes, cmpopts.SortSlices(lessHashes)); diff != "" {
|
||||
c.t.Errorf("deletion set differs (-want, +got):\n%s", diff)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestCompact(t *testing.T) {
|
||||
fakeState := &State{
|
||||
Keys: []Key{{Kind: Key25519, Votes: 1}},
|
||||
DisablementSecrets: [][]byte{bytes.Repeat([]byte{1}, 32)},
|
||||
}
|
||||
|
||||
// A & B are deleted because the new lastActiveAncestor advances beyond them.
|
||||
// OLD is deleted because it does not match retention criteria, and
|
||||
// though it is a descendant of the new lastActiveAncestor (C), it is not a
|
||||
// descendant of a retained AUM.
|
||||
// G, & H are retained as recent (MinChain=2) ancestors of HEAD.
|
||||
// E & F are retained because they are between retained AUMs (G+) and
|
||||
// their newest checkpoint ancestor.
|
||||
// D is retained because it is the newest checkpoint ancestor from
|
||||
// MinChain-retained AUMs.
|
||||
// G2 is retained because it is a descendant of a retained AUM (G).
|
||||
// F1 is retained because it is new enough by wall-clock time.
|
||||
// F2 is retained because it is a descendant of a retained AUM (F1).
|
||||
// C2 is retained because it is between an ancestor checkpoint and
|
||||
// a retained AUM (F1).
|
||||
// C is retained because it is the new lastActiveAncestor. It is the
|
||||
// new lastActiveAncestor because it is the newest common checkpoint
|
||||
// of all retained AUMs.
|
||||
c := newTestchain(t, `
|
||||
A -> B -> C -> C2 -> D -> E -> F -> G -> H
|
||||
| -> F1 -> F2 | -> G2
|
||||
| -> OLD
|
||||
|
||||
// make {A,B,C,D} compaction candidates
|
||||
A.template = checkpoint
|
||||
B.template = checkpoint
|
||||
C.template = checkpoint
|
||||
D.template = checkpoint
|
||||
|
||||
// tweak seeds of forks so hashes arent identical
|
||||
F1.hashSeed = 1
|
||||
OLD.hashSeed = 2
|
||||
G2.hashSeed = 3
|
||||
`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState}))
|
||||
|
||||
storage := &compactingChonkFake{
|
||||
Mem: (*c.Chonk().(*Mem)),
|
||||
aumAge: map[AUMHash]time.Time{(c.AUMHashes["F1"]): time.Now()},
|
||||
t: t,
|
||||
wantDelete: []AUMHash{c.AUMHashes["A"], c.AUMHashes["B"], c.AUMHashes["OLD"]},
|
||||
}
|
||||
|
||||
lastActiveAncestor, err := Compact(storage, c.AUMHashes["H"], CompactionOptions{MinChain: 2, MinAge: time.Hour})
|
||||
if err != nil {
|
||||
t.Errorf("Compact() failed: %v", err)
|
||||
}
|
||||
if lastActiveAncestor != c.AUMHashes["C"] {
|
||||
t.Errorf("last active ancestor = %v, want %v", lastActiveAncestor, c.AUMHashes["C"])
|
||||
}
|
||||
|
||||
if t.Failed() {
|
||||
for name, hash := range c.AUMHashes {
|
||||
t.Logf("AUM[%q] = %v", name, hash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue