tendermint, PEX

PEX handles peer exchange, finding more peers and building the p2p network.

PEX is a reactor working together with Switch.

Reactor

type Reactor struct {
	p2p.BaseReactor

	book              AddrBook
	config            *ReactorConfig
	ensurePeersPeriod time.Duration // TODO: should go in the config

	// maps to prevent abuse
	requestsSent         *cmap.CMap // ID->struct{}: unanswered send requests
	lastReceivedRequests *cmap.CMap // ID->time.Time: last time peer requested from us

	seedAddrs []*p2p.NetAddress

	attemptsToDial sync.Map // address (string) -> {number of attempts (int), last time dialed (time.Time)}

	// seed/crawled mode fields
	crawlPeerInfos map[p2p.ID]crawlPeerInfo
}

// NewReactor creates new PEX reactor.
func NewReactor(b AddrBook, config *ReactorConfig) *Reactor {
	r := &Reactor{
		book:                 b,
		config:               config,
		ensurePeersPeriod:    defaultEnsurePeersPeriod,
		requestsSent:         cmap.NewCMap(),
		lastReceivedRequests: cmap.NewCMap(),
		crawlPeerInfos:       make(map[p2p.ID]crawlPeerInfo),
	}
	r.BaseReactor = *p2p.NewBaseReactor("Reactor", r)
	return r
}

Config

// ReactorConfig holds reactor specific configuration data.
type ReactorConfig struct {
	// Seed/Crawler mode
	SeedMode bool

	// We want seeds to only advertise good peers. Therefore they should wait at
	// least as long as we expect it to take for a peer to become good before
	// disconnecting.
	SeedDisconnectWaitPeriod time.Duration

	// Maximum pause when redialing a persistent peer (if zero, exponential backoff is used)
	PersistentPeersMaxDialPeriod time.Duration

	// Seeds is a list of addresses reactor may use
	// if it can't connect to peers in the addrbook.
	Seeds []string
}

Channels & Messages

	// PexChannel is a channel for PEX messages
	PexChannel = byte(0x00)	

/*
A pexRequestMessage requests additional peer addresses.
*/
type pexRequestMessage struct {
}
/*
A message with announced peer addresses.
*/
type pexAddrsMessage struct {
	Addrs []*p2p.NetAddress
}

Start

If working on seed mode, PEX starts crawlPeersRoutine(), or ensurePeersRoutine().

// OnStart implements BaseService
func (r *Reactor) OnStart() error {
	err := r.book.Start()
	if err != nil && err != service.ErrAlreadyStarted {
		return err
	}

	numOnline, seedAddrs, err := r.checkSeeds()
	if err != nil {
		return err
	} else if numOnline == 0 && r.book.Empty() {
		return errors.New("address book is empty and couldn't resolve any seed nodes")
	}

	r.seedAddrs = seedAddrs

	// Check if this node should run
	// in seed/crawler mode
	if r.config.SeedMode {
		go r.crawlPeersRoutine()
	} else {
		go r.ensurePeersRoutine()
	}
	return nil
}

crawlPeersRoutine()

It is the Seed Mode.

// Explores the network searching for more peers. (continuous)
// Seed/Crawler Mode causes this node to quickly disconnect
// from peers, except other seed nodes.
func (r *Reactor) crawlPeersRoutine() {
	// If we have any seed nodes, consult them first
	if len(r.seedAddrs) > 0 {
		r.dialSeeds()
	} else {
		// Do an initial crawl
		r.crawlPeers(r.book.GetSelection())
	}

	// Fire periodically
	ticker := time.NewTicker(crawlPeerPeriod)

	for {
		select {
		case <-ticker.C:
			r.attemptDisconnects()
			r.crawlPeers(r.book.GetSelection())
			r.cleanupCrawlPeerInfos()
		case <-r.Quit():
			return
		}
	}
}

ensurePeersRoutine()

It picks up some addresses from AddrBook then dials them. Besides, it will pick a random peer from Switch peer list and then request for more peer addresses. At last, if it is not dialing any peer, it trys dialing the seed node.

// Ensures that sufficient peers are connected. (continuous)
func (r *Reactor) ensurePeersRoutine() {
	var (
		seed   = rand.NewRand()
		jitter = seed.Int63n(r.ensurePeersPeriod.Nanoseconds())
	)

	// Randomize first round of communication to avoid thundering herd.
	// If no peers are present directly start connecting so we guarantee swift
	// setup with the help of configured seeds.
	if r.nodeHasSomePeersOrDialingAny() {
		time.Sleep(time.Duration(jitter))
	}

	// fire once immediately.
	// ensures we dial the seeds right away if the book is empty
	r.ensurePeers()

	// fire periodically
	ticker := time.NewTicker(r.ensurePeersPeriod)
	for {
		select {
		case <-ticker.C:
			r.ensurePeers()
		case <-r.Quit():
			ticker.Stop()
			return
		}
	}
}

AddPeer & RemovePeer

PEX would requests more peers if need when a new outbound peer is added to Switch. As for an inbound peer, PEX adds it and probably asks it for more peer if it is ensured to be trustworthy.

// AddPeer implements Reactor by adding peer to the address book (if inbound)
// or by requesting more addresses (if outbound).
func (r *Reactor) AddPeer(p Peer) {
	if p.IsOutbound() {
		// For outbound peers, the address is already in the books -
		// either via DialPeersAsync or r.Receive.
		// Ask it for more peers if we need.
		if r.book.NeedMoreAddrs() {
			r.RequestAddrs(p)
		}
	} else {
		// inbound peer is its own source
		addr, err := p.NodeInfo().NetAddress()
		if err != nil {
			r.Logger.Error("Failed to get peer NetAddress", "err", err, "peer", p)
			return
		}

		// Make it explicit that addr and src are the same for an inbound peer.
		src := addr

		// add to book. dont RequestAddrs right away because
		// we don't trust inbound as much - let ensurePeersRoutine handle it.
		err = r.book.AddAddress(addr, src)
		r.logErrAddrBook(err)
	}
}

// RemovePeer implements Reactor by resetting peer's requests info.
func (r *Reactor) RemovePeer(p Peer, reason interface{}) {
	id := string(p.ID())
	r.requestsSent.Delete(id)
	r.lastReceivedRequests.Delete(id)
}

Receive

It handles pexRequestMessage and pexAddrsMessage.

pexRequestMessage

A Seed node doesn’t take too many connections with peers.

If we are running at Seed Mode and the message comes from an inbound peer, we select some addresses from AddrBook and send back with pexAddrsMessage. After all, we disconnect the peer.

Otherwise, we enforce a minimum amount of time between requests, then send back some random addresses wiht pexAddrsMessage.

pexAddrsMessage

We simly check if there is an open request, and delete it if so, then add peer address to AddrBook. At the same time, if any address matches with the known seed node, we start a connection towards the seed node immediately. This is to find out more peers as soon as possible.

dialPeer

A helper function to dial the address. It has some strategy to make dialing and mark the address if dialing failed.

func (r *Reactor) dialPeer(addr *p2p.NetAddress) error {
	attempts, lastDialed := r.dialAttemptsInfo(addr)
	if !r.Switch.IsPeerPersistent(addr) && attempts > maxAttemptsToDial {
		// TODO(melekes): have a blacklist in the addrbook with peers whom we've
		// failed to connect to. Then we can clean up attemptsToDial, which acts as
		// a blacklist currently.
		// https://github.com/tendermint/tendermint/issues/3572
		r.book.MarkBad(addr)
		return errMaxAttemptsToDial{}
	}

	// exponential backoff if it's not our first attempt to dial given address
	if attempts > 0 {
		jitterSeconds := time.Duration(tmrand.Float64() * float64(time.Second)) // 1s == (1e9 ns)
		backoffDuration := jitterSeconds + ((1 << uint(attempts)) * time.Second)
		backoffDuration = r.maxBackoffDurationForPeer(addr, backoffDuration)
		sinceLastDialed := time.Since(lastDialed)
		if sinceLastDialed < backoffDuration {
			return errTooEarlyToDial{backoffDuration, lastDialed}
		}
	}

	err := r.Switch.DialPeerWithAddress(addr)
	if err != nil {
		if _, ok := err.(p2p.ErrCurrentlyDialingOrExistingAddress); ok {
			return err
		}

		markAddrInBookBasedOnErr(addr, r.book, err)
		switch err.(type) {
		case p2p.ErrSwitchAuthenticationFailure:
			// NOTE: addr is removed from addrbook in markAddrInBookBasedOnErr
			r.attemptsToDial.Delete(addr.DialString())
		default:
			r.attemptsToDial.Store(addr.DialString(), _attemptsToDial{attempts + 1, time.Now()})
		}
		return errors.Wrapf(err, "dialing failed (attempts: %d)", attempts+1)
	}

	// cleanup any history
	r.attemptsToDial.Delete(addr.DialString())
	return nil
}

AddrBook

PEX manages peers via AddrBook interface. AddrBook is also a Service which follows the lifecycle of a service model.

It maintains the ‘new’ and ‘old’ address bucket, holding all addresses.

// AddrBook is an address book used for tracking peers
// so we can gossip about them to others and select
// peers to dial.
// TODO: break this up?
type AddrBook interface {
	service.Service

	// Add our own addresses so we don't later add ourselves
	AddOurAddress(*p2p.NetAddress)
	// Check if it is our address
	OurAddress(*p2p.NetAddress) bool

	AddPrivateIDs([]string)

	// Add and remove an address
	AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error
	RemoveAddress(*p2p.NetAddress)

	// Check if the address is in the book
	HasAddress(*p2p.NetAddress) bool

	// Do we need more peers?
	NeedMoreAddrs() bool
	// Is Address Book Empty? Answer should not depend on being in your own
	// address book, or private peers
	Empty() bool

	// Pick an address to dial
	PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress

	// Mark address
	MarkGood(p2p.ID)
	MarkAttempt(*p2p.NetAddress)
	MarkBad(*p2p.NetAddress)

	IsGood(*p2p.NetAddress) bool

	// Send a selection of addresses to peers
	GetSelection() []*p2p.NetAddress
	// Send a selection of addresses with bias
	GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress

	Size() int

	// Persist to disk
	Save()
}

// addrBook - concurrency safe peer address manager.
// Implements AddrBook.
type addrBook struct {
	service.BaseService

	// accessed concurrently
	mtx        sync.Mutex
	rand       *tmrand.Rand
	ourAddrs   map[string]struct{}
	privateIDs map[p2p.ID]struct{}
	addrLookup map[p2p.ID]*knownAddress // new & old
	bucketsOld []map[string]*knownAddress
	bucketsNew []map[string]*knownAddress
	nOld       int
	nNew       int

	// immutable after creation
	filePath          string
	key               string // random prefix for bucket placement
	routabilityStrict bool

	wg sync.WaitGroup
}

Initialization

AddrBook is created in NewNode(), set to Switch, and then passed to PEX.

// NewAddrBook creates a new address book.
// Use Start to begin processing asynchronous address updates.
func NewAddrBook(filePath string, routabilityStrict bool) *addrBook {
	am := &addrBook{
		rand:              tmrand.NewRand(),
		ourAddrs:          make(map[string]struct{}),
		privateIDs:        make(map[p2p.ID]struct{}),
		addrLookup:        make(map[p2p.ID]*knownAddress),
		filePath:          filePath,
		routabilityStrict: routabilityStrict,
	}
	am.init()
	am.BaseService = *service.NewBaseService(nil, "AddrBook", am)
	return am
}

Start

Started by PEX start(), it will load address from file addrbook.json, then start a go routine to update the file in a fixed interval - 2m by default.

// OnStart implements Service.
func (a *addrBook) OnStart() error {
	if err := a.BaseService.OnStart(); err != nil {
		return err
	}
	a.loadFromFile(a.filePath)

	// wg.Add to ensure that any invocation of .Wait()
	// later on will wait for saveRoutine to terminate.
	a.wg.Add(1)
	go a.saveRoutine()

	return nil
}

github.com/tendermint/tendermint/p2p/pex.(*addrBook).OnStart at addrbook.go:140
github.com/tendermint/tendermint/libs/service.(*BaseService).Start at service.go:139
github.com/tendermint/tendermint/p2p/pex.(*Reactor).OnStart at pex_reactor.go:146
github.com/tendermint/tendermint/libs/service.(*BaseService).Start at service.go:139
github.com/tendermint/tendermint/p2p.(*Switch).OnStart at switch.go:227
github.com/tendermint/tendermint/libs/service.(*BaseService).Start at service.go:139
github.com/tendermint/tendermint/node.(*Node).OnStart at node.go:791
github.com/tendermint/tendermint/libs/service.(*BaseService).Start at service.go:139
github.com/hyperledger/burrow/core.TendermintLauncher.func1 at processes.go:165
github.com/hyperledger/burrow/core.(*Kernel).Boot at kernel.go:259
github.com/hyperledger/burrow/cmd/burrow/commands.Start.func1.1 at start.go:31
github.com/jawher/mow.cli/internal/flow.(*Step).callDo at flow.go:55
github.com/jawher/mow.cli/internal/flow.(*Step).Run at flow.go:25
github.com/jawher/mow.cli/internal/flow.(*Step).Run at flow.go:29
github.com/jawher/mow.cli/internal/flow.(*Step).Run at flow.go:29
github.com/jawher/mow.cli/internal/flow.(*Step).Run at flow.go:29
github.com/jawher/mow%2ecli.(*Cmd).parse at commands.go:681
github.com/jawher/mow%2ecli.(*Cmd).parse at commands.go:695
github.com/jawher/mow%2ecli.(*Cli).parse at cli.go:76
github.com/jawher/mow%2ecli.(*Cli).Run at cli.go:105
main.main at main.go:15
runtime.main at proc.go:203
runtime.goexit at asm_amd64.s:1357
 - Async stack trace
runtime.rt0_go at asm_amd64.s:220

func (a *addrBook) saveRoutine() {
	defer a.wg.Done()

	saveFileTicker := time.NewTicker(dumpAddressInterval)
out:
	for {
		select {
		case <-saveFileTicker.C:
			a.saveToFile(a.filePath)
		case <-a.Quit():
			break out
		}
	}
	saveFileTicker.Stop()
	a.saveToFile(a.filePath)
}

Basic Operations

• AddAddress, add address to a “new” bucket
• RemoveAddress, removes the address from the book
• HasAddress
• NeedMoreAddrs, number of known addresses < needAddressThreshold (1000)
• PickAddress, picks an address to connect to. The address is picked randomly from an old or new bucket according to the biasTowardsNewAddrs argument
• MarkGood, marks the peer as good and/ moves it into an “old” bucket
• MarkAttempt, marks that an attempt was made to connect to the address
• MarkBad, currently remove the address
• GetSelection, select some addresses randomly
• GetSelectionWithBias, select some addresses randomly according to the biasTowardsNewAddrs argument