Using the three pillars of observability to build, test, and release new products.

20. ✔

30. ✔

37. ✔

50. package prometheus
// Collector is the interface implemented by anything that can be used by
// Prometheus to collect metrics. A Collector has to be registered for
// collection. See Registerer.Register.
type Collector interface {
// Describe sends the super-set of all possible descriptors of metrics
// collected by this Collector to the provided channel and returns once
// the last descriptor has been sent.
Describe(chan<- *Desc)
// Collect is called by the Prometheus registry when collecting
// metrics. The implementation sends each collected metric via the
// provided
Collect(chan<- Metric)
}

70. ✔

73. Service A
Service B
Service C
Queue
Datastore
Service B
Service B
Service C
Service C

75. dorpc.SetLogger(loglvl.Level(
log.KV("component", "grpc"),
loglvl.Error,
))

76. mm, metricsHandler :=
metrics.Prometheus(metrics.PrometheusNamespace(app))
mm = mm.Labels(metrics.L{
"app": app,
"env": env.Env,
})
metrics.SetRoot(mm)

77. func (s *server) initalizeMetrics() {
s.metrics = metricsConfig{
attemptedConvergeChassis:
s.metricsNode.Gauge("attempted_converge_chassis", "number of
chassis converger attempting to converge"),
failedConvergeChassis:
s.metricsNode.Gauge("failed_converge_chassis", "number of
chassis that failed to converge"),
}
}

78. dbDriverName, traceOpt := tracemysql.MySQL()
// note: default buffer size is 64 * 1 << 10
tracerOpt := tracer.WithBufferSize(64 * 1 << 14)
bufferOpt := trace.UseTracerOptions(tracerOpt)
tr := trace.InitTracer(traceOpt, bufferOpt)
defer tr.Close()

86. Service A
Service B
Service C
QueueDatastore
Service B
Service B
Service C
Service C

91. Service B
DHCP server
Virtual
Switch
Service A RPC calls
Iface Y
DHCPv4
gRPC
DHCPv6
Iface X dropletX
DHCP traffic
RPC calls
Hypervisor

93. package dhcp4conn
var _ prometheus.Collector = &collector{}
type collector struct {
ReadBytesTotal *prometheus.Desc
ReadPacketsTotal *prometheus.Desc
WriteBytesTotal *prometheus.Desc
WritePacketsTotal *prometheus.Desc
}

100. workC := make(chan request, Workers)
for i := 0; i < Workers; i++ {
go func() {
defer workWG.Done()
for r := range workC {
s.serve(r.buf, r.from)
}
}()
}

101. type RateMap struct {
mu sync.Mutex
...
rateMap map[string]*rate
}
type RateMapCollector struct {
RequestRate *prometheus.Desc
rm *RateMap
buckets []float64
}
func (r *RateMapCollector) Collect(ch
chan<- prometheus.Metric) {
...
ch <-
prometheus.MustNewConstHistogram(
r.RequestRate,
count, sum,
rateCount)
}

102. Rate Limiter
Centralized
LoggingCentralized
LoggingCentralized
LoggingCentralized
Logging
Elastalert
emits log line

105. ●
●
●