| 2 | (window.GOFRAMEWORK_FLOWS = window.GOFRAMEWORK_FLOWS || []).push({"id":"metrics","title":"Metrics pushed through an outage","group":"metrics","summary":"A program counts requests and times them; metrics-exporters-otlp pushes them every 10 seconds to an OpenTelemetry receiver, which refuses connections, then hangs, then is busy, then comes back. The program never waits, nothing piles up, and no count is lost.","scenario":"registry := metrics.NewRegistry()\nrequests := registry.Counter(\"requests\", \"Requests handled.\")\nlatency := registry.Histogram(\"latency\", \"Time to handle a request.\", metrics.Seconds)\n\nexporter := otlp.New(registry,\n\totlp.Endpoint(\"http://collector:4318\"),\n\totlp.Every(10*time.Second),\n\totlp.Timeout(3*time.Second))\n\n// on each request\nrequests.Inc()\nlatency.Observe(took.Seconds())\n\n// at exit\nexporter.Close(ctx)","recorded":{"date":"2026-10-17","go":"go1.27.2","clock":"testing/synctest's fake clock: the times are the program's own, without waiting for them","modules":{"codecs-data-protobuf":"40a7234","metrics-exporters-otlp":"3d96ea4","metrics-spec":"bac0dff"}},"width":1100,"height":560,"nodes":[{"id":"app","kind":"outside","title":"Your code","x":30,"y":40,"w":220,"h":120},{"id":"registry","kind":"state","title":"Registry","module":"metrics-spec","x":30,"y":300,"w":220,"h":130},{"id":"pusher","kind":"decision","title":"Pusher","module":"metrics-spec","x":420,"y":40,"w":240,"h":150},{"id":"exporter","kind":"effect","title":"OTLP exporter","module":"metrics-exporters-otlp","x":420,"y":300,"w":240,"h":150},{"id":"receiver","kind":"outside","title":"OTLP receiver","x":830,"y":300,"w":240,"h":150}],"edges":[{"id":1,"from":"app","to":"registry","label":"Inc, Observe: one atomic add"},{"id":2,"from":"pusher","to":"registry","label":"Snapshot, every 10 s"},{"id":3,"from":"pusher","to":"exporter","label":"push the snapshot","bend":-14},{"id":4,"from":"exporter","to":"receiver","label":"POST /v1/metrics","bend":-14},{"id":5,"from":"receiver","to":"exporter","label":"the answer","bend":-14},{"id":6,"from":"exporter","to":"exporter","label":"pause, then retry"},{"id":7,"from":"exporter","to":"pusher","label":"the push's result","bend":-14},{"id":8,"from":"app","to":"pusher","label":"Close: one last push"}],"events":[{"chapter":0,"at":0,"node":"pusher","kind":"new","text":"otlp.New starts the Pusher: a push every 10 s, each request capped at 3 s.","state":{"app":{"lines":[["requests handled","0"]]},"exporter":{"lines":[["requests sent","0"],["retries","0"]]},"pusher":{"lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","0"],["holds latency","0 observations"]]},"registry":{"lines":[["requests","0"],["latency","0 observations"]]}}},{"chapter":0,"at":2,"edge":1,"node":"registry","kind":"record","text":"Your code handles 3 requests: requests.Inc() and latency.Observe(…) for each, one atomic add apiece.","state":{"app":{"lines":[["requests handled","3"]]},"exporter":{"lines":[["requests sent","0"],["retries","0"]]},"pusher":{"lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","0"],["holds latency","0 observations"]]},"registry":{"lines":[["requests","3"],["latency","3 observations"]]}}},{"chapter":0,"at":6,"edge":1,"node":"registry","kind":"record","text":"Your code handles 2 requests: requests.Inc() and latency.Observe(…) for each, one atomic add apiece.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"lines":[["requests sent","0"],["retries","0"]]},"pusher":{"lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","0"],["holds latency","0 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":0,"at":10,"edge":2,"node":"registry","kind":"snapshot","text":"The ticker fires: the Pusher reads a snapshot of the registry, requests = 5.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"status":"busy","lines":[["requests sent","0"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","0"],["holds latency","0 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":0,"at":10,"edge":3,"node":"exporter","kind":"push","text":"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: 260 bytes, 213 gzipped.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"status":"busy","lines":[["requests sent","1"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","0"],["holds latency","0 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":0,"at":10,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 5 and 5 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"status":"busy","lines":[["requests sent","1"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","0"],["holds latency","0 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":0,"at":10,"edge":5,"node":"exporter","kind":"accepted","text":"200 OK: the receiver now holds requests = 5 and 5 latency observations.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"status":"busy","lines":[["requests sent","1"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","0"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":0,"at":10,"edge":7,"node":"pusher","kind":"done","text":"Push 1 done. The receiver is up to date.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"lines":[["requests sent","1"],["retries","0"]]},"pusher":{"lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":1,"at":12,"node":"receiver","kind":"receiver-down","text":"The receiver goes down: connections are refused.","state":{"app":{"lines":[["requests handled","5"]]},"exporter":{"lines":[["requests sent","1"],["retries","0"]]},"pusher":{"lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","5"],["latency","5 observations"]]}}},{"chapter":1,"at":15,"edge":1,"node":"registry","kind":"record","text":"Your code handles 4 requests: requests.Inc() and latency.Observe(…) for each, one atomic add apiece.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"lines":[["requests sent","1"],["retries","0"]]},"pusher":{"lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20,"edge":2,"node":"registry","kind":"snapshot","text":"The ticker fires: the Pusher reads a snapshot of the registry, requests = 9.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","1"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20,"edge":3,"node":"exporter","kind":"push","text":"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: 265 bytes, 218 gzipped.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","2"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","2"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20,"edge":5,"node":"exporter","kind":"refused","text":"Connection refused. OTLP says a failure to connect is worth retrying.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","2"],["retries","0"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20.662,"edge":6,"node":"exporter","kind":"pause","text":"After a pause of 0.66 s, the exporter tries again: attempt 2 of this push.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","3"],["retries","1"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20.662,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","3"],["retries","1"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":20.662,"edge":5,"node":"exporter","kind":"refused","text":"Connection refused. OTLP says a failure to connect is worth retrying.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","3"],["retries","1"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":22.049,"edge":6,"node":"exporter","kind":"pause","text":"After a pause of 1.39 s, the exporter tries again: attempt 3 of this push.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","4"],["retries","2"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":22.049,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","4"],["retries","2"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":22.049,"edge":5,"node":"exporter","kind":"refused","text":"Connection refused. OTLP says a failure to connect is worth retrying.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","4"],["retries","2"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"off","lines":[["state","refusing connections"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":24,"node":"receiver","kind":"receiver-hung","text":"The receiver hangs: it accepts connections and never answers.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","4"],["retries","2"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"error","lines":[["state","hung"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":24.87,"edge":6,"node":"exporter","kind":"pause","text":"After a pause of 2.82 s, the exporter tries again: attempt 4 of this push.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","5"],["retries","3"],["this push","attempt 4"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"error","lines":[["state","hung"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":24.87,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","5"],["retries","3"],["this push","attempt 4"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"error","lines":[["state","hung"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":27.87,"edge":5,"node":"exporter","kind":"timeout","text":"No answer: the attempt's 3 s Timeout ends it. The connection is dropped, nothing waits on it.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","5"],["retries","3"],["this push","attempt 4"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","0"]]},"receiver":{"status":"error","lines":[["state","hung"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":1,"at":27.87,"edge":7,"node":"pusher","kind":"failed","text":"Push 2 failed: context deadline exceeded. Nothing is queued: the next push carries these counts again.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"lines":[["requests sent","5"],["retries","3"]]},"pusher":{"lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","hung"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":29,"node":"receiver","kind":"receiver-busy","text":"The receiver is back but overloaded: it answers 503 with Retry-After: 2.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"lines":[["requests sent","5"],["retries","3"]]},"pusher":{"lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":30,"edge":2,"node":"registry","kind":"snapshot","text":"The ticker fires: the Pusher reads a snapshot of the registry, requests = 9.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","5"],["retries","3"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":30,"edge":3,"node":"exporter","kind":"push","text":"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: 265 bytes, 218 gzipped.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","6"],["retries","3"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":30,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","6"],["retries","3"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":30,"edge":5,"node":"exporter","kind":"busy","text":"503 Service Unavailable, Retry-After: 2. OTLP says to retry 503, waiting as long as asked.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","6"],["retries","3"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":32,"edge":6,"node":"exporter","kind":"pause","text":"After a pause of 2 s, the exporter tries again: attempt 2 of this push.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","7"],["retries","4"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":32,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","7"],["retries","4"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":32,"edge":5,"node":"exporter","kind":"busy","text":"503 Service Unavailable, Retry-After: 2. OTLP says to retry 503, waiting as long as asked.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","7"],["retries","4"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"error","lines":[["state","busy: 503"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":32,"node":"receiver","kind":"receiver-up","text":"The receiver is healthy again.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","7"],["retries","4"],["this push","attempt 2"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":34,"edge":6,"node":"exporter","kind":"pause","text":"After a pause of 2 s, the exporter tries again: attempt 3 of this push.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","8"],["retries","5"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":34,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 9 and 9 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","8"],["retries","5"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","5"],["holds latency","5 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":34,"edge":5,"node":"exporter","kind":"accepted","text":"200 OK: the receiver now holds requests = 9 and 9 latency observations.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"status":"busy","lines":[["requests sent","8"],["retries","5"],["this push","attempt 3"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","1"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":34,"edge":7,"node":"pusher","kind":"done","text":"Push 3 done. The receiver is up to date.","state":{"app":{"lines":[["requests handled","9"]]},"exporter":{"lines":[["requests sent","8"],["retries","5"]]},"pusher":{"lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","9"],["latency","9 observations"]]}}},{"chapter":2,"at":36,"edge":1,"node":"registry","kind":"record","text":"Your code handles 2 requests: requests.Inc() and latency.Observe(…) for each, one atomic add apiece.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"lines":[["requests sent","8"],["retries","5"]]},"pusher":{"lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"edge":8,"node":"pusher","kind":"close","text":"Your code exits: exporter.Close(ctx) stops the ticker and pushes once more.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"lines":[["requests sent","8"],["retries","5"]]},"pusher":{"lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"edge":2,"node":"registry","kind":"snapshot","text":"For Close's last push, the Pusher reads a snapshot of the registry, requests = 11.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"status":"busy","lines":[["requests sent","8"],["retries","5"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"edge":3,"node":"exporter","kind":"push","text":"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: 265 bytes, 224 gzipped.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"status":"busy","lines":[["requests sent","9"],["retries","5"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"edge":4,"node":"receiver","kind":"post","text":"POST /v1/metrics, carrying requests = 11 and 11 latency observations: everything counted since the start.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"status":"busy","lines":[["requests sent","9"],["retries","5"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","9"],["holds latency","9 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"edge":5,"node":"exporter","kind":"accepted","text":"200 OK: the receiver now holds requests = 11 and 11 latency observations.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"status":"busy","lines":[["requests sent","9"],["retries","5"],["this push","attempt 1"]]},"pusher":{"status":"busy","lines":[["every","10 s"],["pushes done","2"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","11"],["holds latency","11 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"edge":7,"node":"pusher","kind":"done","text":"Push 4 done. The receiver is up to date.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"lines":[["requests sent","9"],["retries","5"]]},"pusher":{"lines":[["every","10 s"],["pushes done","3"],["failed","1"],["last error","context deadline exceeded"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","11"],["holds latency","11 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}},{"chapter":3,"at":38,"node":"pusher","kind":"closed","text":"Close returned \u003cnil\u003e. Every request counted reached the receiver.","state":{"app":{"lines":[["requests handled","11"]]},"exporter":{"lines":[["requests sent","9"],["retries","5"]]},"pusher":{"status":"off","lines":[["every","10 s"],["pushes done","3"],["failed","1"],["state","closed"]]},"receiver":{"status":"ok","lines":[["state","up"],["holds requests","11"],["holds latency","11 observations"]]},"registry":{"lines":[["requests","11"],["latency","11 observations"]]}}}],"chapters":["Your code counts; every 10 s the counts are pushed","The receiver refuses, then hangs: pushes fail, nothing piles up","Busy, then back: the next push carries every count","Your code exits: Close pushes what is left"],"answers":{"accepted":{"title":"Accepted","why":"Because every push carries the totals since the start, the receiver's numbers are right again as soon as one push gets through, however many failed before.","how":"A 2xx answer ends send; a partial success, if the receiver refused some data points, is counted in Stats().Rejected.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"Exporter.partialSuccess","line":274,"source":"// partialSuccess counts what an ExportMetricsServiceResponse says was\n// refused.\nfunc (thisRef *Exporter) partialSuccess(answer []byte) {\n\treader := protobuf.NewReader(answer)\n\tfor reader.Next() {\n\t\tif reader.Field() != responsePartialSuccess || reader.Type() != protobuf.Bytes {\n\t\t\tcontinue\n\t\t}\n\t\tpartial := protobuf.NewReader(reader.Bytes())\n\t\tvar rejected int64\n\t\tvar message string\n\t\tfor partial.Next() {\n\t\t\tswitch partial.Field() {\n\t\t\tcase partialRejected:\n\t\t\t\trejected = partial.Int()\n\t\t\tcase partialMessage:\n\t\t\t\tmessage = string(partial.Bytes())\n\t\t\t}\n\t\t}\n\t\tif rejected \u003e 0 {\n\t\t\tthisRef.rejected.Add(uint64(rejected))\n\t\t}\n\t\tif message != \"\" {\n\t\t\tthisRef.lastRejection.Store(\u0026message)\n\t\t}\n\t}\n}"}]},"busy":{"title":"503 and Retry-After","why":"A busy receiver says how long to wait; waiting that long, rather than hammering it, is what OTLP asks and what helps it recover.","how":"post returns the Retry-After pause with the refusal; send waits exactly that long instead of its own growing pause, unless the push has no time left.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"retryAfter","line":323,"source":"// retryAfter reads a Retry-After header: seconds, or a date.\nfunc retryAfter(header string) time.Duration {\n\tif header == \"\" {\n\t\treturn 0\n\t}\n\tif seconds, err := strconv.Atoi(header); err == nil \u0026\u0026 seconds \u003e 0 {\n\t\treturn time.Duration(seconds) * time.Second\n\t}\n\tif at, err := http.ParseTime(header); err == nil {\n\t\treturn max(time.Until(at), 0)\n\t}\n\treturn 0\n}"},{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"retryable","line":230,"source":"func retryable(code int) bool {\n\treturn code == http.StatusTooManyRequests || code == http.StatusBadGateway ||\n\t\tcode == http.StatusServiceUnavailable || code == http.StatusGatewayTimeout\n}"}]},"close":{"title":"Close","why":"What was recorded since the last push would be lost if the program just exited. Close stops the ticker, ends a push still retrying, and sends one last snapshot within the caller's deadline.","how":"Close cancels the Pusher's context, waits for its goroutine, then pushes once more with the caller's context.","code":[{"module":"metrics-spec","file":"push.go","symbol":"Pusher.Close","line":92,"source":"// Close stops the pushing, a push under way included, and sends one last\n// snapshot until ctx ends: what was recorded since the last push is not\n// lost. Closing again returns ErrClosed.\nfunc (thisRef *Pusher) Close(ctx context.Context) error {\n\tif thisRef.closed.Swap(true) {\n\t\treturn ErrClosed\n\t}\n\tthisRef.stop()\n\t\u003c-thisRef.done // quick: the push under way saw its context end\n\treturn thisRef.pushNow(ctx)\n}"}]},"closed":{"title":"Closed","why":"After Close, every request the program counted is at the receiver; Flush and Close return ErrClosed from then on.","how":"Close returned the last push's error: nil here.","code":[{"module":"metrics-spec","file":"push.go","symbol":"Pusher.Close","line":92,"source":"// Close stops the pushing, a push under way included, and sends one last\n// snapshot until ctx ends: what was recorded since the last push is not\n// lost. Closing again returns ErrClosed.\nfunc (thisRef *Pusher) Close(ctx context.Context) error {\n\tif thisRef.closed.Swap(true) {\n\t\treturn ErrClosed\n\t}\n\tthisRef.stop()\n\t\u003c-thisRef.done // quick: the push under way saw its context end\n\treturn thisRef.pushNow(ctx)\n}"}]},"done":{"title":"A push done","why":"The Pusher counts what happened, so a program can see in its own metrics or logs whether its pushes get through.","how":"pushNow calls the exporter's push and counts the outcome; one push at a time, under a mutex.","code":[{"module":"metrics-spec","file":"push.go","symbol":"Pusher.pushNow","line":68,"source":"// pushNow pushes one snapshot and counts the outcome.\nfunc (thisRef *Pusher) pushNow(ctx context.Context) error {\n\tthisRef.pushing.Lock()\n\tdefer thisRef.pushing.Unlock()\n\terr := thisRef.push(ctx, thisRef.source.Snapshot())\n\tif err != nil {\n\t\tthisRef.failures.Add(1)\n\t\tthisRef.lastError.Store(\u0026err)\n\t\treturn err\n\t}\n\tthisRef.pushes.Add(1)\n\treturn nil\n}"}]},"failed":{"title":"A push failed","why":"The Pusher keeps no queue of failed snapshots. A queue would grow while the receiver is down and flood it when it comes back; cumulative values make it unnecessary: the next push carries everything this one would have. Memory stays flat however long the outage lasts.","how":"The push's context ended, so send returned its last error; pushNow counts the failure and keeps the error for Stats().LastError.","code":[{"module":"metrics-spec","file":"push.go","symbol":"Pusher.pushNow","line":68,"source":"// pushNow pushes one snapshot and counts the outcome.\nfunc (thisRef *Pusher) pushNow(ctx context.Context) error {\n\tthisRef.pushing.Lock()\n\tdefer thisRef.pushing.Unlock()\n\terr := thisRef.push(ctx, thisRef.source.Snapshot())\n\tif err != nil {\n\t\tthisRef.failures.Add(1)\n\t\tthisRef.lastError.Store(\u0026err)\n\t\treturn err\n\t}\n\tthisRef.pushes.Add(1)\n\treturn nil\n}"}]},"new":{"title":"The exporter starts","why":"Pushing runs on a goroutine of its own, so the program's code never waits for the network. There is no global registry: the program makes one and hands it to the exporter, so two exporters, or a test, never share counters by accident.","how":"otlp.New starts a metrics.Pusher with the exporter's push function and the interval; the Pusher's goroutine waits on a ticker.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"New","line":123,"source":"// New starts pushing source's snapshots.\nfunc New(source metrics.Source, options ...Option) *Exporter {\n\tset := settings{\n\t\tendpoint: \"http://localhost:4318\", headers: http.Header{}, interval: time.Minute,\n\t\ttimeout: 10 * time.Second, client: http.DefaultClient, gzip: true, maxBody: 4 \u003c\u003c 20,\n\t}\n\tfor _, option := range options {\n\t\toption(\u0026set)\n\t}\n\tnamed := false\n\tfor _, pair := range set.attributes {\n\t\tnamed = named || pair[0] == \"service.name\"\n\t}\n\tif !named {\n\t\texecutable, _ := os.Executable()\n\t\tset.attributes = append([][2]string{{\"service.name\", \"unknown_service:\" + filepath.Base(executable)}}, set.attributes...)\n\t}\n\texporter := \u0026Exporter{settings: set, url: set.endpoint + \"/v1/metrics\", encoder: newEncoder(set.maxBody, set.attributes)}\n\texporter.compressor = gzip.NewWriter(\u0026exporter.compressed)\n\texporter.pusher = metrics.NewPusher(source, set.interval, exporter.push)\n\treturn exporter\n}"},{"module":"metrics-spec","file":"push.go","symbol":"NewPusher","line":41,"source":"// NewPusher starts calling push with source's snapshot every interval;\n// each call gets a context ending at the next interval, so a slow backend\n// never piles calls up. Close stops it.\nfunc NewPusher(source Source, interval time.Duration, push func(ctx context.Context, snapshot Snapshot) error) *Pusher {\n\tif interval \u003c= 0 {\n\t\tpanic(\"metrics: a Pusher's interval must be more than 0\")\n\t}\n\tstopped, stop := context.WithCancel(context.Background())\n\tpusher := \u0026Pusher{source: source, interval: interval, push: push, stopped: stopped, stop: stop, done: make(chan struct{})}\n\tgo pusher.run()\n\treturn pusher\n}"}]},"pause":{"title":"Pausing before a retry","why":"Pauses grow, from about 1 second doubling to 30, with jitter so that many programs retrying at once do not hit the receiver together. A pause longer than the time the push has left is not waited: the next push will carry the same data anyway.","how":"send keeps the pause, waits a random time between half of it and all of it (or the Retry-After), then doubles it.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"Exporter.send","line":193,"source":"// send posts a body, retrying what OTLP says to retry (429, 502, 503,\n// 504, and failures to connect) with growing pauses, or the pause the\n// receiver asks for, until ctx ends.\nfunc (thisRef *Exporter) send(ctx context.Context, body []byte) error {\n\tpayload := body\n\tif thisRef.gzip {\n\t\tthisRef.compressed.Reset()\n\t\tthisRef.compressor.Reset(\u0026thisRef.compressed)\n\t\tthisRef.compressor.Write(body)\n\t\tthisRef.compressor.Close()\n\t\tpayload = thisRef.compressed.Bytes()\n\t}\n\tpause := time.Second\n\tfor attempt := 0; ; attempt++ {\n\t\tif attempt \u003e 0 {\n\t\t\tthisRef.retries.Add(1)\n\t\t}\n\t\tretryAfter, err := thisRef.post(ctx, payload)\n\t\tvar status *StatusError\n\t\tif err == nil || ctx.Err() != nil || errors.As(err, \u0026status) \u0026\u0026 !retryable(status.Code) {\n\t\t\treturn err\n\t\t}\n\t\twait := pause/2 + rand.N(pause/2) // jitter, so many exporters do not retry at once\n\t\tif retryAfter \u003e 0 {\n\t\t\twait = retryAfter\n\t\t}\n\t\tif deadline, ok := ctx.Deadline(); ok \u0026\u0026 time.Until(deadline) \u003c wait {\n\t\t\treturn err // no time left to wait: the next push carries this one's data\n\t\t}\n\t\ttimer := time.NewTimer(wait)\n\t// …\n}"}]},"post":{"title":"Posting to the receiver","why":"Each request is capped by its own Timeout, so a receiver that never answers costs one attempt, not the whole push. At most 64 KiB of an answer is read: a receiver cannot make the program buffer without end.","how":"Exporter.post sends the body with the headers OTLP/HTTP wants, under a context with the Timeout, and reads at most 64 KiB of the answer.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"Exporter.post","line":237,"source":"// post sends one request; on a retryable refusal, it returns the pause\n// the receiver asked for, if any.\nfunc (thisRef *Exporter) post(ctx context.Context, payload []byte) (time.Duration, error) {\n\tctx, cancel := context.WithTimeout(ctx, thisRef.timeout)\n\tdefer cancel()\n\trequest, err := http.NewRequestWithContext(ctx, http.MethodPost, thisRef.url, bytes.NewReader(payload))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tfor key, values := range thisRef.headers {\n\t\trequest.Header[key] = values\n\t}\n\trequest.Header.Set(\"Content-Type\", \"application/x-protobuf\")\n\trequest.Header.Set(\"User-Agent\", \"goframework-metrics-exporters-otlp\")\n\tif thisRef.gzip {\n\t\trequest.Header.Set(\"Content-Encoding\", \"gzip\")\n\t}\n\tresponse, err := thisRef.client.Do(request)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tdefer response.Body.Close()\n\tanswer, err := io.ReadAll(io.LimitReader(response.Body, maxResponse))\n\tif err != nil \u0026\u0026 response.StatusCode/100 != 2 {\n\t\treturn 0, err\n\t}\n\tprotobufAnswer := strings.HasPrefix(response.Header.Get(\"Content-Type\"), \"application/x-protobuf\")\n\tif response.StatusCode/100 == 2 {\n\t\tif protobufAnswer {\n\t\t\tthisRef.partialSuccess(answer)\n\t// …\n}"}]},"push":{"title":"Encoding the push","why":"OTLP is protobuf, but generated code and gRPC would add megabytes and two dozen modules to a program that only counts things. The encoder writes the few messages OTLP needs by hand, checked against OpenTelemetry's own generated types.","how":"encoder.encode walks the snapshot's families and writes each as OTLP's Sum, Gauge or ExponentialHistogram, splitting bodies past MaxBody; the body is gzipped before sending.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"Exporter.push","line":160,"source":"func (thisRef *Exporter) push(ctx context.Context, snapshot metrics.Snapshot) error {\n\tvar failed error\n\tfor _, body := range thisRef.encoder.encode(snapshot) {\n\t\tif err := thisRef.send(ctx, body); err != nil {\n\t\t\tfailed = err\n\t\t\tif ctx.Err() != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn failed\n}"},{"module":"metrics-exporters-otlp","file":"encode.go","symbol":"encoder.encode","line":109,"source":"// encode returns snapshot's request bodies; they are the encoder's until\n// the next call.\nfunc (thisRef *encoder) encode(snapshot metrics.Snapshot) [][]byte {\n\tthisRef.bodies = thisRef.bodies[:0]\n\tnow := unixNano(snapshot.Time)\n\toverflow := metrics.Family{\n\t\tName: \"metrics_series_overflow\", Kind: metrics.KindCounter, Labels: []string{\"metric\"},\n\t\tHelp: \"Label sets recorded into an instrument's overflow series, past its MaxSeries.\",\n\t}\n\tfor _, family := range snapshot.Families {\n\t\tthisRef.family(family, now)\n\t\tif family.Overflowed \u003e 0 {\n\t\t\tseries := metrics.Series{Labels: []string{family.Name}, Value: float64(family.Overflowed)}\n\t\t\tif count := len(family.Series); count \u003e 0 \u0026\u0026 family.Series[count-1].Overflow {\n\t\t\t\tlast := family.Series[count-1]\n\t\t\t\tseries.Start = last.Start // the overflow series began with the first label set it took\n\t\t\t}\n\t\t\toverflow.Series = append(overflow.Series, series)\n\t\t}\n\t}\n\tthisRef.family(overflow, now)\n\tthisRef.flush()\n\treturn thisRef.bodies\n}"}]},"receiver-busy":{"title":"The receiver is busy","why":"An overloaded receiver sheds load with 503 and says when to come back.","how":"The fake receiver answers 503 with Retry-After: 2.","code":[{"module":"tools/flows","file":"metrics_test.go","symbol":"receiver.RoundTrip","line":55,"source":"func (thisRef *receiver) RoundTrip(request *http.Request) (*http.Response, error) {\n\tcompressed, _ := io.ReadAll(request.Body)\n\treader, err := gzip.NewReader(bytes.NewReader(compressed))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbody, _ := io.ReadAll(reader)\n\trequests, latency := decodeRequest(body)\n\n\tthisRef.mutex.Lock()\n\tmode := thisRef.mode\n\tattempt := thisRef.inPush\n\tthisRef.inPush++\n\tthisRef.attempts++\n\tpause := time.Since(thisRef.lastAnswer)\n\tthisRef.mutex.Unlock()\n\n\tr := thisRef.recorder\n\tif attempt == 0 {\n\t\tr.emit(\"exporter\", 3, \"push\", \"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: %d bytes, %d gzipped.\", len(body), len(compressed))\n\t} else {\n\t\tr.emit(\"exporter\", 6, \"pause\", \"After a pause of %s, the exporter tries again: attempt %d of this push.\", seconds(pause), attempt+1)\n\t}\n\tr.emit(\"receiver\", 4, \"post\", \"POST /v1/metrics, carrying requests = %v and %d latency observations: everything counted since the start.\", requests, latency)\n\n\tanswered := func() {\n\t\tthisRef.mutex.Lock()\n\t\tthisRef.lastAnswer = time.Now()\n\t\tthisRef.mutex.Unlock()\n\t}\n\t// …\n}"}]},"receiver-down":{"title":"The receiver goes down","why":"Something the program does not control: a restart, a network change. The exporter has to cope without the program noticing.","how":"In this recording, the receiver is a fake one in memory that refuses, hangs or answers 503 on the scenario's schedule; the exporter is the real one.","code":[{"module":"tools/flows","file":"metrics_test.go","symbol":"receiver.RoundTrip","line":55,"source":"func (thisRef *receiver) RoundTrip(request *http.Request) (*http.Response, error) {\n\tcompressed, _ := io.ReadAll(request.Body)\n\treader, err := gzip.NewReader(bytes.NewReader(compressed))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbody, _ := io.ReadAll(reader)\n\trequests, latency := decodeRequest(body)\n\n\tthisRef.mutex.Lock()\n\tmode := thisRef.mode\n\tattempt := thisRef.inPush\n\tthisRef.inPush++\n\tthisRef.attempts++\n\tpause := time.Since(thisRef.lastAnswer)\n\tthisRef.mutex.Unlock()\n\n\tr := thisRef.recorder\n\tif attempt == 0 {\n\t\tr.emit(\"exporter\", 3, \"push\", \"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: %d bytes, %d gzipped.\", len(body), len(compressed))\n\t} else {\n\t\tr.emit(\"exporter\", 6, \"pause\", \"After a pause of %s, the exporter tries again: attempt %d of this push.\", seconds(pause), attempt+1)\n\t}\n\tr.emit(\"receiver\", 4, \"post\", \"POST /v1/metrics, carrying requests = %v and %d latency observations: everything counted since the start.\", requests, latency)\n\n\tanswered := func() {\n\t\tthisRef.mutex.Lock()\n\t\tthisRef.lastAnswer = time.Now()\n\t\tthisRef.mutex.Unlock()\n\t}\n\t// …\n}"}]},"receiver-hung":{"title":"The receiver hangs","why":"The worst failure for a client: connections are accepted and never answered. Only timeouts get a client out of it.","how":"The fake receiver waits until the request's context ends, as a hung server would.","code":[{"module":"tools/flows","file":"metrics_test.go","symbol":"receiver.RoundTrip","line":55,"source":"func (thisRef *receiver) RoundTrip(request *http.Request) (*http.Response, error) {\n\tcompressed, _ := io.ReadAll(request.Body)\n\treader, err := gzip.NewReader(bytes.NewReader(compressed))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbody, _ := io.ReadAll(reader)\n\trequests, latency := decodeRequest(body)\n\n\tthisRef.mutex.Lock()\n\tmode := thisRef.mode\n\tattempt := thisRef.inPush\n\tthisRef.inPush++\n\tthisRef.attempts++\n\tpause := time.Since(thisRef.lastAnswer)\n\tthisRef.mutex.Unlock()\n\n\tr := thisRef.recorder\n\tif attempt == 0 {\n\t\tr.emit(\"exporter\", 3, \"push\", \"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: %d bytes, %d gzipped.\", len(body), len(compressed))\n\t} else {\n\t\tr.emit(\"exporter\", 6, \"pause\", \"After a pause of %s, the exporter tries again: attempt %d of this push.\", seconds(pause), attempt+1)\n\t}\n\tr.emit(\"receiver\", 4, \"post\", \"POST /v1/metrics, carrying requests = %v and %d latency observations: everything counted since the start.\", requests, latency)\n\n\tanswered := func() {\n\t\tthisRef.mutex.Lock()\n\t\tthisRef.lastAnswer = time.Now()\n\t\tthisRef.mutex.Unlock()\n\t}\n\t// …\n}"}]},"receiver-up":{"title":"The receiver is back","why":"Nothing has to be replayed: the next push brings the receiver up to date.","how":"The fake receiver accepts requests again and keeps the values they carry.","code":[{"module":"tools/flows","file":"metrics_test.go","symbol":"receiver.RoundTrip","line":55,"source":"func (thisRef *receiver) RoundTrip(request *http.Request) (*http.Response, error) {\n\tcompressed, _ := io.ReadAll(request.Body)\n\treader, err := gzip.NewReader(bytes.NewReader(compressed))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbody, _ := io.ReadAll(reader)\n\trequests, latency := decodeRequest(body)\n\n\tthisRef.mutex.Lock()\n\tmode := thisRef.mode\n\tattempt := thisRef.inPush\n\tthisRef.inPush++\n\tthisRef.attempts++\n\tpause := time.Since(thisRef.lastAnswer)\n\tthisRef.mutex.Unlock()\n\n\tr := thisRef.recorder\n\tif attempt == 0 {\n\t\tr.emit(\"exporter\", 3, \"push\", \"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: %d bytes, %d gzipped.\", len(body), len(compressed))\n\t} else {\n\t\tr.emit(\"exporter\", 6, \"pause\", \"After a pause of %s, the exporter tries again: attempt %d of this push.\", seconds(pause), attempt+1)\n\t}\n\tr.emit(\"receiver\", 4, \"post\", \"POST /v1/metrics, carrying requests = %v and %d latency observations: everything counted since the start.\", requests, latency)\n\n\tanswered := func() {\n\t\tthisRef.mutex.Lock()\n\t\tthisRef.lastAnswer = time.Now()\n\t\tthisRef.mutex.Unlock()\n\t}\n\t// …\n}"}]},"record":{"title":"Recording a request","why":"Recording happens on every request, so it must cost what an atomic add costs and allocate nothing: 5 ns for a counter, 14 ns for a histogram, measured. Exporting reads snapshots later and never makes recording wait.","how":"Counter.Inc is one atomic add on the series. Histogram.Observe finds the value's exponential bucket from its binary exponent and a small table, adds one to it, and adds the value to one of eight sum stripes, picked by bucket, so goroutines rarely collide.","code":[{"module":"metrics-spec","file":"instruments.go","symbol":"Counter.Inc","line":16,"source":"// Inc adds 1.\nfunc (thisRef *Counter) Inc() {\n\tif thisRef != nil {\n\t\tthisRef.whole.Add(1)\n\t}\n}"},{"module":"metrics-spec","file":"histogram.go","symbol":"Histogram.Observe","line":68,"source":"// Observe records v. NaN is not recorded; nor are infinities in\n// exponential buckets, which have no place for them.\nfunc (thisRef *Histogram) Observe(v float64) {\n\tif thisRef == nil || v != v {\n\t\treturn\n\t}\n\tif thisRef.bounds != nil {\n\t\tbucket := sort.SearchFloat64s(thisRef.bounds, v) // the first bound not below v\n\t\tthisRef.counts[bucket].Add(1)\n\t\tif !math.IsInf(v, 0) {\n\t\t\tthisRef.addSum(v, int32(bucket))\n\t\t}\n\t\treturn\n\t}\n\tvar bucket int32\n\tswitch {\n\tcase v \u003e 0 \u0026\u0026 v \u003c= math.MaxFloat64:\n\t\tbucket = index(v, thisRef.scale)\n\t\tthisRef.positive.add(bucket)\n\tcase v \u003c 0 \u0026\u0026 v \u003e= -math.MaxFloat64:\n\t\tbucket = index(-v, thisRef.scale)\n\t\tthisRef.negative.add(bucket)\n\tcase v == 0:\n\t\tthisRef.zero.Add(1)\n\tdefault:\n\t\treturn // an infinity\n\t}\n\tthisRef.addSum(v, bucket)\n}"}]},"refused":{"title":"Connection refused","why":"OTLP's specification says which failures are worth retrying: a failure to connect, 429, 502, 503 and 504. A refused connection usually means the receiver is restarting, so the exporter tries again shortly.","how":"Exporter.send gets the error back from post; it is not a refusal OTLP says to stop at, so send pauses and tries again while the push has time.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"Exporter.send","line":193,"source":"// send posts a body, retrying what OTLP says to retry (429, 502, 503,\n// 504, and failures to connect) with growing pauses, or the pause the\n// receiver asks for, until ctx ends.\nfunc (thisRef *Exporter) send(ctx context.Context, body []byte) error {\n\tpayload := body\n\tif thisRef.gzip {\n\t\tthisRef.compressed.Reset()\n\t\tthisRef.compressor.Reset(\u0026thisRef.compressed)\n\t\tthisRef.compressor.Write(body)\n\t\tthisRef.compressor.Close()\n\t\tpayload = thisRef.compressed.Bytes()\n\t}\n\tpause := time.Second\n\tfor attempt := 0; ; attempt++ {\n\t\tif attempt \u003e 0 {\n\t\t\tthisRef.retries.Add(1)\n\t\t}\n\t\tretryAfter, err := thisRef.post(ctx, payload)\n\t\tvar status *StatusError\n\t\tif err == nil || ctx.Err() != nil || errors.As(err, \u0026status) \u0026\u0026 !retryable(status.Code) {\n\t\t\treturn err\n\t\t}\n\t\twait := pause/2 + rand.N(pause/2) // jitter, so many exporters do not retry at once\n\t\tif retryAfter \u003e 0 {\n\t\t\twait = retryAfter\n\t\t}\n\t\tif deadline, ok := ctx.Deadline(); ok \u0026\u0026 time.Until(deadline) \u003c wait {\n\t\t\treturn err // no time left to wait: the next push carries this one's data\n\t\t}\n\t\ttimer := time.NewTimer(wait)\n\t// …\n}"},{"module":"tools/flows","file":"metrics_test.go","symbol":"receiver.RoundTrip","line":55,"source":"func (thisRef *receiver) RoundTrip(request *http.Request) (*http.Response, error) {\n\tcompressed, _ := io.ReadAll(request.Body)\n\treader, err := gzip.NewReader(bytes.NewReader(compressed))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbody, _ := io.ReadAll(reader)\n\trequests, latency := decodeRequest(body)\n\n\tthisRef.mutex.Lock()\n\tmode := thisRef.mode\n\tattempt := thisRef.inPush\n\tthisRef.inPush++\n\tthisRef.attempts++\n\tpause := time.Since(thisRef.lastAnswer)\n\tthisRef.mutex.Unlock()\n\n\tr := thisRef.recorder\n\tif attempt == 0 {\n\t\tr.emit(\"exporter\", 3, \"push\", \"The Pusher hands the snapshot to the exporter, which encodes it as OTLP protobuf: %d bytes, %d gzipped.\", len(body), len(compressed))\n\t} else {\n\t\tr.emit(\"exporter\", 6, \"pause\", \"After a pause of %s, the exporter tries again: attempt %d of this push.\", seconds(pause), attempt+1)\n\t}\n\tr.emit(\"receiver\", 4, \"post\", \"POST /v1/metrics, carrying requests = %v and %d latency observations: everything counted since the start.\", requests, latency)\n\n\tanswered := func() {\n\t\tthisRef.mutex.Lock()\n\t\tthisRef.lastAnswer = time.Now()\n\t\tthisRef.mutex.Unlock()\n\t}\n\t// …\n}"}]},"snapshot":{"title":"The Pusher takes a snapshot","why":"Counters and histograms are cumulative: a snapshot holds everything since the start, not what changed. So a snapshot can be taken at any time, a lost one costs nothing, and the next carries it all.","how":"The Pusher's goroutine wakes on its ticker, gives the push a context that ends when the next push is due, and calls the source's Snapshot.","code":[{"module":"metrics-spec","file":"push.go","symbol":"Pusher.run","line":51,"source":"func (thisRef *Pusher) run() {\n\tdefer close(thisRef.done)\n\tticker := time.NewTicker(thisRef.interval)\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase \u003c-thisRef.stopped.Done():\n\t\t\treturn\n\t\tcase \u003c-ticker.C:\n\t\t\tctx, cancel := context.WithTimeout(thisRef.stopped, thisRef.interval)\n\t\t\tthisRef.pushNow(ctx)\n\t\t\tcancel()\n\t\t}\n\t}\n}"},{"module":"metrics-spec","file":"metrics.go","symbol":"Registry.Snapshot","line":245,"source":"// Snapshot reads every instrument and collector: the families, sorted by\n// name, each series sorted by its label values.\nfunc (thisRef *Registry) Snapshot() Snapshot {\n\tif thisRef == nil {\n\t\treturn Snapshot{}\n\t}\n\tthisRef.mutex.Lock()\n\tinstruments := slices.Clone(thisRef.instruments)\n\tcollectors := slices.Clone(thisRef.collectors)\n\tthisRef.mutex.Unlock()\n\tnow := thisRef.now()\n\tsnapshot := Snapshot{Time: now}\n\tfor _, each := range instruments {\n\t\tdefinition := each.describe()\n\t\tfamily := Family{Name: definition.name, Help: definition.help, Kind: definition.kind, Unit: definition.unit, Labels: definition.labels}\n\t\teach.collect(\u0026family, now)\n\t\tsnapshot.Families = append(snapshot.Families, family)\n\t}\n\tfor _, collector := range collectors {\n\t\tcollector.Collect(func(family Family) { snapshot.Families = append(snapshot.Families, family) })\n\t}\n\tslices.SortStableFunc(snapshot.Families, func(a, b Family) int {\n\t\tif a.Name \u003c b.Name {\n\t\t\treturn -1\n\t\t}\n\t\tif a.Name \u003e b.Name {\n\t\t\treturn 1\n\t\t}\n\t\treturn 0\n\t})\n\t// …\n}"}]},"timeout":{"title":"No answer","why":"A hung receiver accepts the connection and never answers: without a timeout the push would wait for ever. Each attempt has its own deadline, and the push as a whole ends when the next one is due.","how":"post's context comes from context.WithTimeout(ctx, Timeout); when it ends, the HTTP client drops the request and send decides whether there is time for another attempt.","code":[{"module":"metrics-exporters-otlp","file":"otlp.go","symbol":"Exporter.post","line":237,"source":"// post sends one request; on a retryable refusal, it returns the pause\n// the receiver asked for, if any.\nfunc (thisRef *Exporter) post(ctx context.Context, payload []byte) (time.Duration, error) {\n\tctx, cancel := context.WithTimeout(ctx, thisRef.timeout)\n\tdefer cancel()\n\trequest, err := http.NewRequestWithContext(ctx, http.MethodPost, thisRef.url, bytes.NewReader(payload))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tfor key, values := range thisRef.headers {\n\t\trequest.Header[key] = values\n\t}\n\trequest.Header.Set(\"Content-Type\", \"application/x-protobuf\")\n\trequest.Header.Set(\"User-Agent\", \"goframework-metrics-exporters-otlp\")\n\tif thisRef.gzip {\n\t\trequest.Header.Set(\"Content-Encoding\", \"gzip\")\n\t}\n\tresponse, err := thisRef.client.Do(request)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tdefer response.Body.Close()\n\tanswer, err := io.ReadAll(io.LimitReader(response.Body, maxResponse))\n\tif err != nil \u0026\u0026 response.StatusCode/100 != 2 {\n\t\treturn 0, err\n\t}\n\tprotobufAnswer := strings.HasPrefix(response.Header.Get(\"Content-Type\"), \"application/x-protobuf\")\n\tif response.StatusCode/100 == 2 {\n\t\tif protobufAnswer {\n\t\t\tthisRef.partialSuccess(answer)\n\t// …\n}"}]}},"questions":[{"q":"Why not keep failed pushes in a queue and send them later?","a":"A queue grows while the receiver is down, then floods it when it is back, and is lost anyway if the program exits. Counters and histograms are cumulative, so the next push already carries everything: nothing needs keeping. Only a gauge's values between two pushes are lost, as they are between two scrapes."},{"q":"What happens to memory during a long outage?","a":"It stays flat: the exporter holds one snapshot at a time. In the fault test (benchmarks/interop), 2000 series pushed every second through 3-minute outages kept the heap at 3 MB."},{"q":"Does recording slow down while the receiver is down?","a":"No. Recording is an atomic add on the series; the Pusher reads snapshots on its own goroutine, and nothing in recording waits for it."},{"q":"Why gzip?","a":"Metrics compress well, often tenfold, and every receiver tested (the OpenTelemetry Collector, Prometheus, VictoriaMetrics) accepts gzip. Gzip(false) turns it off."},{"q":"Why is there no gRPC?","a":"OTLP over HTTP carries the same protobuf messages; gRPC would add its library and HTTP/2 machinery to every program that counts things. OpenTelemetry Go's exporter adds 9.6 MB to a binary; this one 0.28 MB with the Prometheus exporter."},{"q":"How long does a push try?","a":"Until the next push is due: each push gets a context ending one interval later, and each attempt its own Timeout. Close's last push tries until the caller's context ends."},{"q":"Which receivers was it checked against?","a":"Prometheus 3.15's OTLP receiver, the OpenTelemetry Collector 0.163 and VictoriaMetrics 1.154, in a VM, every value read back; and OpenTelemetry's generated protobuf types on 200 random snapshots."},{"q":"What does the receiver see after the outage?","a":"The totals: the first push that gets through carries requests and latency counted since the start, so the receiver's graphs show no hole in counts, only in the time between."}]}); |