The collector's OTLP/HTTP receiver already feeds the ClickHouse logs, traces and metrics pipelines; say how 012 reaches it, list spans in obs-status, and add key latency (OTLP summary) and slowest spans panels to the runtime dashboard.
- commit
- 5b5b276cd44ce87e03fd08a7904dc9d8be519b6e
- parent
- 585081ea09e404da31f3fb9c5e6a2f17c6b73e92
- author
- Dami <dami.etoile@gmail.com>
| M | deploy/ | 88 | ++++++++++++++++++++++++++++++- |
| M | deploy/ | 10 | ++- |
| M | scripts/ | 7 | ++- |
@@ -1,7 +1,7 @@
11 {22 "uid": "o12-runtime",33 "title": "012 runtime",4- "description": "012's own event log (012 --log), via the collector's filelog receiver",4+ "description": "012's own events: its log, from the collector's filelog receiver (012 --log) or OTLP (012 --otlp), and its OTLP traces and metrics",55 "tags": [66 "012"77 ],@@ -354,6 +354,92 @@
354354 "mode": "multi"355355 }356356 }357+ },358+ {359+ "id": 8,360+ "type": "timeseries",361+ "title": "Key-to-frame latency (OTLP metrics)",362+ "description": "o12.key.latency, the per-second summary 012 sends over OTLP (012 --otlp): p50, p95 and max, in milliseconds",363+ "datasource": {364+ "type": "grafana-clickhouse-datasource",365+ "uid": "clickhouse"366+ },367+ "gridPos": {368+ "x": 0,369+ "y": 31,370+ "w": 12,371+ "h": 8372+ },373+ "targets": [374+ {375+ "datasource": {376+ "type": "grafana-clickhouse-datasource",377+ "uid": "clickhouse"378+ },379+ "editorType": "sql",380+ "format": 0,381+ "queryType": "timeseries",382+ "rawSql": "SELECT $__timeInterval(TimeUnix) AS time, max(ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 0.5)]) AS p50, max(ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 0.95)]) AS p95, max(ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 1)]) AS max FROM otel.otel_metrics_summary WHERE $__timeFilter(TimeUnix) AND ServiceName = '012' AND MetricName = 'o12.key.latency' GROUP BY time ORDER BY time",383+ "refId": "A"384+ }385+ ],386+ "fieldConfig": {387+ "defaults": {388+ "unit": "ms",389+ "custom": {390+ "lineWidth": 2,391+ "fillOpacity": 8,392+ "showPoints": "auto",393+ "spanNulls": true394+ }395+ },396+ "overrides": []397+ },398+ "options": {399+ "legend": {400+ "displayMode": "list",401+ "placement": "bottom"402+ },403+ "tooltip": {404+ "mode": "multi"405+ }406+ }407+ },408+ {409+ "id": 9,410+ "type": "table",411+ "title": "Slowest spans (OTLP traces)",412+ "description": "The slowest spans 012 sent over OTLP; each is the root of its own trace, and the log record of the same event carries its TraceId",413+ "datasource": {414+ "type": "grafana-clickhouse-datasource",415+ "uid": "clickhouse"416+ },417+ "gridPos": {418+ "x": 12,419+ "y": 31,420+ "w": 12,421+ "h": 8422+ },423+ "targets": [424+ {425+ "datasource": {426+ "type": "grafana-clickhouse-datasource",427+ "uid": "clickhouse"428+ },429+ "editorType": "sql",430+ "format": 1,431+ "queryType": "table",432+ "rawSql": "SELECT Timestamp AS time, SpanName AS span, round(Duration / 1e6, 2) AS ms, StatusCode AS status, StatusMessage AS error, TraceId AS trace, toString(SpanAttributes) AS attributes FROM otel.otel_traces WHERE $__timeFilter(Timestamp) AND ServiceName = '012' ORDER BY Duration DESC LIMIT 50",433+ "refId": "A"434+ }435+ ],436+ "fieldConfig": {437+ "defaults": {},438+ "overrides": []439+ },440+ "options": {441+ "showHeader": true442+ }357443 }358444 ],359445 "templating": {@@ -1,6 +1,10 @@
1-# OpenTelemetry Collector for 012: OTLP (gRPC and HTTP) for any2-# instrumented process, and the filelog receiver for 012's own JSON3-# event logs (012 --log .deps/obs/logs/012.jsonl), all into ClickHouse.1+# OpenTelemetry Collector for 012, all into ClickHouse. Two ways in:2+# - OTLP/HTTP on 4318 (published on 127.0.0.1 only): 012 --otlp3+# http://localhost:4318, or OTEL_EXPORTER_OTLP_ENDPOINT, sends its4+# logs, traces and metrics (JSON encoding); gRPC on 4317 for others.5+# - The filelog receiver, the alternative for logs only: 012 --log6+# .deps/obs/logs/012.jsonl. Use one or the other for a session, or its7+# logs arrive twice.48 extensions:59 health_check:610 endpoint: 0.0.0.0:13133@@ -23,8 +23,9 @@ Observability stack is up:
2323 ClickHouse http://localhost:8123 (user o12, password o12; databases otel, stress)2424 Collector OTLP on localhost:4317 (gRPC) and localhost:4318 (HTTP)2525 26-Point 012 at it (its JSON event log is tailed by the collector):27- export O12_LOG=$logs/012.jsonl26+Point 012 at it, one way or the other (both would store its logs twice):27+ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 # OTLP: logs, traces, metrics28+ export O12_LOG=$logs/012.jsonl # or: a JSON log the collector tails2829 2930 make stress loads each run into ClickHouse while the stack is up;3031 make stress-load loads the runs recorded so far.@@ -44,6 +45,8 @@ status)
4445 query "SELECT database || '.' || name, total_rows FROM system.tables WHERE database IN ('otel', 'stress') AND engine NOT LIKE '%View%' ORDER BY 1 FORMAT PrettyCompactNoEscapes"4546 echo "012 events by type (last hour):"4647 query "SELECT LogAttributes['event'] AS event, count() AS n, round(quantile(0.95)(toFloat64OrZero(LogAttributes['dur_ms'])), 2) AS p95_ms FROM otel.otel_logs WHERE ServiceName = '012' AND Timestamp > now() - INTERVAL 1 HOUR GROUP BY event ORDER BY n DESC FORMAT PrettyCompactNoEscapes" || true48+ echo "012 spans by name (last hour, OTLP):"49+ query "SELECT SpanName AS span, count() AS n, round(quantile(0.95)(Duration) / 1e6, 2) AS p95_ms FROM otel.otel_traces WHERE ServiceName = '012' AND Timestamp > now() - INTERVAL 1 HOUR GROUP BY span ORDER BY n DESC FORMAT PrettyCompactNoEscapes" || true4750 ;;4851 *)4952 echo "usage: $0 up|down|status" >&2