1{2 "uid": "o12-runtime",3 "title": "012 runtime",4 "description": "012's own events: its log, from the collector's filelog receiver (012 --log) or OTLP (012 --otlp), and its OTLP traces and metrics",5 "tags": [6 "012"7 ],8 "timezone": "browser",9 "schemaVersion": 39,10 "version": 1,11 "editable": true,12 "refresh": "10s",13 "time": {14 "from": "now-1h",15 "to": "now"16 },17 "panels": [18 {19 "id": 1,20 "type": "timeseries",21 "title": "Recalculation time",22 "description": "recalc events: incremental and full recalculations, in milliseconds",23 "datasource": {24 "type": "grafana-clickhouse-datasource",25 "uid": "clickhouse"26 },27 "gridPos": {28 "x": 0,29 "y": 0,30 "w": 12,31 "h": 832 },33 "targets": [34 {35 "datasource": {36 "type": "grafana-clickhouse-datasource",37 "uid": "clickhouse"38 },39 "editorType": "sql",40 "format": 0,41 "queryType": "timeseries",42 "rawSql": "SELECT $__timeInterval(Timestamp) AS time, quantile(0.5)(toFloat64OrZero(LogAttributes['dur_ms'])) AS p50, quantile(0.95)(toFloat64OrZero(LogAttributes['dur_ms'])) AS p95, max(toFloat64OrZero(LogAttributes['dur_ms'])) AS max FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND LogAttributes['event'] = 'recalc' GROUP BY time ORDER BY time",43 "refId": "A"44 }45 ],46 "fieldConfig": {47 "defaults": {48 "unit": "ms",49 "custom": {50 "lineWidth": 2,51 "fillOpacity": 8,52 "showPoints": "auto",53 "spanNulls": true54 }55 },56 "overrides": []57 },58 "options": {59 "legend": {60 "displayMode": "list",61 "placement": "bottom"62 },63 "tooltip": {64 "mode": "multi"65 }66 }67 },68 {69 "id": 2,70 "type": "timeseries",71 "title": "Frame and key-to-frame time (p95 per second)",72 "description": "frames summaries: View() time and the time from a key press to its frame; 16 ms is one frame at 60 Hz",73 "datasource": {74 "type": "grafana-clickhouse-datasource",75 "uid": "clickhouse"76 },77 "gridPos": {78 "x": 12,79 "y": 0,80 "w": 12,81 "h": 882 },83 "targets": [84 {85 "datasource": {86 "type": "grafana-clickhouse-datasource",87 "uid": "clickhouse"88 },89 "editorType": "sql",90 "format": 0,91 "queryType": "timeseries",92 "rawSql": "SELECT $__timeInterval(Timestamp) AS time, max(toFloat64OrZero(LogAttributes['render_p95_ms'])) AS render_p95, max(toFloat64OrZero(LogAttributes['key_p95_ms'])) AS key_to_frame_p95, max(toFloat64OrZero(LogAttributes['key_max_ms'])) AS key_to_frame_max FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND LogAttributes['event'] = 'frames' GROUP BY time ORDER BY time",93 "refId": "A"94 }95 ],96 "fieldConfig": {97 "defaults": {98 "unit": "ms",99 "custom": {100 "lineWidth": 2,101 "fillOpacity": 8,102 "showPoints": "auto",103 "spanNulls": true104 }105 },106 "overrides": []107 },108 "options": {109 "legend": {110 "displayMode": "list",111 "placement": "bottom"112 },113 "tooltip": {114 "mode": "multi"115 }116 }117 },118 {119 "id": 3,120 "type": "timeseries",121 "title": "Heap and cells",122 "description": "Live heap (left) and stored cells (right), from the per-second frame summaries",123 "datasource": {124 "type": "grafana-clickhouse-datasource",125 "uid": "clickhouse"126 },127 "gridPos": {128 "x": 0,129 "y": 8,130 "w": 12,131 "h": 8132 },133 "targets": [134 {135 "datasource": {136 "type": "grafana-clickhouse-datasource",137 "uid": "clickhouse"138 },139 "editorType": "sql",140 "format": 0,141 "queryType": "timeseries",142 "rawSql": "SELECT $__timeInterval(Timestamp) AS time, max(toFloat64OrZero(LogAttributes['heap_bytes'])) AS heap, max(toFloat64OrZero(LogAttributes['cells'])) AS cells FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND LogAttributes['event'] = 'frames' GROUP BY time ORDER BY time",143 "refId": "A"144 }145 ],146 "fieldConfig": {147 "defaults": {148 "unit": "bytes",149 "custom": {150 "lineWidth": 2,151 "fillOpacity": 8,152 "showPoints": "auto",153 "spanNulls": true154 }155 },156 "overrides": [157 {158 "matcher": {159 "id": "byName",160 "options": "cells"161 },162 "properties": [163 {164 "id": "custom.axisPlacement",165 "value": "right"166 },167 {168 "id": "unit",169 "value": "short"170 }171 ]172 }173 ]174 },175 "options": {176 "legend": {177 "displayMode": "list",178 "placement": "bottom"179 },180 "tooltip": {181 "mode": "multi"182 }183 }184 },185 {186 "id": 4,187 "type": "timeseries",188 "title": "JEV answer time",189 "description": "jev events: request latency and questions still queued when it was sent",190 "datasource": {191 "type": "grafana-clickhouse-datasource",192 "uid": "clickhouse"193 },194 "gridPos": {195 "x": 12,196 "y": 8,197 "w": 12,198 "h": 8199 },200 "targets": [201 {202 "datasource": {203 "type": "grafana-clickhouse-datasource",204 "uid": "clickhouse"205 },206 "editorType": "sql",207 "format": 0,208 "queryType": "timeseries",209 "rawSql": "SELECT $__timeInterval(Timestamp) AS time, quantile(0.5)(toFloat64OrZero(LogAttributes['dur_ms'])) AS p50, quantile(0.95)(toFloat64OrZero(LogAttributes['dur_ms'])) AS p95, max(toFloat64OrZero(LogAttributes['queued'])) AS queued FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND LogAttributes['event'] = 'jev' GROUP BY time ORDER BY time",210 "refId": "A"211 }212 ],213 "fieldConfig": {214 "defaults": {215 "unit": "ms",216 "custom": {217 "lineWidth": 2,218 "fillOpacity": 8,219 "showPoints": "auto",220 "spanNulls": true221 }222 },223 "overrides": []224 },225 "options": {226 "legend": {227 "displayMode": "list",228 "placement": "bottom"229 },230 "tooltip": {231 "mode": "multi"232 }233 }234 },235 {236 "id": 5,237 "type": "table",238 "title": "Imports and exports",239 "description": "File operations with their sizes and throughput",240 "datasource": {241 "type": "grafana-clickhouse-datasource",242 "uid": "clickhouse"243 },244 "gridPos": {245 "x": 0,246 "y": 16,247 "w": 12,248 "h": 8249 },250 "targets": [251 {252 "datasource": {253 "type": "grafana-clickhouse-datasource",254 "uid": "clickhouse"255 },256 "editorType": "sql",257 "format": 1,258 "queryType": "table",259 "rawSql": "SELECT Timestamp AS time, LogAttributes['event'] AS event, LogAttributes['format'] AS format, toFloat64OrZero(LogAttributes['rows']) AS rows, toFloat64OrZero(LogAttributes['cells']) AS cells, round(toFloat64OrZero(LogAttributes['dur_ms']), 1) AS ms, round(toFloat64OrZero(LogAttributes['bytes']) / 1048576 / (toFloat64OrZero(LogAttributes['dur_ms']) / 1000), 1) AS mb_per_s FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND LogAttributes['event'] IN ('import', 'export', 'save', 'open') ORDER BY time DESC LIMIT 50",260 "refId": "A"261 }262 ],263 "fieldConfig": {264 "defaults": {},265 "overrides": []266 },267 "options": {268 "showHeader": true,269 "cellHeight": "sm"270 }271 },272 {273 "id": 6,274 "type": "table",275 "title": "Slowest operations",276 "description": "The 25 slowest spans in the range: recalcs, commands, sorts, finds, imports",277 "datasource": {278 "type": "grafana-clickhouse-datasource",279 "uid": "clickhouse"280 },281 "gridPos": {282 "x": 12,283 "y": 16,284 "w": 12,285 "h": 8286 },287 "targets": [288 {289 "datasource": {290 "type": "grafana-clickhouse-datasource",291 "uid": "clickhouse"292 },293 "editorType": "sql",294 "format": 1,295 "queryType": "table",296 "rawSql": "SELECT Timestamp AS time, LogAttributes['event'] AS event, LogAttributes['id'] AS command, round(toFloat64OrZero(LogAttributes['dur_ms']), 2) AS ms, toFloat64OrZero(LogAttributes['evaluated']) AS evaluated, toFloat64OrZero(LogAttributes['cells']) AS cells FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND LogAttributes['event'] NOT IN ('frames', 'start', 'frame') ORDER BY ms DESC LIMIT 25",297 "refId": "A"298 }299 ],300 "fieldConfig": {301 "defaults": {},302 "overrides": []303 },304 "options": {305 "showHeader": true,306 "cellHeight": "sm"307 }308 },309 {310 "id": 7,311 "type": "timeseries",312 "title": "Events per minute",313 "description": "What 012 has been doing, by event type",314 "datasource": {315 "type": "grafana-clickhouse-datasource",316 "uid": "clickhouse"317 },318 "gridPos": {319 "x": 0,320 "y": 24,321 "w": 24,322 "h": 7323 },324 "targets": [325 {326 "datasource": {327 "type": "grafana-clickhouse-datasource",328 "uid": "clickhouse"329 },330 "editorType": "sql",331 "format": 0,332 "queryType": "timeseries",333 "rawSql": "SELECT toStartOfMinute(Timestamp) AS time, LogAttributes['event'] AS event, count() AS n FROM otel.otel_logs WHERE $__timeFilter(Timestamp) AND ServiceName = '012' GROUP BY time, event ORDER BY time",334 "refId": "A"335 }336 ],337 "fieldConfig": {338 "defaults": {339 "custom": {340 "lineWidth": 2,341 "fillOpacity": 8,342 "showPoints": "auto",343 "spanNulls": true344 }345 },346 "overrides": []347 },348 "options": {349 "legend": {350 "displayMode": "list",351 "placement": "bottom"352 },353 "tooltip": {354 "mode": "multi"355 }356 }357 },358 {359 "id": 8,360 "type": "timeseries",361 "title": "Key-to-frame latency (OTLP metrics)",362 "description": "o12.key.latency, the per-second summary 012 sends over OTLP (012 --otlp): p50, p95 and max, in milliseconds",363 "datasource": {364 "type": "grafana-clickhouse-datasource",365 "uid": "clickhouse"366 },367 "gridPos": {368 "x": 0,369 "y": 31,370 "w": 12,371 "h": 8372 },373 "targets": [374 {375 "datasource": {376 "type": "grafana-clickhouse-datasource",377 "uid": "clickhouse"378 },379 "editorType": "sql",380 "format": 0,381 "queryType": "timeseries",382 "rawSql": "SELECT $__timeInterval(TimeUnix) AS time, max(ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 0.5)]) AS p50, max(ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 0.95)]) AS p95, max(ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 1)]) AS max FROM otel.otel_metrics_summary WHERE $__timeFilter(TimeUnix) AND ServiceName = '012' AND MetricName = 'o12.key.latency' GROUP BY time ORDER BY time",383 "refId": "A"384 }385 ],386 "fieldConfig": {387 "defaults": {388 "unit": "ms",389 "custom": {390 "lineWidth": 2,391 "fillOpacity": 8,392 "showPoints": "auto",393 "spanNulls": true394 }395 },396 "overrides": []397 },398 "options": {399 "legend": {400 "displayMode": "list",401 "placement": "bottom"402 },403 "tooltip": {404 "mode": "multi"405 }406 }407 },408 {409 "id": 9,410 "type": "table",411 "title": "Slowest spans (OTLP traces)",412 "description": "The slowest spans 012 sent over OTLP, roots and nested ones; a trace id opens that trace in the trace view below",413 "datasource": {414 "type": "grafana-clickhouse-datasource",415 "uid": "clickhouse"416 },417 "gridPos": {418 "x": 12,419 "y": 31,420 "w": 12,421 "h": 8422 },423 "targets": [424 {425 "datasource": {426 "type": "grafana-clickhouse-datasource",427 "uid": "clickhouse"428 },429 "editorType": "sql",430 "format": 1,431 "queryType": "table",432 "rawSql": "SELECT Timestamp AS time, SpanName AS span, round(Duration / 1e6, 2) AS ms, StatusCode AS status, StatusMessage AS error, TraceId AS trace, toString(SpanAttributes) AS attributes FROM otel.otel_traces WHERE $__timeFilter(Timestamp) AND ServiceName = '012' ORDER BY Duration DESC LIMIT 50",433 "refId": "A"434 }435 ],436 "fieldConfig": {437 "defaults": {},438 "overrides": [439 {440 "matcher": {441 "id": "byName",442 "options": "trace"443 },444 "properties": [445 {446 "id": "links",447 "value": [448 {449 "title": "Show this trace",450 "url": "/d/o12-runtime/012-runtime?var-trace=${__value.raw}&${__url_time_range}"451 }452 ]453 }454 ]455 }456 ]457 },458 "options": {459 "showHeader": true460 }461 },462 {463 "id": 10,464 "type": "table",465 "title": "Recent traces with nested spans",466 "description": "Traces of more than one span, newest first: the root span, its command id, how many spans it holds, which spans nest in it and how long the root took. A trace id opens it in the trace view below",467 "datasource": {468 "type": "grafana-clickhouse-datasource",469 "uid": "clickhouse"470 },471 "gridPos": {472 "x": 0,473 "y": 39,474 "w": 24,475 "h": 8476 },477 "targets": [478 {479 "datasource": {480 "type": "grafana-clickhouse-datasource",481 "uid": "clickhouse"482 },483 "editorType": "sql",484 "format": 1,485 "queryType": "table",486 "refId": "A",487 "rawSql": "SELECT min(Timestamp) AS time, anyIf(SpanName, ParentSpanId = '') AS root, anyIf(SpanAttributes['id'], ParentSpanId = '') AS command, count() AS spans, arrayStringConcat(arraySort(groupUniqArrayIf(SpanName, ParentSpanId != '')), ', ') AS nested, round(maxIf(Duration, ParentSpanId = '') / 1e6, 2) AS ms, countIf(StatusCode = 'Error') AS errors, TraceId AS trace FROM otel.otel_traces WHERE $__timeFilter(Timestamp) AND ServiceName = '012' GROUP BY TraceId HAVING spans > 1 ORDER BY time DESC LIMIT 100"488 }489 ],490 "fieldConfig": {491 "defaults": {},492 "overrides": [493 {494 "matcher": {495 "id": "byName",496 "options": "trace"497 },498 "properties": [499 {500 "id": "links",501 "value": [502 {503 "title": "Show this trace",504 "url": "/d/o12-runtime/012-runtime?var-trace=${__value.raw}&${__url_time_range}"505 }506 ]507 }508 ]509 }510 ]511 },512 "options": {513 "showHeader": true514 }515 },516 {517 "id": 11,518 "type": "traces",519 "title": "Trace ${trace}",520 "description": "The spans of one trace as a tree with their timing: the trace id in the Trace id box at the top, or the latest trace with nested spans when it is empty",521 "datasource": {522 "type": "grafana-clickhouse-datasource",523 "uid": "clickhouse"524 },525 "gridPos": {526 "x": 0,527 "y": 47,528 "w": 24,529 "h": 14530 },531 "targets": [532 {533 "datasource": {534 "type": "grafana-clickhouse-datasource",535 "uid": "clickhouse"536 },537 "editorType": "sql",538 "format": 3,539 "queryType": "traces",540 "refId": "A",541 "rawSql": "SELECT TraceId AS traceID, SpanId AS spanID, ParentSpanId AS parentSpanID, ServiceName AS serviceName, SpanName AS operationName, multiply(toUnixTimestamp64Nano(Timestamp), 0.000001) AS startTime, multiply(Duration, 0.000001) AS duration, arrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) AS tags, arrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) AS serviceTags, if(StatusCode IN ('Error', 'STATUS_CODE_ERROR'), 2, 0) AS statusCode FROM otel.otel_traces WHERE ServiceName = '012' AND TraceId = if('${trace}' != '', '${trace}', (SELECT TraceId FROM otel.otel_traces WHERE $__timeFilter(Timestamp) AND ServiceName = '012' AND ParentSpanId != '' ORDER BY Timestamp DESC LIMIT 1)) ORDER BY Timestamp LIMIT 1000"542 }543 ],544 "options": {}545 }546 ],547 "templating": {548 "list": [549 {550 "name": "trace",551 "label": "Trace id",552 "type": "textbox",553 "query": "",554 "current": {555 "text": "",556 "value": ""557 },558 "description": "The trace the trace view shows; empty shows the latest trace with nested spans"559 }560 ]561 },562 "annotations": {563 "list": []564 }565}