12# obs.sh up|down|status - the local observability stack in3# deploy/observability (OpenTelemetry Collector, ClickHouse, Grafana).4# make obs-up, obs-down and obs-status call this.5set -euo pipefail67root=$(cd "$(dirname "$0")/.." && pwd)8compose=(docker compose -f "$root/deploy/observability/docker-compose.yml")9logs="$root/.deps/obs/logs"10ch=${CLICKHOUSE_URL:-http://localhost:8123}11auth="${CLICKHOUSE_USER:-o12}:${CLICKHOUSE_PASSWORD:-o12}"1213query() { curl -fsS -u "$auth" --get --data-urlencode "query=$1" "$ch/" 2>/dev/null; }1415case "${1:-}" in16up)17 mkdir -p "$logs"18 "${compose[@]}" up -d --wait19 cat <<EOF2021Observability stack is up:22 Grafana http://localhost:3000 (dashboards: 012 runtime, 012 stress runs)23 ClickHouse http://localhost:8123 (user o12, password o12; databases otel, stress)24 Collector OTLP on localhost:4317 (gRPC) and localhost:4318 (HTTP); health on localhost:131332526Point 012 at it, one way or the other (both would store its logs twice):27 export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 # OTLP: logs, traces, metrics28 export O12_LOG=$logs/012.jsonl # or: a JSON log the collector tails2930make stress loads each run into ClickHouse while the stack is up;31make stress-load loads the runs recorded so far.32EOF33 ;;34down)35 "${compose[@]}" down36 ;;37status)38 "${compose[@]}" ps39 echo40 if ! query "SELECT 1" >/dev/null; then41 echo "ClickHouse: not answering at $ch"42 exit 043 fi44 echo "ClickHouse row counts:"45 query "SELECT database || '.' || name, total_rows FROM system.tables WHERE database IN ('otel', 'stress') AND engine NOT LIKE '%View%' ORDER BY 1 FORMAT PrettyCompactNoEscapes"46 echo "012 events by type (last hour):"47 query "SELECT LogAttributes['event'] AS event, count() AS n, round(quantile(0.95)(toFloat64OrZero(LogAttributes['dur_ms'])), 2) AS p95_ms FROM otel.otel_logs WHERE ServiceName = '012' AND Timestamp > now() - INTERVAL 1 HOUR GROUP BY event ORDER BY n DESC FORMAT PrettyCompactNoEscapes" || true48 echo "012 spans by name (last hour, OTLP):"49 query "SELECT SpanName AS span, count() AS n, round(quantile(0.95)(Duration) / 1e6, 2) AS p95_ms FROM otel.otel_traces WHERE ServiceName = '012' AND Timestamp > now() - INTERVAL 1 HOUR GROUP BY span ORDER BY n DESC FORMAT PrettyCompactNoEscapes" || true50 ;;51*)52 echo "usage: $0 up|down|status" >&253 exit 254 ;;55esac