Phase 0.6: Tiered OLAP benchmark harness

crates/scouter_dataframe/Cargo.toml

@@ -8,10 +8,14 @@ authors = [
 ]
 license = "MIT"
 description = "DataFusion client for long-term storage of scouter data"
+autobenches = false
 
 [lib]
 doctest = false
 
+[features]
+bench-jaeger = []
+
 [dependencies]
 scouter-settings = { workspace = true }
 scouter-types = { workspace = true }
@@ -79,3 +83,11 @@ harness = false
 [[bench]]
 name = "hot_path_bench"
 harness = false
+
+[[bin]]
+name = "bench_tier_filter"
+path = "benches/tiers.rs"
+
+[[bin]]
+name = "bench_compare"
+path = "benches/tiers.rs"

crates/scouter_dataframe/bench_metrics/README.md

@@ -0,0 +1,41 @@
+# Tier 0 Benchmark Baselines
+
+This directory contains the committed Tier 0 artifacts produced by `make bench.core`.
+
+Tier 0 is the blocking OLAP smoke baseline. It is intentionally small enough for PR verification,
+but it must still prove the measured path ran:
+
+- every registered Tier 0 group must write an artifact;
+- non-sentinel groups must report measured `bench.query.end_to_end` iterations, not a single
+  setup probe;
+- non-sentinel groups must report `query_entrypoint` and `result_rows`;
+- non-sentinel groups must report object-store operations observed through the production
+  object-store spans;
+- `t0_refresh_origin_sentinel` is allowed to report zero workload metrics, because it only guards
+  that refresh-on-request accounting stays at zero.
+
+## End-to-end measurement boundary
+
+`bench.query.end_to_end` is the primary Tier 0 metric. It wraps the public in-process query
+entry point and includes the returned batches or domain objects, so future phases can catch
+regressions in planning, metadata lookup, snapshot freshness, DataFusion execution, and result
+assembly.
+
+Every non-sentinel Tier 0 artifact must include:
+
+- `query_entrypoint`: the in-process boundary being measured.
+- `result_rows`: the number of returned rows or spans from the probe query.
+- `spans["bench.query.end_to_end"]`: at least 10 measured iterations with non-zero total time.
+
+`df.collect` is diagnostic only. An improvement in `df.collect` cannot mask a
+`bench.query.end_to_end.p95_us` regression, though diagnostic regressions still fail Tier 0 when
+both baseline and run artifacts carry the span.
+
+`t0_refresh_origin_sentinel` does not execute a query, so it may omit
+`bench.query.end_to_end`, `query_entrypoint`, and `result_rows`.
+
+Baseline JSON files are refreshed by an explicit reviewer-visible PR after a corrected Tier 0
+Criterion run. CI compares artifacts but never writes committed baselines.
+
+The comparator hard-fails only Tier 0 artifacts. Tier 1 and Tier 2 artifacts are advisory and are
+intended for extended or release certification runs.

crates/scouter_dataframe/bench_metrics/t0_bifrost_smoke.json

@@ -0,0 +1,56 @@
+{
+  "commit": "553a607685000971ef06678e913db7429d263148",
+  "bench_group": "t0_bifrost_smoke",
+  "tier": 0,
+  "blocking": true,
+  "scenario_class": "bifrost_smoke",
+  "runtime_budget_secs": 120,
+  "actual_runtime_secs": 51.151038333,
+  "fixture_rows": 1000,
+  "fixture_spans": 0,
+  "storage_profile": "P1_local_nvme",
+  "query_entrypoint": "dataset_engine_manager.query",
+  "result_rows": 256,
+  "spans": {
+    "bench.query.end_to_end": {
+      "count": 23341,
+      "p50_us": 927,
+      "p95_us": 5301,
+      "p99_us": 27795,
+      "sum_us": 49786477
+    },
+    "log.list": {
+      "count": 1,
+      "p50_us": 212,
+      "p95_us": 212,
+      "p99_us": 212,
+      "sum_us": 212
+    },
+    "object_store.request": {
+      "count": 23343,
+      "p50_us": 2,
+      "p95_us": 6,
+      "p99_us": 12,
+      "sum_us": 119172
+    },
+    "snap.build": {
+      "count": 1,
+      "p50_us": 221,
+      "p95_us": 221,
+      "p99_us": 221,
+      "sum_us": 221
+    }
+  },
+  "object_store_counts": {
+    "list": 1,
+    "list_with_delimiter": 0,
+    "head": 0,
+    "get": 1,
+    "get_range": 23341,
+    "put": 0,
+    "delete": 0,
+    "copy": 0,
+    "bytes": 0
+  },
+  "refresh_on_request_path_total": 0
+}

crates/scouter_dataframe/bench_metrics/t0_cold_query_smoke.json

@@ -0,0 +1,161 @@
+{
+  "commit": "553a607685000971ef06678e913db7429d263148",
+  "bench_group": "t0_cold_query_smoke",
+  "tier": 0,
+  "blocking": true,
+  "scenario_class": "cold_query",
+  "runtime_budget_secs": 120,
+  "actual_runtime_secs": 8.955178292,
+  "fixture_rows": 10080,
+  "fixture_spans": 10080,
+  "storage_profile": "P1_local_nvme",
+  "query_entrypoint": "trace_query_service.query_spans",
+  "result_rows": 5,
+  "spans": {
+    "arrow.convert": {
+      "count": 2423,
+      "p50_us": 44,
+      "p95_us": 57,
+      "p99_us": 98,
+      "sum_us": 113536
+    },
+    "bench.query.end_to_end": {
+      "count": 2423,
+      "p50_us": 3498,
+      "p95_us": 4237,
+      "p99_us": 4898,
+      "sum_us": 8610882
+    },
+    "create_checkpoint_for": {
+      "count": 1,
+      "p50_us": 4545,
+      "p95_us": 4545,
+      "p99_us": 4545,
+      "sum_us": 4545
+    },
+    "delta.catalog.swap": {
+      "count": 1,
+      "p50_us": 2,
+      "p95_us": 2,
+      "p99_us": 2,
+      "sum_us": 2
+    },
+    "delta.snapshot.refresh": {
+      "count": 1,
+      "p50_us": 4791,
+      "p95_us": 4791,
+      "p99_us": 4791,
+      "sum_us": 4791
+    },
+    "df.collect": {
+      "count": 2423,
+      "p50_us": 2406,
+      "p95_us": 3024,
+      "p99_us": 3417,
+      "sum_us": 5971006
+    },
+    "df.logical.build": {
+      "count": 9692,
+      "p50_us": 5,
+      "p95_us": 37,
+      "p99_us": 42,
+      "sum_us": 109562
+    },
+    "df.physical.plan": {
+      "count": 2423,
+      "p50_us": 884,
+      "p95_us": 1115,
+      "p99_us": 1337,
+      "sum_us": 2206267
+    },
+    "df.table.resolve": {
+      "count": 2423,
+      "p50_us": 22,
+      "p95_us": 33,
+      "p99_us": 52,
+      "sum_us": 57510
+    },
+    "last_checkpoint.read": {
+      "count": 1,
+      "p50_us": 172,
+      "p95_us": 172,
+      "p99_us": 172,
+      "sum_us": 172
+    },
+    "log.list": {
+      "count": 2,
+      "p50_us": 456,
+      "p95_us": 456,
+      "p99_us": 456,
+      "sum_us": 742
+    },
+    "log_seg.for_snap": {
+      "count": 1,
+      "p50_us": 465,
+      "p95_us": 465,
+      "p99_us": 465,
+      "sum_us": 465
+    },
+    "log_seg.load_p_m": {
+      "count": 2,
+      "p50_us": 1535,
+      "p95_us": 1535,
+      "p99_us": 1535,
+      "sum_us": 2427
+    },
+    "object_store.request": {
+      "count": 9691,
+      "p50_us": 1,
+      "p95_us": 2,
+      "p99_us": 4,
+      "sum_us": 13708
+    },
+    "snap": {
+      "count": 6,
+      "p50_us": 3435,
+      "p95_us": 10894,
+      "p99_us": 10894,
+      "sum_us": 24550
+    },
+    "snap.build": {
+      "count": 2,
+      "p50_us": 2064,
+      "p95_us": 2064,
+      "p99_us": 2064,
+      "sum_us": 3670
+    },
+    "snap.checkpoint": {
+      "count": 1,
+      "p50_us": 3425,
+      "p95_us": 3425,
+      "p99_us": 3425,
+      "sum_us": 3425
+    },
+    "trace.tree.build": {
+      "count": 2423,
+      "p50_us": 10,
+      "p95_us": 14,
+      "p99_us": 20,
+      "sum_us": 24851
+    },
+    "update_incremental": {
+      "count": 1,
+      "p50_us": 4749,
+      "p95_us": 4749,
+      "p99_us": 4749,
+      "sum_us": 4749
+    }
+  },
+  "object_store_counts": {
+    "list": 3,
+    "list_with_delimiter": 0,
+    "head": 1,
+    "get": 4,
+    "get_range": 9681,
+    "put": 2,
+    "delete": 0,
+    "copy": 0,
+    "bytes": 0
+  },
+  "refresh_on_request_path_total": 0
+}

crates/scouter_dataframe/bench_metrics/t0_hot_path_cold_query_smoke.json

@@ -0,0 +1,84 @@
+{
+  "commit": "553a607685000971ef06678e913db7429d263148",
+  "bench_group": "t0_hot_path_cold_query_smoke",
+  "tier": 0,
+  "blocking": true,
+  "scenario_class": "cold_query",
+  "runtime_budget_secs": 120,
+  "actual_runtime_secs": 9.834912333,
+  "fixture_rows": 10000,
+  "fixture_spans": 10000,
+  "storage_profile": "P1_local_nvme",
+  "query_entrypoint": "trace_query_service.query_spans",
+  "result_rows": 5,
+  "spans": {
+    "arrow.convert": {
+      "count": 2223,
+      "p50_us": 43,
+      "p95_us": 57,
+      "p99_us": 97,
+      "sum_us": 101569
+    },
+    "bench.query.end_to_end": {
+      "count": 2223,
+      "p50_us": 4059,
+      "p95_us": 4956,
+      "p99_us": 7507,
+      "sum_us": 9469517
+    },
+    "df.collect": {
+      "count": 2223,
+      "p50_us": 2797,
+      "p95_us": 3462,
+      "p99_us": 4834,
+      "sum_us": 6512581
+    },
+    "df.logical.build": {
+      "count": 17784,
+      "p50_us": 1,
+      "p95_us": 33,
+      "p99_us": 39,
+      "sum_us": 108283
+    },
+    "df.physical.plan": {
+      "count": 2223,
+      "p50_us": 1050,
+      "p95_us": 1392,
+      "p99_us": 2041,
+      "sum_us": 2530619
+    },
+    "df.table.resolve": {
+      "count": 2223,
+      "p50_us": 22,
+      "p95_us": 34,
+      "p99_us": 67,
+      "sum_us": 52642
+    },
+    "object_store.request": {
+      "count": 8892,
+      "p50_us": 1,
+      "p95_us": 2,
+      "p99_us": 3,
+      "sum_us": 6310
+    },
+    "trace.tree.build": {
+      "count": 2223,
+      "p50_us": 10,
+      "p95_us": 14,
+      "p99_us": 22,
+      "sum_us": 23339
+    }
+  },
+  "object_store_counts": {
+    "list": 0,
+    "list_with_delimiter": 0,
+    "head": 0,
+    "get": 0,
+    "get_range": 8892,
+    "put": 0,
+    "delete": 0,
+    "copy": 0,
+    "bytes": 0
+  },
+  "refresh_on_request_path_total": 0
+}

crates/scouter_dataframe/bench_metrics/t0_refresh_origin_sentinel.json

@@ -0,0 +1,25 @@
+{
+  "commit": "553a607685000971ef06678e913db7429d263148",
+  "bench_group": "t0_refresh_origin_sentinel",
+  "tier": 0,
+  "blocking": true,
+  "scenario_class": "refresh_origin_sentinel",
+  "runtime_budget_secs": 30,
+  "actual_runtime_secs": 9.58e-7,
+  "fixture_rows": 0,
+  "fixture_spans": 0,
+  "storage_profile": "P1_local_nvme",
+  "spans": {},
+  "object_store_counts": {
+    "list": 0,
+    "list_with_delimiter": 0,
+    "head": 0,
+    "get": 0,
+    "get_range": 0,
+    "put": 0,
+    "delete": 0,
+    "copy": 0,
+    "bytes": 0
+  },
+  "refresh_on_request_path_total": 0
+}