# HELP python_gc_objects_collected_total Objects collected during gc # TYPE python_gc_objects_collected_total counter python_gc_objects_collected_total{generation="0"} 26092.0 python_gc_objects_collected_total{generation="1"} 3.522661e+06 python_gc_objects_collected_total{generation="2"} 1.953524e+06 # HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC # TYPE python_gc_objects_uncollectable_total counter python_gc_objects_uncollectable_total{generation="0"} 0.0 python_gc_objects_uncollectable_total{generation="1"} 0.0 python_gc_objects_uncollectable_total{generation="2"} 0.0 # HELP python_gc_collections_total Number of times this generation was collected # TYPE python_gc_collections_total counter python_gc_collections_total{generation="0"} 54931.0 python_gc_collections_total{generation="1"} 4993.0 python_gc_collections_total{generation="2"} 371.0 # HELP python_info Python platform information # TYPE python_info gauge python_info{implementation="CPython",major="3",minor="10",patchlevel="20",version="3.10.20"} 1.0 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes 1.03202816e+09 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes 2.86244864e+08 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds 1.78490210344e+09 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total 3919.46 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge process_open_fds 121.0 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 # HELP miner_initiated_file_uploads_total Total miner initiated file uploads # TYPE miner_initiated_file_uploads_total counter miner_initiated_file_uploads_total{file_type="partition",layer_idx="1",miner_type="at_home"} 24.0 miner_initiated_file_uploads_total{file_type="partition",layer_idx="0",miner_type="at_home"} 28.0 miner_initiated_file_uploads_total{file_type="partition",layer_idx="2",miner_type="at_home"} 27.0 miner_initiated_file_uploads_total{file_type="weights",layer_idx="2",miner_type="at_home"} 5.0 miner_initiated_file_uploads_total{file_type="weights",layer_idx="0",miner_type="at_home"} 4.0 miner_initiated_file_uploads_total{file_type="weights",layer_idx="1",miner_type="at_home"} 8.0 # HELP miner_initiated_file_uploads_created Total miner initiated file uploads # TYPE miner_initiated_file_uploads_created gauge miner_initiated_file_uploads_created{file_type="partition",layer_idx="1",miner_type="at_home"} 1.7849022282337198e+09 miner_initiated_file_uploads_created{file_type="partition",layer_idx="0",miner_type="at_home"} 1.7849022364879684e+09 miner_initiated_file_uploads_created{file_type="partition",layer_idx="2",miner_type="at_home"} 1.7849024392833045e+09 miner_initiated_file_uploads_created{file_type="weights",layer_idx="2",miner_type="at_home"} 1.7849025855245962e+09 miner_initiated_file_uploads_created{file_type="weights",layer_idx="0",miner_type="at_home"} 1.784903875627367e+09 miner_initiated_file_uploads_created{file_type="weights",layer_idx="1",miner_type="at_home"} 1.7849039401457129e+09 # HELP miner_completed_file_uploads_total Total miner completed file uploads # TYPE miner_completed_file_uploads_total counter miner_completed_file_uploads_total{layer_idx="1",miner_type="at_home"} 36.0 miner_completed_file_uploads_total{layer_idx="0",miner_type="at_home"} 31.0 miner_completed_file_uploads_total{layer_idx="2",miner_type="at_home"} 45.0 # HELP miner_completed_file_uploads_created Total miner completed file uploads # TYPE miner_completed_file_uploads_created gauge miner_completed_file_uploads_created{layer_idx="1",miner_type="at_home"} 1.7849022445819511e+09 miner_completed_file_uploads_created{layer_idx="0",miner_type="at_home"} 1.7849022635664873e+09 miner_completed_file_uploads_created{layer_idx="2",miner_type="at_home"} 1.7849023491782165e+09 # HELP miner_state_resets_total Total miner state resets # TYPE miner_state_resets_total counter miner_state_resets_total{layer_idx="2",miner_status="initializing",miner_type="at_home"} 2.0 # HELP miner_state_resets_created Total miner state resets # TYPE miner_state_resets_created gauge miner_state_resets_created{layer_idx="2",miner_status="initializing",miner_type="at_home"} 1.7849041522573004e+09 # HELP cache_ops_total Cache operations for backend entities # TYPE cache_ops_total counter cache_ops_total{entity_type="miner_state",op="hit"} 50560.0 cache_ops_total{entity_type="run_state",op="hit"} 58615.0 cache_ops_total{entity_type="miner_state",op="miss"} 13301.0 cache_ops_total{entity_type="run_info",op="hit"} 650.0 cache_ops_total{entity_type="miner_counts",op="hit"} 988.0 cache_ops_total{entity_type="miner_state",op="write"} 4058.0 cache_ops_total{entity_type="enclave_payload",op="hit"} 1829.0 cache_ops_total{entity_type="run_info",op="miss"} 442.0 cache_ops_total{entity_type="run_info",op="write"} 442.0 cache_ops_total{entity_type="enclave_payload",op="miss"} 562.0 cache_ops_total{entity_type="enclave_payload",op="write"} 570.0 cache_ops_total{entity_type="run_state",op="miss"} 205.0 cache_ops_total{entity_type="run_state",op="write"} 216.0 cache_ops_total{entity_type="miner_counts",op="miss"} 104.0 cache_ops_total{entity_type="miner_counts",op="write"} 104.0 cache_ops_total{entity_type="miner_counts",op="invalidate"} 93.0 # HELP cache_ops_created Cache operations for backend entities # TYPE cache_ops_created gauge cache_ops_created{entity_type="miner_state",op="hit"} 1.7849021251279266e+09 cache_ops_created{entity_type="run_state",op="hit"} 1.7849021253803031e+09 cache_ops_created{entity_type="miner_state",op="miss"} 1.7849021269635742e+09 cache_ops_created{entity_type="run_info",op="hit"} 1.784902132012754e+09 cache_ops_created{entity_type="miner_counts",op="hit"} 1.7849021320144367e+09 cache_ops_created{entity_type="miner_state",op="write"} 1.7849021353386004e+09 cache_ops_created{entity_type="enclave_payload",op="hit"} 1.784902141523181e+09 cache_ops_created{entity_type="run_info",op="miss"} 1.7849021453751783e+09 cache_ops_created{entity_type="run_info",op="write"} 1.7849021454840643e+09 cache_ops_created{entity_type="enclave_payload",op="miss"} 1.7849021488412974e+09 cache_ops_created{entity_type="enclave_payload",op="write"} 1.7849021489914181e+09 cache_ops_created{entity_type="run_state",op="miss"} 1.7849021695230417e+09 cache_ops_created{entity_type="run_state",op="write"} 1.7849021697103837e+09 cache_ops_created{entity_type="miner_counts",op="miss"} 1.7849021839525738e+09 cache_ops_created{entity_type="miner_counts",op="write"} 1.7849021842085223e+09 cache_ops_created{entity_type="miner_counts",op="invalidate"} 1.784902531503766e+09 # HELP cache_latency_seconds Cache operation latency in seconds # TYPE cache_latency_seconds histogram cache_latency_seconds_bucket{entity_type="miner_state",le="0.001",op="read"} 9590.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.005",op="read"} 44473.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.01",op="read"} 57367.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.025",op="read"} 63283.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.05",op="read"} 63556.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.1",op="read"} 63740.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.25",op="read"} 63785.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.5",op="read"} 63791.0 cache_latency_seconds_bucket{entity_type="miner_state",le="1.0",op="read"} 63791.0 cache_latency_seconds_bucket{entity_type="miner_state",le="2.5",op="read"} 63791.0 cache_latency_seconds_bucket{entity_type="miner_state",le="5.0",op="read"} 63791.0 cache_latency_seconds_bucket{entity_type="miner_state",le="+Inf",op="read"} 63861.0 cache_latency_seconds_count{entity_type="miner_state",op="read"} 63861.0 cache_latency_seconds_sum{entity_type="miner_state",op="read"} 9431.662106752396 cache_latency_seconds_bucket{entity_type="run_state",le="0.001",op="read"} 4700.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.005",op="read"} 40903.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.01",op="read"} 52656.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.025",op="read"} 58048.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.05",op="read"} 58304.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.1",op="read"} 58518.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.25",op="read"} 58626.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.5",op="read"} 58764.0 cache_latency_seconds_bucket{entity_type="run_state",le="1.0",op="read"} 58807.0 cache_latency_seconds_bucket{entity_type="run_state",le="2.5",op="read"} 58807.0 cache_latency_seconds_bucket{entity_type="run_state",le="5.0",op="read"} 58807.0 cache_latency_seconds_bucket{entity_type="run_state",le="+Inf",op="read"} 58820.0 cache_latency_seconds_count{entity_type="run_state",op="read"} 58820.0 cache_latency_seconds_sum{entity_type="run_state",op="read"} 2065.037190914154 cache_latency_seconds_bucket{entity_type="run_info",le="0.001",op="read"} 119.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.005",op="read"} 805.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.01",op="read"} 990.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.025",op="read"} 1088.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.05",op="read"} 1089.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.1",op="read"} 1090.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.25",op="read"} 1092.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.5",op="read"} 1092.0 cache_latency_seconds_bucket{entity_type="run_info",le="1.0",op="read"} 1092.0 cache_latency_seconds_bucket{entity_type="run_info",le="2.5",op="read"} 1092.0 cache_latency_seconds_bucket{entity_type="run_info",le="5.0",op="read"} 1092.0 cache_latency_seconds_bucket{entity_type="run_info",le="+Inf",op="read"} 1092.0 cache_latency_seconds_count{entity_type="run_info",op="read"} 1092.0 cache_latency_seconds_sum{entity_type="run_info",op="read"} 4.670082330703735 cache_latency_seconds_bucket{entity_type="miner_state",le="0.001",op="write"} 181.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.005",op="write"} 2731.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.01",op="write"} 3632.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.025",op="write"} 4021.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.05",op="write"} 4039.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.1",op="write"} 4053.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.25",op="write"} 4056.0 cache_latency_seconds_bucket{entity_type="miner_state",le="0.5",op="write"} 4058.0 cache_latency_seconds_bucket{entity_type="miner_state",le="1.0",op="write"} 4058.0 cache_latency_seconds_bucket{entity_type="miner_state",le="2.5",op="write"} 4058.0 cache_latency_seconds_bucket{entity_type="miner_state",le="5.0",op="write"} 4058.0 cache_latency_seconds_bucket{entity_type="miner_state",le="+Inf",op="write"} 4058.0 cache_latency_seconds_count{entity_type="miner_state",op="write"} 4058.0 cache_latency_seconds_sum{entity_type="miner_state",op="write"} 20.410115003585815 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.001",op="read"} 411.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.005",op="read"} 1716.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.01",op="read"} 2155.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.025",op="read"} 2378.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.05",op="read"} 2386.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.1",op="read"} 2389.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.25",op="read"} 2390.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.5",op="read"} 2390.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="1.0",op="read"} 2390.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="2.5",op="read"} 2390.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="5.0",op="read"} 2390.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="+Inf",op="read"} 2391.0 cache_latency_seconds_count{entity_type="enclave_payload",op="read"} 2391.0 cache_latency_seconds_sum{entity_type="enclave_payload",op="read"} 140.22573041915894 cache_latency_seconds_bucket{entity_type="run_info",le="0.001",op="write"} 2.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.005",op="write"} 284.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.01",op="write"} 382.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.025",op="write"} 440.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.05",op="write"} 441.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.1",op="write"} 442.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.25",op="write"} 442.0 cache_latency_seconds_bucket{entity_type="run_info",le="0.5",op="write"} 442.0 cache_latency_seconds_bucket{entity_type="run_info",le="1.0",op="write"} 442.0 cache_latency_seconds_bucket{entity_type="run_info",le="2.5",op="write"} 442.0 cache_latency_seconds_bucket{entity_type="run_info",le="5.0",op="write"} 442.0 cache_latency_seconds_bucket{entity_type="run_info",le="+Inf",op="write"} 442.0 cache_latency_seconds_count{entity_type="run_info",op="write"} 442.0 cache_latency_seconds_sum{entity_type="run_info",op="write"} 2.309903383255005 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.001",op="write"} 35.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.005",op="write"} 394.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.01",op="write"} 505.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.025",op="write"} 564.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.05",op="write"} 566.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.1",op="write"} 570.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.25",op="write"} 570.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="0.5",op="write"} 570.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="1.0",op="write"} 570.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="2.5",op="write"} 570.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="5.0",op="write"} 570.0 cache_latency_seconds_bucket{entity_type="enclave_payload",le="+Inf",op="write"} 570.0 cache_latency_seconds_count{entity_type="enclave_payload",op="write"} 570.0 cache_latency_seconds_sum{entity_type="enclave_payload",op="write"} 2.8459525108337402 cache_latency_seconds_bucket{entity_type="run_state",le="0.001",op="write"} 4.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.005",op="write"} 131.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.01",op="write"} 185.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.025",op="write"} 213.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.05",op="write"} 215.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.1",op="write"} 216.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.25",op="write"} 216.0 cache_latency_seconds_bucket{entity_type="run_state",le="0.5",op="write"} 216.0 cache_latency_seconds_bucket{entity_type="run_state",le="1.0",op="write"} 216.0 cache_latency_seconds_bucket{entity_type="run_state",le="2.5",op="write"} 216.0 cache_latency_seconds_bucket{entity_type="run_state",le="5.0",op="write"} 216.0 cache_latency_seconds_bucket{entity_type="run_state",le="+Inf",op="write"} 216.0 cache_latency_seconds_count{entity_type="run_state",op="write"} 216.0 cache_latency_seconds_sum{entity_type="run_state",op="write"} 1.2169699668884277 # HELP cache_latency_seconds_created Cache operation latency in seconds # TYPE cache_latency_seconds_created gauge cache_latency_seconds_created{entity_type="miner_state",op="read"} 1.7849021251276891e+09 cache_latency_seconds_created{entity_type="run_state",op="read"} 1.7849021253791761e+09 cache_latency_seconds_created{entity_type="run_info",op="read"} 1.7849021320103238e+09 cache_latency_seconds_created{entity_type="miner_state",op="write"} 1.784902135338493e+09 cache_latency_seconds_created{entity_type="enclave_payload",op="read"} 1.784902141523022e+09 cache_latency_seconds_created{entity_type="run_info",op="write"} 1.7849021454839687e+09 cache_latency_seconds_created{entity_type="enclave_payload",op="write"} 1.7849021489913006e+09 cache_latency_seconds_created{entity_type="run_state",op="write"} 1.7849021697102468e+09 # HELP sample_purges_total Total sample purges # TYPE sample_purges_total counter # HELP activation_process_count Distribution of activation.process_count values # TYPE activation_process_count histogram # HELP discarded_activations_after_sync_total Number of activations that were discarded missing during sync # TYPE discarded_activations_after_sync_total counter # HELP bridge_sync_requests_total Total bridge /sync HTTP requests # TYPE bridge_sync_requests_total counter # HELP bridge_sync_request_duration_seconds Wall time to complete bridge /sync HTTP requests (including body parse and Redis) # TYPE bridge_sync_request_duration_seconds histogram # HELP http_requests_total Total number of requests by method, status and handler. # TYPE http_requests_total counter http_requests_total{handler="/healthcheck",method="GET",status="2xx"} 2059.0 http_requests_total{handler="/miner/all_layers_training",method="GET",status="2xx"} 34442.0 http_requests_total{handler="/validator/submit_task_result",method="POST",status="2xx"} 138.0 http_requests_total{handler="/miner/heartbeat",method="GET",status="2xx"} 6087.0 http_requests_total{handler="/miner/register/status/{queue_id}",method="GET",status="2xx"} 10932.0 http_requests_total{handler="/miner/request_attestation_challenge",method="POST",status="2xx"} 3462.0 http_requests_total{handler="/common/get_run_info",method="GET",status="2xx"} 1092.0 http_requests_total{handler="/miner/get_activations",method="GET",status="2xx"} 3906.0 http_requests_total{handler="/miner/register",method="POST",status="2xx"} 268.0 http_requests_total{handler="/miner/submit_activation",method="POST",status="2xx"} 2565.0 http_requests_total{handler="/common/get_merged_partitions",method="GET",status="2xx"} 57.0 http_requests_total{handler="/miner/get_layer_optimizer_state",method="GET",status="2xx"} 13.0 http_requests_total{handler="/miner/report_loss",method="POST",status="2xx"} 355.0 http_requests_total{handler="/miner/fleet-telemetry",method="POST",status="2xx"} 136.0 http_requests_total{handler="/miner/get_weight_path_per_layer",method="GET",status="2xx"} 487.0 http_requests_total{handler="/miner/get_partitions",method="GET",status="2xx"} 464.0 http_requests_total{handler="/miner/initiate_file_upload",method="POST",status="2xx"} 202.0 http_requests_total{handler="/miner/get_previous_partitions",method="GET",status="4xx"} 3.0 http_requests_total{handler="/miner/complete_multipart_upload",method="POST",status="2xx"} 112.0 http_requests_total{handler="/miner/get_previous_partitions",method="GET",status="2xx"} 97.0 http_requests_total{handler="/miner/submit_merged_partitions",method="POST",status="2xx"} 88.0 http_requests_total{handler="/miner/notify_orchestrator_of_state_call",method="GET",status="2xx"} 106.0 http_requests_total{handler="/miner/submit_activation",method="POST",status="4xx"} 15.0 http_requests_total{handler="/common/get_run_epoch",method="GET",status="2xx"} 52.0 http_requests_total{handler="/validator/fetch_task",method="GET",status="2xx"} 182.0 http_requests_total{handler="/miner/register",method="POST",status="5xx"} 1.0 http_requests_total{handler="/miner/register/confirm/{queue_id}",method="POST",status="2xx"} 24.0 http_requests_total{handler="/miner/payout_coldkey",method="POST",status="2xx"} 17.0 http_requests_total{handler="/validator/global_miner_scores",method="GET",status="2xx"} 2.0 http_requests_total{handler="/miner/submit_weights",method="POST",status="2xx"} 15.0 http_requests_total{handler="/profiler/status",method="GET",status="2xx"} 1.0 http_requests_total{handler="/miner/register/confirm/{queue_id}",method="POST",status="4xx"} 4.0 http_requests_total{handler="/miner/get_activations",method="GET",status="4xx"} 1.0 http_requests_total{handler="/miner/request_attestation_challenge",method="POST",status="5xx"} 1.0 # HELP http_requests_created Total number of requests by method, status and handler. # TYPE http_requests_created gauge http_requests_created{handler="/healthcheck",method="GET",status="2xx"} 1.7849021137041204e+09 http_requests_created{handler="/miner/all_layers_training",method="GET",status="2xx"} 1.7849021255384343e+09 http_requests_created{handler="/validator/submit_task_result",method="POST",status="2xx"} 1.7849021256865687e+09 http_requests_created{handler="/miner/heartbeat",method="GET",status="2xx"} 1.7849021263553128e+09 http_requests_created{handler="/miner/register/status/{queue_id}",method="GET",status="2xx"} 1.7849021276802802e+09 http_requests_created{handler="/miner/request_attestation_challenge",method="POST",status="2xx"} 1.7849021315424876e+09 http_requests_created{handler="/common/get_run_info",method="GET",status="2xx"} 1.7849021320822518e+09 http_requests_created{handler="/miner/get_activations",method="GET",status="2xx"} 1.784902133102772e+09 http_requests_created{handler="/miner/register",method="POST",status="2xx"} 1.784902136663909e+09 http_requests_created{handler="/miner/submit_activation",method="POST",status="2xx"} 1.784902142987666e+09 http_requests_created{handler="/common/get_merged_partitions",method="GET",status="2xx"} 1.7849021434865544e+09 http_requests_created{handler="/miner/get_layer_optimizer_state",method="GET",status="2xx"} 1.7849021474074187e+09 http_requests_created{handler="/miner/report_loss",method="POST",status="2xx"} 1.784902160043071e+09 http_requests_created{handler="/miner/fleet-telemetry",method="POST",status="2xx"} 1.7849021671550891e+09 http_requests_created{handler="/miner/get_weight_path_per_layer",method="GET",status="2xx"} 1.7849021853166735e+09 http_requests_created{handler="/miner/get_partitions",method="GET",status="2xx"} 1.7849021887837026e+09 http_requests_created{handler="/miner/initiate_file_upload",method="POST",status="2xx"} 1.7849022165145445e+09 http_requests_created{handler="/miner/get_previous_partitions",method="GET",status="4xx"} 1.7849022443299408e+09 http_requests_created{handler="/miner/complete_multipart_upload",method="POST",status="2xx"} 1.784902244589098e+09 http_requests_created{handler="/miner/get_previous_partitions",method="GET",status="2xx"} 1.7849022547523217e+09 http_requests_created{handler="/miner/submit_merged_partitions",method="POST",status="2xx"} 1.78490226022069e+09 http_requests_created{handler="/miner/notify_orchestrator_of_state_call",method="GET",status="2xx"} 1.7849023125335472e+09 http_requests_created{handler="/miner/submit_activation",method="POST",status="4xx"} 1.784902357898879e+09 http_requests_created{handler="/common/get_run_epoch",method="GET",status="2xx"} 1.784902414555988e+09 http_requests_created{handler="/validator/fetch_task",method="GET",status="2xx"} 1.784902457251497e+09 http_requests_created{handler="/miner/register",method="POST",status="5xx"} 1.7849025027245758e+09 http_requests_created{handler="/miner/register/confirm/{queue_id}",method="POST",status="2xx"} 1.784902593072908e+09 http_requests_created{handler="/miner/payout_coldkey",method="POST",status="2xx"} 1.784902623748923e+09 http_requests_created{handler="/validator/global_miner_scores",method="GET",status="2xx"} 1.784902734673162e+09 http_requests_created{handler="/miner/submit_weights",method="POST",status="2xx"} 1.7849036688784976e+09 http_requests_created{handler="/profiler/status",method="GET",status="2xx"} 1.784904323439447e+09 http_requests_created{handler="/miner/register/confirm/{queue_id}",method="POST",status="4xx"} 1.784904724229395e+09 http_requests_created{handler="/miner/get_activations",method="GET",status="4xx"} 1.784910367555541e+09 http_requests_created{handler="/miner/request_attestation_challenge",method="POST",status="5xx"} 1.7849111535306976e+09 # HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. # TYPE http_request_size_bytes summary http_request_size_bytes_count{handler="/healthcheck"} 2059.0 http_request_size_bytes_sum{handler="/healthcheck"} 0.0 http_request_size_bytes_count{handler="/miner/all_layers_training"} 34442.0 http_request_size_bytes_sum{handler="/miner/all_layers_training"} 0.0 http_request_size_bytes_count{handler="/validator/submit_task_result"} 138.0 http_request_size_bytes_sum{handler="/validator/submit_task_result"} 18907.0 http_request_size_bytes_count{handler="/miner/heartbeat"} 6087.0 http_request_size_bytes_sum{handler="/miner/heartbeat"} 12174.0 http_request_size_bytes_count{handler="/miner/register/status/{queue_id}"} 10932.0 http_request_size_bytes_sum{handler="/miner/register/status/{queue_id}"} 21836.0 http_request_size_bytes_count{handler="/miner/request_attestation_challenge"} 3463.0 http_request_size_bytes_sum{handler="/miner/request_attestation_challenge"} 160870.0 http_request_size_bytes_count{handler="/common/get_run_info"} 1092.0 http_request_size_bytes_sum{handler="/common/get_run_info"} 0.0 http_request_size_bytes_count{handler="/miner/get_activations"} 3907.0 http_request_size_bytes_sum{handler="/miner/get_activations"} 93772.0 http_request_size_bytes_count{handler="/miner/register"} 269.0 http_request_size_bytes_sum{handler="/miner/register"} 1.382268e+06 http_request_size_bytes_count{handler="/miner/submit_activation"} 2580.0 http_request_size_bytes_sum{handler="/miner/submit_activation"} 1.8228223e+07 http_request_size_bytes_count{handler="/common/get_merged_partitions"} 57.0 http_request_size_bytes_sum{handler="/common/get_merged_partitions"} 0.0 http_request_size_bytes_count{handler="/miner/get_layer_optimizer_state"} 13.0 http_request_size_bytes_sum{handler="/miner/get_layer_optimizer_state"} 0.0 http_request_size_bytes_count{handler="/miner/report_loss"} 355.0 http_request_size_bytes_sum{handler="/miner/report_loss"} 31666.0 http_request_size_bytes_count{handler="/miner/fleet-telemetry"} 136.0 http_request_size_bytes_sum{handler="/miner/fleet-telemetry"} 632137.0 http_request_size_bytes_count{handler="/miner/get_weight_path_per_layer"} 487.0 http_request_size_bytes_sum{handler="/miner/get_weight_path_per_layer"} 0.0 http_request_size_bytes_count{handler="/miner/get_partitions"} 464.0 http_request_size_bytes_sum{handler="/miner/get_partitions"} 0.0 http_request_size_bytes_count{handler="/miner/initiate_file_upload"} 202.0 http_request_size_bytes_sum{handler="/miner/initiate_file_upload"} 12205.0 http_request_size_bytes_count{handler="/miner/get_previous_partitions"} 100.0 http_request_size_bytes_sum{handler="/miner/get_previous_partitions"} 367.0 http_request_size_bytes_count{handler="/miner/complete_multipart_upload"} 112.0 http_request_size_bytes_sum{handler="/miner/complete_multipart_upload"} 210875.0 http_request_size_bytes_count{handler="/miner/submit_merged_partitions"} 88.0 http_request_size_bytes_sum{handler="/miner/submit_merged_partitions"} 424376.0 http_request_size_bytes_count{handler="/miner/notify_orchestrator_of_state_call"} 106.0 http_request_size_bytes_sum{handler="/miner/notify_orchestrator_of_state_call"} 0.0 http_request_size_bytes_count{handler="/common/get_run_epoch"} 52.0 http_request_size_bytes_sum{handler="/common/get_run_epoch"} 0.0 http_request_size_bytes_count{handler="/validator/fetch_task"} 182.0 http_request_size_bytes_sum{handler="/validator/fetch_task"} 0.0 http_request_size_bytes_count{handler="/miner/register/confirm/{queue_id}"} 28.0 http_request_size_bytes_sum{handler="/miner/register/confirm/{queue_id}"} 126668.0 http_request_size_bytes_count{handler="/miner/payout_coldkey"} 17.0 http_request_size_bytes_sum{handler="/miner/payout_coldkey"} 1190.0 http_request_size_bytes_count{handler="/validator/global_miner_scores"} 2.0 http_request_size_bytes_sum{handler="/validator/global_miner_scores"} 0.0 http_request_size_bytes_count{handler="/miner/submit_weights"} 15.0 http_request_size_bytes_sum{handler="/miner/submit_weights"} 70201.0 http_request_size_bytes_count{handler="/profiler/status"} 1.0 http_request_size_bytes_sum{handler="/profiler/status"} 0.0 # HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. # TYPE http_request_size_bytes_created gauge http_request_size_bytes_created{handler="/healthcheck"} 1.7849021137041628e+09 http_request_size_bytes_created{handler="/miner/all_layers_training"} 1.784902125538549e+09 http_request_size_bytes_created{handler="/validator/submit_task_result"} 1.7849021256866038e+09 http_request_size_bytes_created{handler="/miner/heartbeat"} 1.7849021263553555e+09 http_request_size_bytes_created{handler="/miner/register/status/{queue_id}"} 1.784902127680321e+09 http_request_size_bytes_created{handler="/miner/request_attestation_challenge"} 1.7849021315425277e+09 http_request_size_bytes_created{handler="/common/get_run_info"} 1.7849021320822875e+09 http_request_size_bytes_created{handler="/miner/get_activations"} 1.7849021331028147e+09 http_request_size_bytes_created{handler="/miner/register"} 1.7849021366639462e+09 http_request_size_bytes_created{handler="/miner/submit_activation"} 1.7849021429877098e+09 http_request_size_bytes_created{handler="/common/get_merged_partitions"} 1.784902143486606e+09 http_request_size_bytes_created{handler="/miner/get_layer_optimizer_state"} 1.784902147407449e+09 http_request_size_bytes_created{handler="/miner/report_loss"} 1.7849021600431383e+09 http_request_size_bytes_created{handler="/miner/fleet-telemetry"} 1.7849021671551273e+09 http_request_size_bytes_created{handler="/miner/get_weight_path_per_layer"} 1.784902185316715e+09 http_request_size_bytes_created{handler="/miner/get_partitions"} 1.7849021887837477e+09 http_request_size_bytes_created{handler="/miner/initiate_file_upload"} 1.7849022165145867e+09 http_request_size_bytes_created{handler="/miner/get_previous_partitions"} 1.784902244329966e+09 http_request_size_bytes_created{handler="/miner/complete_multipart_upload"} 1.7849022445891335e+09 http_request_size_bytes_created{handler="/miner/submit_merged_partitions"} 1.7849022602207363e+09 http_request_size_bytes_created{handler="/miner/notify_orchestrator_of_state_call"} 1.7849023125335982e+09 http_request_size_bytes_created{handler="/common/get_run_epoch"} 1.784902414556036e+09 http_request_size_bytes_created{handler="/validator/fetch_task"} 1.78490245725154e+09 http_request_size_bytes_created{handler="/miner/register/confirm/{queue_id}"} 1.7849025930729432e+09 http_request_size_bytes_created{handler="/miner/payout_coldkey"} 1.7849026237491486e+09 http_request_size_bytes_created{handler="/validator/global_miner_scores"} 1.7849027346732037e+09 http_request_size_bytes_created{handler="/miner/submit_weights"} 1.7849036688785355e+09 http_request_size_bytes_created{handler="/profiler/status"} 1.7849043234394789e+09 # HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. # TYPE http_response_size_bytes summary http_response_size_bytes_count{handler="/healthcheck"} 2059.0 http_response_size_bytes_sum{handler="/healthcheck"} 41180.0 http_response_size_bytes_count{handler="/miner/all_layers_training"} 34442.0 http_response_size_bytes_sum{handler="/miner/all_layers_training"} 138993.0 http_response_size_bytes_count{handler="/validator/submit_task_result"} 138.0 http_response_size_bytes_sum{handler="/validator/submit_task_result"} 552.0 http_response_size_bytes_count{handler="/miner/heartbeat"} 6087.0 http_response_size_bytes_sum{handler="/miner/heartbeat"} 428431.0 http_response_size_bytes_count{handler="/miner/register/status/{queue_id}"} 10932.0 http_response_size_bytes_sum{handler="/miner/register/status/{queue_id}"} 4.064143e+06 http_response_size_bytes_count{handler="/miner/request_attestation_challenge"} 3463.0 http_response_size_bytes_sum{handler="/miner/request_attestation_challenge"} 3.254864e+06 http_response_size_bytes_count{handler="/common/get_run_info"} 1092.0 http_response_size_bytes_sum{handler="/common/get_run_info"} 288192.0 http_response_size_bytes_count{handler="/miner/get_activations"} 3907.0 http_response_size_bytes_sum{handler="/miner/get_activations"} 1.173227e+06 http_response_size_bytes_count{handler="/miner/register"} 269.0 http_response_size_bytes_sum{handler="/miner/register"} 194399.0 http_response_size_bytes_count{handler="/miner/submit_activation"} 2580.0 http_response_size_bytes_sum{handler="/miner/submit_activation"} 36110.0 http_response_size_bytes_count{handler="/common/get_merged_partitions"} 57.0 http_response_size_bytes_sum{handler="/common/get_merged_partitions"} 1.229954e+06 http_response_size_bytes_count{handler="/miner/get_layer_optimizer_state"} 13.0 http_response_size_bytes_sum{handler="/miner/get_layer_optimizer_state"} 4454.0 http_response_size_bytes_count{handler="/miner/report_loss"} 355.0 http_response_size_bytes_sum{handler="/miner/report_loss"} 17269.0 http_response_size_bytes_count{handler="/miner/fleet-telemetry"} 136.0 http_response_size_bytes_sum{handler="/miner/fleet-telemetry"} 14408.0 http_response_size_bytes_count{handler="/miner/get_weight_path_per_layer"} 487.0 http_response_size_bytes_sum{handler="/miner/get_weight_path_per_layer"} 1.922142e+06 http_response_size_bytes_count{handler="/miner/get_partitions"} 464.0 http_response_size_bytes_sum{handler="/miner/get_partitions"} 13810.0 http_response_size_bytes_count{handler="/miner/initiate_file_upload"} 202.0 http_response_size_bytes_sum{handler="/miner/initiate_file_upload"} 1.779126e+06 http_response_size_bytes_count{handler="/miner/get_previous_partitions"} 100.0 http_response_size_bytes_sum{handler="/miner/get_previous_partitions"} 59171.0 http_response_size_bytes_count{handler="/miner/complete_multipart_upload"} 112.0 http_response_size_bytes_sum{handler="/miner/complete_multipart_upload"} 15840.0 http_response_size_bytes_count{handler="/miner/submit_merged_partitions"} 88.0 http_response_size_bytes_sum{handler="/miner/submit_merged_partitions"} 1559.0 http_response_size_bytes_count{handler="/miner/notify_orchestrator_of_state_call"} 106.0 http_response_size_bytes_sum{handler="/miner/notify_orchestrator_of_state_call"} 12937.0 http_response_size_bytes_count{handler="/common/get_run_epoch"} 52.0 http_response_size_bytes_sum{handler="/common/get_run_epoch"} 1882.0 http_response_size_bytes_count{handler="/validator/fetch_task"} 182.0 http_response_size_bytes_sum{handler="/validator/fetch_task"} 16599.0 http_response_size_bytes_count{handler="/miner/register/confirm/{queue_id}"} 28.0 http_response_size_bytes_sum{handler="/miner/register/confirm/{queue_id}"} 9410.0 http_response_size_bytes_count{handler="/miner/payout_coldkey"} 17.0 http_response_size_bytes_sum{handler="/miner/payout_coldkey"} 68.0 http_response_size_bytes_count{handler="/validator/global_miner_scores"} 2.0 http_response_size_bytes_sum{handler="/validator/global_miner_scores"} 1142.0 http_response_size_bytes_count{handler="/miner/submit_weights"} 15.0 http_response_size_bytes_sum{handler="/miner/submit_weights"} 1374.0 http_response_size_bytes_count{handler="/profiler/status"} 1.0 http_response_size_bytes_sum{handler="/profiler/status"} 41.0 # HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. # TYPE http_response_size_bytes_created gauge http_response_size_bytes_created{handler="/healthcheck"} 1.7849021137042174e+09 http_response_size_bytes_created{handler="/miner/all_layers_training"} 1.7849021255386176e+09 http_response_size_bytes_created{handler="/validator/submit_task_result"} 1.7849021256867087e+09 http_response_size_bytes_created{handler="/miner/heartbeat"} 1.7849021263554032e+09 http_response_size_bytes_created{handler="/miner/register/status/{queue_id}"} 1.7849021276803763e+09 http_response_size_bytes_created{handler="/miner/request_attestation_challenge"} 1.7849021315425775e+09 http_response_size_bytes_created{handler="/common/get_run_info"} 1.7849021320823388e+09 http_response_size_bytes_created{handler="/miner/get_activations"} 1.7849021331028676e+09 http_response_size_bytes_created{handler="/miner/register"} 1.7849021366639857e+09 http_response_size_bytes_created{handler="/miner/submit_activation"} 1.784902142987757e+09 http_response_size_bytes_created{handler="/common/get_merged_partitions"} 1.7849021434867077e+09 http_response_size_bytes_created{handler="/miner/get_layer_optimizer_state"} 1.7849021474074883e+09 http_response_size_bytes_created{handler="/miner/report_loss"} 1.7849021600431857e+09 http_response_size_bytes_created{handler="/miner/fleet-telemetry"} 1.7849021671551623e+09 http_response_size_bytes_created{handler="/miner/get_weight_path_per_layer"} 1.7849021853167667e+09 http_response_size_bytes_created{handler="/miner/get_partitions"} 1.784902188783792e+09 http_response_size_bytes_created{handler="/miner/initiate_file_upload"} 1.7849022165147402e+09 http_response_size_bytes_created{handler="/miner/get_previous_partitions"} 1.7849022443299985e+09 http_response_size_bytes_created{handler="/miner/complete_multipart_upload"} 1.784902244589189e+09 http_response_size_bytes_created{handler="/miner/submit_merged_partitions"} 1.7849022602207856e+09 http_response_size_bytes_created{handler="/miner/notify_orchestrator_of_state_call"} 1.7849023125336618e+09 http_response_size_bytes_created{handler="/common/get_run_epoch"} 1.7849024145560956e+09 http_response_size_bytes_created{handler="/validator/fetch_task"} 1.7849024572515953e+09 http_response_size_bytes_created{handler="/miner/register/confirm/{queue_id}"} 1.78490259307298e+09 http_response_size_bytes_created{handler="/miner/payout_coldkey"} 1.7849026237492173e+09 http_response_size_bytes_created{handler="/validator/global_miner_scores"} 1.784902734673253e+09 http_response_size_bytes_created{handler="/miner/submit_weights"} 1.7849036688785775e+09 http_response_size_bytes_created{handler="/profiler/status"} 1.7849043234395137e+09 # HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. # TYPE http_request_duration_highr_seconds histogram http_request_duration_highr_seconds_bucket{le="0.01"} 838.0 http_request_duration_highr_seconds_bucket{le="0.025"} 1894.0 http_request_duration_highr_seconds_bucket{le="0.05"} 2028.0 http_request_duration_highr_seconds_bucket{le="0.075"} 2046.0 http_request_duration_highr_seconds_bucket{le="0.1"} 2057.0 http_request_duration_highr_seconds_bucket{le="0.25"} 11289.0 http_request_duration_highr_seconds_bucket{le="0.5"} 38479.0 http_request_duration_highr_seconds_bucket{le="0.75"} 50152.0 http_request_duration_highr_seconds_bucket{le="1.0"} 59571.0 http_request_duration_highr_seconds_bucket{le="1.5"} 65318.0 http_request_duration_highr_seconds_bucket{le="2.0"} 65992.0 http_request_duration_highr_seconds_bucket{le="2.5"} 66337.0 http_request_duration_highr_seconds_bucket{le="3.0"} 66572.0 http_request_duration_highr_seconds_bucket{le="3.5"} 66679.0 http_request_duration_highr_seconds_bucket{le="4.0"} 66756.0 http_request_duration_highr_seconds_bucket{le="4.5"} 66823.0 http_request_duration_highr_seconds_bucket{le="5.0"} 66863.0 http_request_duration_highr_seconds_bucket{le="7.5"} 66985.0 http_request_duration_highr_seconds_bucket{le="10.0"} 67065.0 http_request_duration_highr_seconds_bucket{le="30.0"} 67206.0 http_request_duration_highr_seconds_bucket{le="60.0"} 67206.0 http_request_duration_highr_seconds_bucket{le="+Inf"} 67386.0 http_request_duration_highr_seconds_count 67386.0 http_request_duration_highr_seconds_sum 64944.47178055195 # HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. # TYPE http_request_duration_highr_seconds_created gauge http_request_duration_highr_seconds_created 1.7849021109896863e+09 # HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. # TYPE http_request_duration_seconds histogram http_request_duration_seconds_bucket{handler="/healthcheck",le="0.1",method="GET"} 2034.0 http_request_duration_seconds_bucket{handler="/healthcheck",le="0.5",method="GET"} 2059.0 http_request_duration_seconds_bucket{handler="/healthcheck",le="1.0",method="GET"} 2059.0 http_request_duration_seconds_bucket{handler="/healthcheck",le="+Inf",method="GET"} 2059.0 http_request_duration_seconds_count{handler="/healthcheck",method="GET"} 2059.0 http_request_duration_seconds_sum{handler="/healthcheck",method="GET"} 32.24938214849681 http_request_duration_seconds_bucket{handler="/miner/all_layers_training",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/all_layers_training",le="0.5",method="GET"} 23187.0 http_request_duration_seconds_bucket{handler="/miner/all_layers_training",le="1.0",method="GET"} 33789.0 http_request_duration_seconds_bucket{handler="/miner/all_layers_training",le="+Inf",method="GET"} 34442.0 http_request_duration_seconds_count{handler="/miner/all_layers_training",method="GET"} 34442.0 http_request_duration_seconds_sum{handler="/miner/all_layers_training",method="GET"} 19980.986210404197 http_request_duration_seconds_bucket{handler="/validator/submit_task_result",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/validator/submit_task_result",le="0.5",method="POST"} 121.0 http_request_duration_seconds_bucket{handler="/validator/submit_task_result",le="1.0",method="POST"} 138.0 http_request_duration_seconds_bucket{handler="/validator/submit_task_result",le="+Inf",method="POST"} 138.0 http_request_duration_seconds_count{handler="/validator/submit_task_result",method="POST"} 138.0 http_request_duration_seconds_sum{handler="/validator/submit_task_result",method="POST"} 48.796171987080015 http_request_duration_seconds_bucket{handler="/miner/heartbeat",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/heartbeat",le="0.5",method="GET"} 5546.0 http_request_duration_seconds_bucket{handler="/miner/heartbeat",le="1.0",method="GET"} 6034.0 http_request_duration_seconds_bucket{handler="/miner/heartbeat",le="+Inf",method="GET"} 6087.0 http_request_duration_seconds_count{handler="/miner/heartbeat",method="GET"} 6087.0 http_request_duration_seconds_sum{handler="/miner/heartbeat",method="GET"} 3706.8278370663757 http_request_duration_seconds_bucket{handler="/miner/register/status/{queue_id}",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/register/status/{queue_id}",le="0.5",method="GET"} 12.0 http_request_duration_seconds_bucket{handler="/miner/register/status/{queue_id}",le="1.0",method="GET"} 6711.0 http_request_duration_seconds_bucket{handler="/miner/register/status/{queue_id}",le="+Inf",method="GET"} 10932.0 http_request_duration_seconds_count{handler="/miner/register/status/{queue_id}",method="GET"} 10932.0 http_request_duration_seconds_sum{handler="/miner/register/status/{queue_id}",method="GET"} 20269.25955434621 http_request_duration_seconds_bucket{handler="/miner/request_attestation_challenge",le="0.1",method="POST"} 7.0 http_request_duration_seconds_bucket{handler="/miner/request_attestation_challenge",le="0.5",method="POST"} 3091.0 http_request_duration_seconds_bucket{handler="/miner/request_attestation_challenge",le="1.0",method="POST"} 3404.0 http_request_duration_seconds_bucket{handler="/miner/request_attestation_challenge",le="+Inf",method="POST"} 3463.0 http_request_duration_seconds_count{handler="/miner/request_attestation_challenge",method="POST"} 3463.0 http_request_duration_seconds_sum{handler="/miner/request_attestation_challenge",method="POST"} 4053.285138648702 http_request_duration_seconds_bucket{handler="/common/get_run_info",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/common/get_run_info",le="0.5",method="GET"} 1020.0 http_request_duration_seconds_bucket{handler="/common/get_run_info",le="1.0",method="GET"} 1085.0 http_request_duration_seconds_bucket{handler="/common/get_run_info",le="+Inf",method="GET"} 1092.0 http_request_duration_seconds_count{handler="/common/get_run_info",method="GET"} 1092.0 http_request_duration_seconds_sum{handler="/common/get_run_info",method="GET"} 344.485805850476 http_request_duration_seconds_bucket{handler="/miner/get_activations",le="0.1",method="GET"} 1.0 http_request_duration_seconds_bucket{handler="/miner/get_activations",le="0.5",method="GET"} 2568.0 http_request_duration_seconds_bucket{handler="/miner/get_activations",le="1.0",method="GET"} 2807.0 http_request_duration_seconds_bucket{handler="/miner/get_activations",le="+Inf",method="GET"} 3907.0 http_request_duration_seconds_count{handler="/miner/get_activations",method="GET"} 3907.0 http_request_duration_seconds_sum{handler="/miner/get_activations",method="GET"} 6033.924054212519 http_request_duration_seconds_bucket{handler="/miner/register",le="0.1",method="POST"} 1.0 http_request_duration_seconds_bucket{handler="/miner/register",le="0.5",method="POST"} 1.0 http_request_duration_seconds_bucket{handler="/miner/register",le="1.0",method="POST"} 92.0 http_request_duration_seconds_bucket{handler="/miner/register",le="+Inf",method="POST"} 269.0 http_request_duration_seconds_count{handler="/miner/register",method="POST"} 269.0 http_request_duration_seconds_sum{handler="/miner/register",method="POST"} 1615.7163788040634 http_request_duration_seconds_bucket{handler="/miner/submit_activation",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/submit_activation",le="0.5",method="POST"} 152.0 http_request_duration_seconds_bucket{handler="/miner/submit_activation",le="1.0",method="POST"} 1369.0 http_request_duration_seconds_bucket{handler="/miner/submit_activation",le="+Inf",method="POST"} 2580.0 http_request_duration_seconds_count{handler="/miner/submit_activation",method="POST"} 2580.0 http_request_duration_seconds_sum{handler="/miner/submit_activation",method="POST"} 6102.382450973615 http_request_duration_seconds_bucket{handler="/common/get_merged_partitions",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/common/get_merged_partitions",le="0.5",method="GET"} 38.0 http_request_duration_seconds_bucket{handler="/common/get_merged_partitions",le="1.0",method="GET"} 55.0 http_request_duration_seconds_bucket{handler="/common/get_merged_partitions",le="+Inf",method="GET"} 57.0 http_request_duration_seconds_count{handler="/common/get_merged_partitions",method="GET"} 57.0 http_request_duration_seconds_sum{handler="/common/get_merged_partitions",method="GET"} 28.442574276356027 http_request_duration_seconds_bucket{handler="/miner/get_layer_optimizer_state",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/get_layer_optimizer_state",le="0.5",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/get_layer_optimizer_state",le="1.0",method="GET"} 11.0 http_request_duration_seconds_bucket{handler="/miner/get_layer_optimizer_state",le="+Inf",method="GET"} 13.0 http_request_duration_seconds_count{handler="/miner/get_layer_optimizer_state",method="GET"} 13.0 http_request_duration_seconds_sum{handler="/miner/get_layer_optimizer_state",method="GET"} 141.27227284107357 http_request_duration_seconds_bucket{handler="/miner/report_loss",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/report_loss",le="0.5",method="POST"} 55.0 http_request_duration_seconds_bucket{handler="/miner/report_loss",le="1.0",method="POST"} 290.0 http_request_duration_seconds_bucket{handler="/miner/report_loss",le="+Inf",method="POST"} 355.0 http_request_duration_seconds_count{handler="/miner/report_loss",method="POST"} 355.0 http_request_duration_seconds_sum{handler="/miner/report_loss",method="POST"} 429.54393338772934 http_request_duration_seconds_bucket{handler="/miner/fleet-telemetry",le="0.1",method="POST"} 10.0 http_request_duration_seconds_bucket{handler="/miner/fleet-telemetry",le="0.5",method="POST"} 108.0 http_request_duration_seconds_bucket{handler="/miner/fleet-telemetry",le="1.0",method="POST"} 130.0 http_request_duration_seconds_bucket{handler="/miner/fleet-telemetry",le="+Inf",method="POST"} 136.0 http_request_duration_seconds_count{handler="/miner/fleet-telemetry",method="POST"} 136.0 http_request_duration_seconds_sum{handler="/miner/fleet-telemetry",method="POST"} 188.5956336298259 http_request_duration_seconds_bucket{handler="/miner/get_weight_path_per_layer",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/get_weight_path_per_layer",le="0.5",method="GET"} 90.0 http_request_duration_seconds_bucket{handler="/miner/get_weight_path_per_layer",le="1.0",method="GET"} 384.0 http_request_duration_seconds_bucket{handler="/miner/get_weight_path_per_layer",le="+Inf",method="GET"} 487.0 http_request_duration_seconds_count{handler="/miner/get_weight_path_per_layer",method="GET"} 487.0 http_request_duration_seconds_sum{handler="/miner/get_weight_path_per_layer",method="GET"} 455.0858465387719 http_request_duration_seconds_bucket{handler="/miner/get_partitions",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/get_partitions",le="0.5",method="GET"} 165.0 http_request_duration_seconds_bucket{handler="/miner/get_partitions",le="1.0",method="GET"} 456.0 http_request_duration_seconds_bucket{handler="/miner/get_partitions",le="+Inf",method="GET"} 464.0 http_request_duration_seconds_count{handler="/miner/get_partitions",method="GET"} 464.0 http_request_duration_seconds_sum{handler="/miner/get_partitions",method="GET"} 255.42909677978605 http_request_duration_seconds_bucket{handler="/miner/initiate_file_upload",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/initiate_file_upload",le="0.5",method="POST"} 98.0 http_request_duration_seconds_bucket{handler="/miner/initiate_file_upload",le="1.0",method="POST"} 194.0 http_request_duration_seconds_bucket{handler="/miner/initiate_file_upload",le="+Inf",method="POST"} 202.0 http_request_duration_seconds_count{handler="/miner/initiate_file_upload",method="POST"} 202.0 http_request_duration_seconds_sum{handler="/miner/initiate_file_upload",method="POST"} 115.64755174482707 http_request_duration_seconds_bucket{handler="/miner/get_previous_partitions",le="0.1",method="GET"} 3.0 http_request_duration_seconds_bucket{handler="/miner/get_previous_partitions",le="0.5",method="GET"} 26.0 http_request_duration_seconds_bucket{handler="/miner/get_previous_partitions",le="1.0",method="GET"} 98.0 http_request_duration_seconds_bucket{handler="/miner/get_previous_partitions",le="+Inf",method="GET"} 100.0 http_request_duration_seconds_count{handler="/miner/get_previous_partitions",method="GET"} 100.0 http_request_duration_seconds_sum{handler="/miner/get_previous_partitions",method="GET"} 60.58731795742642 http_request_duration_seconds_bucket{handler="/miner/complete_multipart_upload",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/complete_multipart_upload",le="0.5",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/complete_multipart_upload",le="1.0",method="POST"} 77.0 http_request_duration_seconds_bucket{handler="/miner/complete_multipart_upload",le="+Inf",method="POST"} 112.0 http_request_duration_seconds_count{handler="/miner/complete_multipart_upload",method="POST"} 112.0 http_request_duration_seconds_sum{handler="/miner/complete_multipart_upload",method="POST"} 105.14068723004311 http_request_duration_seconds_bucket{handler="/miner/submit_merged_partitions",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/submit_merged_partitions",le="0.5",method="POST"} 7.0 http_request_duration_seconds_bucket{handler="/miner/submit_merged_partitions",le="1.0",method="POST"} 76.0 http_request_duration_seconds_bucket{handler="/miner/submit_merged_partitions",le="+Inf",method="POST"} 88.0 http_request_duration_seconds_count{handler="/miner/submit_merged_partitions",method="POST"} 88.0 http_request_duration_seconds_sum{handler="/miner/submit_merged_partitions",method="POST"} 73.38329977088142 http_request_duration_seconds_bucket{handler="/miner/notify_orchestrator_of_state_call",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/miner/notify_orchestrator_of_state_call",le="0.5",method="GET"} 21.0 http_request_duration_seconds_bucket{handler="/miner/notify_orchestrator_of_state_call",le="1.0",method="GET"} 95.0 http_request_duration_seconds_bucket{handler="/miner/notify_orchestrator_of_state_call",le="+Inf",method="GET"} 106.0 http_request_duration_seconds_count{handler="/miner/notify_orchestrator_of_state_call",method="GET"} 106.0 http_request_duration_seconds_sum{handler="/miner/notify_orchestrator_of_state_call",method="GET"} 194.73695380357094 http_request_duration_seconds_bucket{handler="/common/get_run_epoch",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/common/get_run_epoch",le="0.5",method="GET"} 51.0 http_request_duration_seconds_bucket{handler="/common/get_run_epoch",le="1.0",method="GET"} 52.0 http_request_duration_seconds_bucket{handler="/common/get_run_epoch",le="+Inf",method="GET"} 52.0 http_request_duration_seconds_count{handler="/common/get_run_epoch",method="GET"} 52.0 http_request_duration_seconds_sum{handler="/common/get_run_epoch",method="GET"} 10.910674093756825 http_request_duration_seconds_bucket{handler="/validator/fetch_task",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/validator/fetch_task",le="0.5",method="GET"} 50.0 http_request_duration_seconds_bucket{handler="/validator/fetch_task",le="1.0",method="GET"} 133.0 http_request_duration_seconds_bucket{handler="/validator/fetch_task",le="+Inf",method="GET"} 182.0 http_request_duration_seconds_count{handler="/validator/fetch_task",method="GET"} 182.0 http_request_duration_seconds_sum{handler="/validator/fetch_task",method="GET"} 400.81889472249895 http_request_duration_seconds_bucket{handler="/miner/register/confirm/{queue_id}",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/register/confirm/{queue_id}",le="0.5",method="POST"} 9.0 http_request_duration_seconds_bucket{handler="/miner/register/confirm/{queue_id}",le="1.0",method="POST"} 27.0 http_request_duration_seconds_bucket{handler="/miner/register/confirm/{queue_id}",le="+Inf",method="POST"} 28.0 http_request_duration_seconds_count{handler="/miner/register/confirm/{queue_id}",method="POST"} 28.0 http_request_duration_seconds_sum{handler="/miner/register/confirm/{queue_id}",method="POST"} 15.14387533988338 http_request_duration_seconds_bucket{handler="/miner/payout_coldkey",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/payout_coldkey",le="0.5",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/payout_coldkey",le="1.0",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/payout_coldkey",le="+Inf",method="POST"} 17.0 http_request_duration_seconds_count{handler="/miner/payout_coldkey",method="POST"} 17.0 http_request_duration_seconds_sum{handler="/miner/payout_coldkey",method="POST"} 263.6922129270388 http_request_duration_seconds_bucket{handler="/validator/global_miner_scores",le="0.1",method="GET"} 0.0 http_request_duration_seconds_bucket{handler="/validator/global_miner_scores",le="0.5",method="GET"} 1.0 http_request_duration_seconds_bucket{handler="/validator/global_miner_scores",le="1.0",method="GET"} 2.0 http_request_duration_seconds_bucket{handler="/validator/global_miner_scores",le="+Inf",method="GET"} 2.0 http_request_duration_seconds_count{handler="/validator/global_miner_scores",method="GET"} 2.0 http_request_duration_seconds_sum{handler="/validator/global_miner_scores",method="GET"} 0.6934477589093149 http_request_duration_seconds_bucket{handler="/miner/submit_weights",le="0.1",method="POST"} 0.0 http_request_duration_seconds_bucket{handler="/miner/submit_weights",le="0.5",method="POST"} 2.0 http_request_duration_seconds_bucket{handler="/miner/submit_weights",le="1.0",method="POST"} 2.0 http_request_duration_seconds_bucket{handler="/miner/submit_weights",le="+Inf",method="POST"} 15.0 http_request_duration_seconds_count{handler="/miner/submit_weights",method="POST"} 15.0 http_request_duration_seconds_sum{handler="/miner/submit_weights",method="POST"} 17.427900718757883 http_request_duration_seconds_bucket{handler="/profiler/status",le="0.1",method="GET"} 1.0 http_request_duration_seconds_bucket{handler="/profiler/status",le="0.5",method="GET"} 1.0 http_request_duration_seconds_bucket{handler="/profiler/status",le="1.0",method="GET"} 1.0 http_request_duration_seconds_bucket{handler="/profiler/status",le="+Inf",method="GET"} 1.0 http_request_duration_seconds_count{handler="/profiler/status",method="GET"} 1.0 http_request_duration_seconds_sum{handler="/profiler/status",method="GET"} 0.00662258907686919 # HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. # TYPE http_request_duration_seconds_created gauge http_request_duration_seconds_created{handler="/healthcheck",method="GET"} 1.7849021137042825e+09 http_request_duration_seconds_created{handler="/miner/all_layers_training",method="GET"} 1.784902125538714e+09 http_request_duration_seconds_created{handler="/validator/submit_task_result",method="POST"} 1.7849021256867692e+09 http_request_duration_seconds_created{handler="/miner/heartbeat",method="GET"} 1.784902126355453e+09 http_request_duration_seconds_created{handler="/miner/register/status/{queue_id}",method="GET"} 1.7849021276804395e+09 http_request_duration_seconds_created{handler="/miner/request_attestation_challenge",method="POST"} 1.7849021315426683e+09 http_request_duration_seconds_created{handler="/common/get_run_info",method="GET"} 1.7849021320823944e+09 http_request_duration_seconds_created{handler="/miner/get_activations",method="GET"} 1.7849021331029296e+09 http_request_duration_seconds_created{handler="/miner/register",method="POST"} 1.7849021366640425e+09 http_request_duration_seconds_created{handler="/miner/submit_activation",method="POST"} 1.7849021429878163e+09 http_request_duration_seconds_created{handler="/common/get_merged_partitions",method="GET"} 1.7849021434867775e+09 http_request_duration_seconds_created{handler="/miner/get_layer_optimizer_state",method="GET"} 1.784902147407541e+09 http_request_duration_seconds_created{handler="/miner/report_loss",method="POST"} 1.7849021600432425e+09 http_request_duration_seconds_created{handler="/miner/fleet-telemetry",method="POST"} 1.784902167155206e+09 http_request_duration_seconds_created{handler="/miner/get_weight_path_per_layer",method="GET"} 1.7849021853168297e+09 http_request_duration_seconds_created{handler="/miner/get_partitions",method="GET"} 1.784902188783846e+09 http_request_duration_seconds_created{handler="/miner/initiate_file_upload",method="POST"} 1.7849022165148103e+09 http_request_duration_seconds_created{handler="/miner/get_previous_partitions",method="GET"} 1.784902244330054e+09 http_request_duration_seconds_created{handler="/miner/complete_multipart_upload",method="POST"} 1.7849022445892348e+09 http_request_duration_seconds_created{handler="/miner/submit_merged_partitions",method="POST"} 1.784902260220843e+09 http_request_duration_seconds_created{handler="/miner/notify_orchestrator_of_state_call",method="GET"} 1.7849023125337367e+09 http_request_duration_seconds_created{handler="/common/get_run_epoch",method="GET"} 1.7849024145561604e+09 http_request_duration_seconds_created{handler="/validator/fetch_task",method="GET"} 1.7849024572516587e+09 http_request_duration_seconds_created{handler="/miner/register/confirm/{queue_id}",method="POST"} 1.7849025930730307e+09 http_request_duration_seconds_created{handler="/miner/payout_coldkey",method="POST"} 1.784902623749289e+09 http_request_duration_seconds_created{handler="/validator/global_miner_scores",method="GET"} 1.784902734673305e+09 http_request_duration_seconds_created{handler="/miner/submit_weights",method="POST"} 1.7849036688786254e+09 http_request_duration_seconds_created{handler="/profiler/status",method="GET"} 1.784904323439558e+09 # HELP iota_pg_transactions_fail_total Counter for the number of failed async queries # TYPE iota_pg_transactions_fail_total counter iota_pg_transactions_fail_total{project="iota",subsystem="database"} 4.0 # HELP iota_pg_transactions_fail_created Counter for the number of failed async queries # TYPE iota_pg_transactions_fail_created gauge iota_pg_transactions_fail_created{project="iota",subsystem="database"} 1.7849021118832016e+09 # HELP iota_pg_transactions_success_total Counter for the number of successful async queries # TYPE iota_pg_transactions_success_total counter iota_pg_transactions_success_total{project="iota",subsystem="database"} 205524.0 # HELP iota_pg_transactions_success_created Counter for the number of successful async queries # TYPE iota_pg_transactions_success_created gauge iota_pg_transactions_success_created{project="iota",subsystem="database"} 1.7849021118832467e+09 # HELP iota_pg_transactions_total Counter for the number of total async queries # TYPE iota_pg_transactions_total counter iota_pg_transactions_total{project="iota",subsystem="database"} 205538.0 # HELP iota_pg_transactions_created Counter for the number of total async queries # TYPE iota_pg_transactions_created gauge iota_pg_transactions_created{project="iota",subsystem="database"} 1.7849021118832796e+09 # HELP iota_pg_transactions_duration Gauge for async transaction duration # TYPE iota_pg_transactions_duration gauge iota_pg_transactions_duration{project="iota",subsystem="database"} 0.25103388796560466 # HELP snowpipe_rows_sent_total Total rows sent to Snowflake via Snowpipe Streaming # TYPE snowpipe_rows_sent_total counter snowpipe_rows_sent_total{message_type="loss_report",source_service="orchestrator",table="TRAINING_METRICS"} 322.0 snowpipe_rows_sent_total{message_type="prometheus_snapshot",source_service="miner",table="MINER_TELEMETRY"} 629.0 snowpipe_rows_sent_total{message_type="node_location",source_service="miner",table="LIFECYCLE_EVENTS"} 8.0 # HELP snowpipe_rows_sent_created Total rows sent to Snowflake via Snowpipe Streaming # TYPE snowpipe_rows_sent_created gauge snowpipe_rows_sent_created{message_type="loss_report",source_service="orchestrator",table="TRAINING_METRICS"} 1.7849021600263133e+09 snowpipe_rows_sent_created{message_type="prometheus_snapshot",source_service="miner",table="MINER_TELEMETRY"} 1.7849022323118172e+09 snowpipe_rows_sent_created{message_type="node_location",source_service="miner",table="LIFECYCLE_EVENTS"} 1.7849032810688157e+09 # HELP snowpipe_errors_total Total Snowpipe send errors # TYPE snowpipe_errors_total counter # HELP snowpipe_send_latency_seconds Latency of Snowpipe append_rows calls # TYPE snowpipe_send_latency_seconds histogram snowpipe_send_latency_seconds_bucket{le="0.001",table="TRAINING_METRICS"} 152.0 snowpipe_send_latency_seconds_bucket{le="0.005",table="TRAINING_METRICS"} 312.0 snowpipe_send_latency_seconds_bucket{le="0.01",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="0.025",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="0.05",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="0.1",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="0.25",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="0.5",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="1.0",table="TRAINING_METRICS"} 321.0 snowpipe_send_latency_seconds_bucket{le="2.5",table="TRAINING_METRICS"} 322.0 snowpipe_send_latency_seconds_bucket{le="5.0",table="TRAINING_METRICS"} 322.0 snowpipe_send_latency_seconds_bucket{le="+Inf",table="TRAINING_METRICS"} 322.0 snowpipe_send_latency_seconds_count{table="TRAINING_METRICS"} 322.0 snowpipe_send_latency_seconds_sum{table="TRAINING_METRICS"} 1.4873002178501338 snowpipe_send_latency_seconds_bucket{le="0.001",table="MINER_TELEMETRY"} 13.0 snowpipe_send_latency_seconds_bucket{le="0.005",table="MINER_TELEMETRY"} 26.0 snowpipe_send_latency_seconds_bucket{le="0.01",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="0.025",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="0.05",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="0.1",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="0.25",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="0.5",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="1.0",table="MINER_TELEMETRY"} 27.0 snowpipe_send_latency_seconds_bucket{le="2.5",table="MINER_TELEMETRY"} 28.0 snowpipe_send_latency_seconds_bucket{le="5.0",table="MINER_TELEMETRY"} 28.0 snowpipe_send_latency_seconds_bucket{le="+Inf",table="MINER_TELEMETRY"} 28.0 snowpipe_send_latency_seconds_count{table="MINER_TELEMETRY"} 28.0 snowpipe_send_latency_seconds_sum{table="MINER_TELEMETRY"} 1.1672569909133017 snowpipe_send_latency_seconds_bucket{le="0.001",table="LIFECYCLE_EVENTS"} 3.0 snowpipe_send_latency_seconds_bucket{le="0.005",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="0.01",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="0.025",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="0.05",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="0.1",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="0.25",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="0.5",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="1.0",table="LIFECYCLE_EVENTS"} 7.0 snowpipe_send_latency_seconds_bucket{le="2.5",table="LIFECYCLE_EVENTS"} 8.0 snowpipe_send_latency_seconds_bucket{le="5.0",table="LIFECYCLE_EVENTS"} 8.0 snowpipe_send_latency_seconds_bucket{le="+Inf",table="LIFECYCLE_EVENTS"} 8.0 snowpipe_send_latency_seconds_count{table="LIFECYCLE_EVENTS"} 8.0 snowpipe_send_latency_seconds_sum{table="LIFECYCLE_EVENTS"} 1.2000465678283945 # HELP snowpipe_send_latency_seconds_created Latency of Snowpipe append_rows calls # TYPE snowpipe_send_latency_seconds_created gauge snowpipe_send_latency_seconds_created{table="TRAINING_METRICS"} 1.7849021600263681e+09 snowpipe_send_latency_seconds_created{table="MINER_TELEMETRY"} 1.7849022323118787e+09 snowpipe_send_latency_seconds_created{table="LIFECYCLE_EVENTS"} 1.7849032810688553e+09 # HELP snowpipe_channel_reopens_total Times a Snowpipe channel was re-opened after going stale # TYPE snowpipe_channel_reopens_total counter