# HELP template_migration_templates_migrated_total Templates migrated onto skills, split by migration phase # TYPE template_migration_templates_migrated_total counter template_migration_templates_migrated_total{phase="attached"} 0.0 template_migration_templates_migrated_total{phase="standalone"} 0.0 # HELP template_migration_templates_migrated_created Templates migrated onto skills, split by migration phase # TYPE template_migration_templates_migrated_created gauge template_migration_templates_migrated_created{phase="attached"} 1.7869676534692943e+09 template_migration_templates_migrated_created{phase="standalone"} 1.7869676534693031e+09 # HELP template_migration_users_migrated_total Users whose candidate templates were processed, by outcome # TYPE template_migration_users_migrated_total counter template_migration_users_migrated_total{status="success"} 0.0 template_migration_users_migrated_total{status="failed"} 0.0 # HELP template_migration_users_migrated_created Users whose candidate templates were processed, by outcome # TYPE template_migration_users_migrated_created gauge template_migration_users_migrated_created{status="success"} 1.786967653469311e+09 template_migration_users_migrated_created{status="failed"} 1.7869676534693177e+09 # HELP template_migration_skills_created_total Placeholder skills created for standalone templates # TYPE template_migration_skills_created_total counter template_migration_skills_created_total 0.0 # HELP template_migration_skills_created_created Placeholder skills created for standalone templates # TYPE template_migration_skills_created_created gauge template_migration_skills_created_created 1.7869676534691749e+09 # HELP template_migration_share_grants_total Share bridges copied from templates onto skills, by grantee kind # TYPE template_migration_share_grants_total counter template_migration_share_grants_total{kind="user"} 0.0 template_migration_share_grants_total{kind="org"} 0.0 # HELP template_migration_share_grants_created Share bridges copied from templates onto skills, by grantee kind # TYPE template_migration_share_grants_created gauge template_migration_share_grants_created{kind="user"} 1.7869676534693246e+09 template_migration_share_grants_created{kind="org"} 1.7869676534693305e+09 # HELP template_migration_anomalies_total Anomalies observed while migrating, by classified type # TYPE template_migration_anomalies_total counter template_migration_anomalies_total{type="soft_deleted_skill"} 0.0 template_migration_anomalies_total{type="ownerless"} 0.0 template_migration_anomalies_total{type="scope_public_no_org"} 0.0 template_migration_anomalies_total{type="share_owner_conflict"} 0.0 template_migration_anomalies_total{type="other"} 0.0 # HELP template_migration_anomalies_created Anomalies observed while migrating, by classified type # TYPE template_migration_anomalies_created gauge template_migration_anomalies_created{type="soft_deleted_skill"} 1.7869676534693363e+09 template_migration_anomalies_created{type="ownerless"} 1.7869676534693434e+09 template_migration_anomalies_created{type="scope_public_no_org"} 1.7869676534693503e+09 template_migration_anomalies_created{type="share_owner_conflict"} 1.7869676534693568e+09 template_migration_anomalies_created{type="other"} 1.786967653469363e+09 # HELP template_migration_ticks_total Migration ticks, by outcome # TYPE template_migration_ticks_total counter template_migration_ticks_total{outcome="completed"} 0.0 template_migration_ticks_total{outcome="disabled"} 4.0 template_migration_ticks_total{outcome="failed"} 0.0 # HELP template_migration_ticks_created Migration ticks, by outcome # TYPE template_migration_ticks_created gauge template_migration_ticks_created{outcome="completed"} 1.7869676534693694e+09 template_migration_ticks_created{outcome="disabled"} 1.7869676534693758e+09 template_migration_ticks_created{outcome="failed"} 1.7869676534693813e+09 # HELP template_migration_tick_duration_seconds Wall-clock duration of one migration tick end to end # TYPE template_migration_tick_duration_seconds histogram template_migration_tick_duration_seconds_bucket{le="1.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="5.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="15.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="30.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="60.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="120.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="300.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="600.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="1200.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="1800.0"} 4.0 template_migration_tick_duration_seconds_bucket{le="+Inf"} 4.0 template_migration_tick_duration_seconds_count 4.0 template_migration_tick_duration_seconds_sum 0.0011425958946347237 # HELP template_migration_tick_duration_seconds_created Wall-clock duration of one migration tick end to end # TYPE template_migration_tick_duration_seconds_created gauge template_migration_tick_duration_seconds_created 1.7869676534692175e+09 # HELP template_migration_user_duration_seconds Wall-clock duration of migrating one user's candidate templates # TYPE template_migration_user_duration_seconds histogram template_migration_user_duration_seconds_bucket{le="0.5"} 0.0 template_migration_user_duration_seconds_bucket{le="1.0"} 0.0 template_migration_user_duration_seconds_bucket{le="2.5"} 0.0 template_migration_user_duration_seconds_bucket{le="5.0"} 0.0 template_migration_user_duration_seconds_bucket{le="10.0"} 0.0 template_migration_user_duration_seconds_bucket{le="30.0"} 0.0 template_migration_user_duration_seconds_bucket{le="60.0"} 0.0 template_migration_user_duration_seconds_bucket{le="120.0"} 0.0 template_migration_user_duration_seconds_bucket{le="300.0"} 0.0 template_migration_user_duration_seconds_bucket{le="600.0"} 0.0 template_migration_user_duration_seconds_bucket{le="+Inf"} 0.0 template_migration_user_duration_seconds_count 0.0 template_migration_user_duration_seconds_sum 0.0 # HELP template_migration_user_duration_seconds_created Wall-clock duration of migrating one user's candidate templates # TYPE template_migration_user_duration_seconds_created gauge template_migration_user_duration_seconds_created 1.7869676534692512e+09 # HELP template_migration_progress_collect_failures_total Scrape-time progress-gauge queries that failed (timeout/DB error). Non-zero means the remaining/deferred gauges below are stale zeros, not a real 0. # TYPE template_migration_progress_collect_failures_total counter template_migration_progress_collect_failures_total 66.0 # HELP template_migration_progress_collect_failures_created Scrape-time progress-gauge queries that failed (timeout/DB error). Non-zero means the remaining/deferred gauges below are stale zeros, not a real 0. # TYPE template_migration_progress_collect_failures_created gauge template_migration_progress_collect_failures_created 1.786967653469277e+09 # HELP template_migration_templates_remaining Candidate templates not yet migrated, by category # TYPE template_migration_templates_remaining gauge template_migration_templates_remaining{category="attached"} 35.0 template_migration_templates_remaining{category="standalone"} 70.0 # HELP template_migration_users_remaining Distinct owners of candidate templates not yet migrated # TYPE template_migration_users_remaining gauge template_migration_users_remaining 19.0 # HELP template_migration_templates_deferred Templates this migration pass deliberately will not pick up, by reason # TYPE template_migration_templates_deferred gauge template_migration_templates_deferred{reason="workflow_referenced"} 0.0 template_migration_templates_deferred{reason="ownerless"} 0.0 template_migration_templates_deferred{reason="max_attempts"} 0.0 # HELP teams_bot_activities_success_total Teams bot activities that completed the full pipeline successfully # TYPE teams_bot_activities_success_total counter teams_bot_activities_success_total 0.0 # HELP teams_bot_activities_success_created Teams bot activities that completed the full pipeline successfully # TYPE teams_bot_activities_success_created gauge teams_bot_activities_success_created 1.7869676562137995e+09 # HELP teams_bot_activities_failed_total Teams bot activities that failed # TYPE teams_bot_activities_failed_total counter teams_bot_activities_failed_total{reason="unknown_invoker"} 0.0 teams_bot_activities_failed_total{reason="feature_not_enabled"} 0.0 teams_bot_activities_failed_total{reason="agent_failed"} 0.0 teams_bot_activities_failed_total{reason="timeout"} 0.0 teams_bot_activities_failed_total{reason="unknown"} 0.0 # HELP teams_bot_activities_failed_created Teams bot activities that failed # TYPE teams_bot_activities_failed_created gauge teams_bot_activities_failed_created{reason="unknown_invoker"} 1.7869676562138739e+09 teams_bot_activities_failed_created{reason="feature_not_enabled"} 1.78696765621388e+09 teams_bot_activities_failed_created{reason="agent_failed"} 1.786967656213886e+09 teams_bot_activities_failed_created{reason="timeout"} 1.786967656213891e+09 teams_bot_activities_failed_created{reason="unknown"} 1.7869676562138963e+09 # HELP teams_bot_followup_total Same-thread follow-up mentions steered into (merged) or deferred behind a running turn # TYPE teams_bot_followup_total counter teams_bot_followup_total{outcome="merged"} 0.0 teams_bot_followup_total{outcome="deferred"} 0.0 teams_bot_followup_total{outcome="fallback_redispatch"} 0.0 # HELP teams_bot_followup_created Same-thread follow-up mentions steered into (merged) or deferred behind a running turn # TYPE teams_bot_followup_created gauge teams_bot_followup_created{outcome="merged"} 1.7869676562138522e+09 teams_bot_followup_created{outcome="deferred"} 1.78696765621386e+09 teams_bot_followup_created{outcome="fallback_redispatch"} 1.786967656213866e+09 # HELP teams_bot_processing_seconds End-to-end processing time per activity (setup → reply sent or failed) # TYPE teams_bot_processing_seconds histogram teams_bot_processing_seconds_bucket{le="5.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="15.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="30.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="60.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="120.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="300.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="600.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="1200.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="1800.0",status="success"} 0.0 teams_bot_processing_seconds_bucket{le="+Inf",status="success"} 0.0 teams_bot_processing_seconds_count{status="success"} 0.0 teams_bot_processing_seconds_sum{status="success"} 0.0 teams_bot_processing_seconds_bucket{le="5.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="15.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="30.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="60.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="120.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="300.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="600.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="1200.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="1800.0",status="failed"} 0.0 teams_bot_processing_seconds_bucket{le="+Inf",status="failed"} 0.0 teams_bot_processing_seconds_count{status="failed"} 0.0 teams_bot_processing_seconds_sum{status="failed"} 0.0 # HELP teams_bot_processing_seconds_created End-to-end processing time per activity (setup → reply sent or failed) # TYPE teams_bot_processing_seconds_created gauge teams_bot_processing_seconds_created{status="success"} 1.7869676562139368e+09 teams_bot_processing_seconds_created{status="failed"} 1.7869676562139807e+09 # HELP teams_bot_canned_reply_failed_total Canned sign-up/disabled/failure replies that never reached the user (connector send exhausted its retries) # TYPE teams_bot_canned_reply_failed_total counter teams_bot_canned_reply_failed_total 0.0 # HELP teams_bot_canned_reply_failed_created Canned sign-up/disabled/failure replies that never reached the user (connector send exhausted its retries) # TYPE teams_bot_canned_reply_failed_created gauge teams_bot_canned_reply_failed_created 1.7869676562140138e+09 # HELP teams_bot_attachments_skipped_total Inbound attachments dropped from the turn after download (download failed or type unresolvable) # TYPE teams_bot_attachments_skipped_total counter teams_bot_attachments_skipped_total 0.0 # HELP teams_bot_attachments_skipped_created Inbound attachments dropped from the turn after download (download failed or type unresolvable) # TYPE teams_bot_attachments_skipped_created gauge teams_bot_attachments_skipped_created 1.7869676562140307e+09 # HELP teams_bot_activities_by_status Teams bot activity records in the DB grouped by processing status # TYPE teams_bot_activities_by_status gauge teams_bot_activities_by_status{status="pending"} 0.0 teams_bot_activities_by_status{status="processing"} 0.0 teams_bot_activities_by_status{status="failed"} 0.0 teams_bot_activities_by_status{status="merged"} 0.0 # HELP grid_cell_executions_total Total grid cell executions by field type and terminal status # TYPE grid_cell_executions_total counter # HELP grid_cell_errors_total Grid cell failures classified by error category # TYPE grid_cell_errors_total counter # HELP grid_cell_execution_duration_seconds Time spent executing a grid cell (dequeue → terminal status) # TYPE grid_cell_execution_duration_seconds histogram # HELP grid_cell_queue_wait_seconds Time between Service Bus enqueue and worker pickup for a grid cell # TYPE grid_cell_queue_wait_seconds histogram grid_cell_queue_wait_seconds_bucket{le="0.1"} 0.0 grid_cell_queue_wait_seconds_bucket{le="0.5"} 0.0 grid_cell_queue_wait_seconds_bucket{le="1.0"} 0.0 grid_cell_queue_wait_seconds_bucket{le="5.0"} 0.0 grid_cell_queue_wait_seconds_bucket{le="15.0"} 0.0 grid_cell_queue_wait_seconds_bucket{le="30.0"} 0.0 grid_cell_queue_wait_seconds_bucket{le="60.0"} 0.0 grid_cell_queue_wait_seconds_bucket{le="300.0"} 0.0 grid_cell_queue_wait_seconds_bucket{le="+Inf"} 0.0 grid_cell_queue_wait_seconds_count 0.0 grid_cell_queue_wait_seconds_sum 0.0 # HELP grid_cell_queue_wait_seconds_created Time between Service Bus enqueue and worker pickup for a grid cell # TYPE grid_cell_queue_wait_seconds_created gauge grid_cell_queue_wait_seconds_created 1.7869676458667555e+09 # HELP grid_cell_retries_total Grid cell retries by reason # TYPE grid_cell_retries_total counter # HELP grid_cell_route_total Grid cell executions by execution path (temporal post-cutover) # TYPE grid_cell_route_total counter # HELP grid_cell_sb_messages_received_total Service Bus messages picked up from the grid cell queues (counts redeliveries) # TYPE grid_cell_sb_messages_received_total counter # HELP grid_cell_sb_dead_lettered_total Grid cell SB messages explicitly dead-lettered by the worker # TYPE grid_cell_sb_dead_lettered_total counter # HELP grid_monitor_publish_total Messages published to the grid monitor SB topic by subject and per-cell outcome. Overlaps grid_cell_writes_resb_publish_total on reclaim republish paths # TYPE grid_monitor_publish_total counter # HELP grid_cell_writes_legacy_drained_total Entries drained from the legacy grid_cell_answers LPOP list and written # TYPE grid_cell_writes_legacy_drained_total counter grid_cell_writes_legacy_drained_total 0.0 # HELP grid_cell_writes_legacy_drained_created Entries drained from the legacy grid_cell_answers LPOP list and written # TYPE grid_cell_writes_legacy_drained_created gauge grid_cell_writes_legacy_drained_created 1.7869676458668299e+09 # HELP grid_cell_writes_acked_total Stream entries XACKed after DB commit + Service Bus publish succeeded # TYPE grid_cell_writes_acked_total counter grid_cell_writes_acked_total 0.0 # HELP grid_cell_writes_acked_created Stream entries XACKed after DB commit + Service Bus publish succeeded # TYPE grid_cell_writes_acked_created gauge grid_cell_writes_acked_created 1.7869676458668375e+09 # HELP grid_cell_writes_reclaimed_total Stream entries reclaimed via XAUTOCLAIM (consumer crash recovery) # TYPE grid_cell_writes_reclaimed_total counter grid_cell_writes_reclaimed_total 0.0 # HELP grid_cell_writes_reclaimed_created Stream entries reclaimed via XAUTOCLAIM (consumer crash recovery) # TYPE grid_cell_writes_reclaimed_created gauge grid_cell_writes_reclaimed_created 1.7869676458668463e+09 # HELP grid_cell_writes_poisoned_total Stream entries dropped or routed to FAILED. Reasons: malformed_payload | exceeded_redeliveries | already_done | already_failed | already_done_exhausted | already_failed_exhausted # TYPE grid_cell_writes_poisoned_total counter # HELP grid_cell_writes_resb_publish_total Cell-complete Service Bus republish attempts on reclaim path # TYPE grid_cell_writes_resb_publish_total counter # HELP grid_cell_writes_producer_dropped_total Cell answers the producer failed to enqueue via BOTH the stream XADD and the legacy list RPUSH fallback — the cell will be stuck IN_PROGRESS with no monitor signal. A non-zero rate needs investigation (Redis outage). # TYPE grid_cell_writes_producer_dropped_total counter grid_cell_writes_producer_dropped_total 0.0 # HELP grid_cell_writes_producer_dropped_created Cell answers the producer failed to enqueue via BOTH the stream XADD and the legacy list RPUSH fallback — the cell will be stuck IN_PROGRESS with no monitor signal. A non-zero rate needs investigation (Redis outage). # TYPE grid_cell_writes_producer_dropped_created gauge grid_cell_writes_producer_dropped_created 1.7869676458668697e+09 # HELP grid_cell_writes_scrape_up 1 when the cell-writes stream-metrics scrape succeeded, 0 when it failed. Use this alongside the per-partition gauges so absent()/rate() alerts can tell a metrics outage from a genuine zero. # TYPE grid_cell_writes_scrape_up gauge grid_cell_writes_scrape_up 0.0 # HELP grid_cell_writes_scrape_errors_total Total cell-writes stream-metrics scrape failures (Redis open / service-level exception). Increments only on outer failure — per-partition fetch errors stay as zeros in the gauges. # TYPE grid_cell_writes_scrape_errors_total counter grid_cell_writes_scrape_errors_total 0.0 # HELP grid_cell_writes_scrape_errors_created Total cell-writes stream-metrics scrape failures (Redis open / service-level exception). Increments only on outer failure — per-partition fetch errors stay as zeros in the gauges. # TYPE grid_cell_writes_scrape_errors_created gauge grid_cell_writes_scrape_errors_created 1.7869676458668938e+09 # HELP grid_run_workflow_total Total grid run workflows by terminal status # TYPE grid_run_workflow_total counter # HELP grid_run_workflow_duration_seconds Time to execute a complete grid run workflow (orchestration activity) # TYPE grid_run_workflow_duration_seconds histogram # HELP grid_run_in_flight Number of grid runs currently executing # TYPE grid_run_in_flight gauge grid_run_in_flight 0.0 # HELP prepare_batch_duration_seconds Time to prepare and batch grid cells for submission # TYPE prepare_batch_duration_seconds histogram prepare_batch_duration_seconds_bucket{le="0.1"} 0.0 prepare_batch_duration_seconds_bucket{le="0.5"} 0.0 prepare_batch_duration_seconds_bucket{le="1.0"} 0.0 prepare_batch_duration_seconds_bucket{le="5.0"} 0.0 prepare_batch_duration_seconds_bucket{le="15.0"} 0.0 prepare_batch_duration_seconds_bucket{le="30.0"} 0.0 prepare_batch_duration_seconds_bucket{le="60.0"} 0.0 prepare_batch_duration_seconds_bucket{le="300.0"} 0.0 prepare_batch_duration_seconds_bucket{le="+Inf"} 0.0 prepare_batch_duration_seconds_count 0.0 prepare_batch_duration_seconds_sum 0.0 # HELP prepare_batch_duration_seconds_created Time to prepare and batch grid cells for submission # TYPE prepare_batch_duration_seconds_created gauge prepare_batch_duration_seconds_created 1.7869676458669312e+09 # HELP grid_run_stranded_cells_total Total grid cells left unresolved (timeout or dependency failure) # TYPE grid_run_stranded_cells_total counter grid_run_stranded_cells_total 0.0 # HELP grid_run_stranded_cells_created Total grid cells left unresolved (timeout or dependency failure) # TYPE grid_run_stranded_cells_created gauge grid_run_stranded_cells_created 1.7869676458669543e+09 # HELP grids_redis_ctx_ops_total Grid-run Redis context-cache operations by op and result # TYPE grids_redis_ctx_ops_total counter # HELP grids_redis_ctx_op_duration_seconds Duration of grid-run Redis context-cache operations # TYPE grids_redis_ctx_op_duration_seconds histogram # HELP grids_redis_ctx_fallback_total Grid-run context-cache fallbacks to the Postgres path, by reason # TYPE grids_redis_ctx_fallback_total counter # HELP grid_cell_shape_executions_total Grid cell executions by column shape (datatype, tool profile, rendered prompt length, @ref count and mode, attached documents) # TYPE grid_cell_shape_executions_total counter # HELP grid_cell_shape_tokens_total Tokens consumed by grid cell executions, by priced token kind, model, and column shape — dollars are token counts times the pricing table applied at display time # TYPE grid_cell_shape_tokens_total counter # HELP grid_cell_rule_match_total Grid cell executions matching each shadow routing hypothesis # TYPE grid_cell_rule_match_total counter # HELP grid_cell_routed_total Grid cell executions re-routed to a promoted rule's target model # TYPE grid_cell_routed_total counter # HELP auth_jwt_verified_total JWT verifications that passed, by routing path # TYPE auth_jwt_verified_total counter auth_jwt_verified_total{key="n_a",path="user"} 0.0 auth_jwt_verified_total{key="primary",path="service"} 0.0 auth_jwt_verified_total{key="fallback",path="service"} 0.0 auth_jwt_verified_total{key="n_a",path="desktop"} 0.0 # HELP auth_jwt_verified_created JWT verifications that passed, by routing path # TYPE auth_jwt_verified_created gauge auth_jwt_verified_created{key="n_a",path="user"} 1.7869676368317196e+09 auth_jwt_verified_created{key="primary",path="service"} 1.7869676368317266e+09 auth_jwt_verified_created{key="fallback",path="service"} 1.7869676368317327e+09 auth_jwt_verified_created{key="n_a",path="desktop"} 1.7869676368317387e+09 # HELP auth_jwt_verification_failed_total JWT verifications that failed, by reason # TYPE auth_jwt_verification_failed_total counter auth_jwt_verification_failed_total{reason="bad_signature"} 0.0 auth_jwt_verification_failed_total{reason="expired"} 0.0 auth_jwt_verification_failed_total{reason="bad_audience"} 0.0 auth_jwt_verification_failed_total{reason="bad_issuer"} 0.0 auth_jwt_verification_failed_total{reason="immature"} 0.0 auth_jwt_verification_failed_total{reason="missing_claim"} 0.0 auth_jwt_verification_failed_total{reason="malformed"} 0.0 auth_jwt_verification_failed_total{reason="invalid"} 0.0 auth_jwt_verification_failed_total{reason="type_mismatch"} 0.0 auth_jwt_verification_failed_total{reason="no_path"} 0.0 auth_jwt_verification_failed_total{reason="jwks_unavailable"} 0.0 auth_jwt_verification_failed_total{reason="unexpected"} 0.0 # HELP auth_jwt_verification_failed_created JWT verifications that failed, by reason # TYPE auth_jwt_verification_failed_created gauge auth_jwt_verification_failed_created{reason="bad_signature"} 1.786967636831745e+09 auth_jwt_verification_failed_created{reason="expired"} 1.7869676368317509e+09 auth_jwt_verification_failed_created{reason="bad_audience"} 1.7869676368317566e+09 auth_jwt_verification_failed_created{reason="bad_issuer"} 1.7869676368317614e+09 auth_jwt_verification_failed_created{reason="immature"} 1.7869676368317711e+09 auth_jwt_verification_failed_created{reason="missing_claim"} 1.7869676368317761e+09 auth_jwt_verification_failed_created{reason="malformed"} 1.7869676368317816e+09 auth_jwt_verification_failed_created{reason="invalid"} 1.786967636831787e+09 auth_jwt_verification_failed_created{reason="type_mismatch"} 1.7869676368317919e+09 auth_jwt_verification_failed_created{reason="no_path"} 1.7869676368317978e+09 auth_jwt_verification_failed_created{reason="jwks_unavailable"} 1.7869676368318105e+09 auth_jwt_verification_failed_created{reason="unexpected"} 1.786967636831821e+09 # HELP auth_refresh_total Refresh attempts across both refresh routes (web /auth/refresh + office addin /office-addin/auth/refresh), by outcome and frontend source. # TYPE auth_refresh_total counter auth_refresh_total{result="success",source="middleware"} 0.0 auth_refresh_total{result="success",source="route_handler"} 0.0 auth_refresh_total{result="success",source="office_addin"} 0.0 auth_refresh_total{result="success",source="desktop_waiting"} 0.0 auth_refresh_total{result="success",source="unknown"} 0.0 auth_refresh_total{result="workos_error",source="middleware"} 0.0 auth_refresh_total{result="workos_error",source="route_handler"} 0.0 auth_refresh_total{result="workos_error",source="office_addin"} 0.0 auth_refresh_total{result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_total{result="workos_error",source="unknown"} 0.0 auth_refresh_total{result="session_ended",source="middleware"} 0.0 auth_refresh_total{result="session_ended",source="route_handler"} 0.0 auth_refresh_total{result="session_ended",source="office_addin"} 0.0 auth_refresh_total{result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_total{result="session_ended",source="unknown"} 0.0 auth_refresh_total{result="user_not_found",source="middleware"} 0.0 auth_refresh_total{result="user_not_found",source="route_handler"} 0.0 auth_refresh_total{result="user_not_found",source="office_addin"} 0.0 auth_refresh_total{result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_total{result="user_not_found",source="unknown"} 0.0 auth_refresh_total{result="coalesced_replay",source="middleware"} 0.0 auth_refresh_total{result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_total{result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_total{result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_total{result="coalesced_replay",source="unknown"} 0.0 auth_refresh_total{result="workos_transient",source="middleware"} 0.0 auth_refresh_total{result="workos_transient",source="route_handler"} 0.0 auth_refresh_total{result="workos_transient",source="office_addin"} 0.0 auth_refresh_total{result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_total{result="workos_transient",source="unknown"} 0.0 auth_refresh_total{result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_total{result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_total{result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_total{result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_total{result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_total{result="no_cookie",source="office_addin"} 0.0 # HELP auth_refresh_created Refresh attempts across both refresh routes (web /auth/refresh + office addin /office-addin/auth/refresh), by outcome and frontend source. # TYPE auth_refresh_created gauge auth_refresh_created{result="success",source="middleware"} 1.7869676368297195e+09 auth_refresh_created{result="success",source="route_handler"} 1.786967636829727e+09 auth_refresh_created{result="success",source="office_addin"} 1.786967636829734e+09 auth_refresh_created{result="success",source="desktop_waiting"} 1.7869676368297405e+09 auth_refresh_created{result="success",source="unknown"} 1.786967636829748e+09 auth_refresh_created{result="workos_error",source="middleware"} 1.7869676368297548e+09 auth_refresh_created{result="workos_error",source="route_handler"} 1.7869676368297615e+09 auth_refresh_created{result="workos_error",source="office_addin"} 1.7869676368297677e+09 auth_refresh_created{result="workos_error",source="desktop_waiting"} 1.7869676368297741e+09 auth_refresh_created{result="workos_error",source="unknown"} 1.7869676368297803e+09 auth_refresh_created{result="session_ended",source="middleware"} 1.78696763682988e+09 auth_refresh_created{result="session_ended",source="route_handler"} 1.7869676368298895e+09 auth_refresh_created{result="session_ended",source="office_addin"} 1.7869676368298974e+09 auth_refresh_created{result="session_ended",source="desktop_waiting"} 1.7869676368299043e+09 auth_refresh_created{result="session_ended",source="unknown"} 1.7869676368299112e+09 auth_refresh_created{result="user_not_found",source="middleware"} 1.7869676368299174e+09 auth_refresh_created{result="user_not_found",source="route_handler"} 1.7869676368299236e+09 auth_refresh_created{result="user_not_found",source="office_addin"} 1.7869676368299298e+09 auth_refresh_created{result="user_not_found",source="desktop_waiting"} 1.7869676368299358e+09 auth_refresh_created{result="user_not_found",source="unknown"} 1.7869676368299422e+09 auth_refresh_created{result="coalesced_replay",source="middleware"} 1.7869676368299484e+09 auth_refresh_created{result="coalesced_replay",source="route_handler"} 1.7869676368299546e+09 auth_refresh_created{result="coalesced_replay",source="office_addin"} 1.7869676368299615e+09 auth_refresh_created{result="coalesced_replay",source="desktop_waiting"} 1.786967636829968e+09 auth_refresh_created{result="coalesced_replay",source="unknown"} 1.786967636829974e+09 auth_refresh_created{result="workos_transient",source="middleware"} 1.78696763682998e+09 auth_refresh_created{result="workos_transient",source="route_handler"} 1.7869676368299859e+09 auth_refresh_created{result="workos_transient",source="office_addin"} 1.7869676368299916e+09 auth_refresh_created{result="workos_transient",source="desktop_waiting"} 1.7869676368299978e+09 auth_refresh_created{result="workos_transient",source="unknown"} 1.7869676368300037e+09 auth_refresh_created{result="coalesce_timeout",source="middleware"} 1.7869676368300097e+09 auth_refresh_created{result="coalesce_timeout",source="route_handler"} 1.7869676368300154e+09 auth_refresh_created{result="coalesce_timeout",source="office_addin"} 1.7869676368300214e+09 auth_refresh_created{result="coalesce_timeout",source="desktop_waiting"} 1.7869676368300273e+09 auth_refresh_created{result="coalesce_timeout",source="unknown"} 1.7869676368300328e+09 auth_refresh_created{result="no_cookie",source="office_addin"} 1.786967636830039e+09 # HELP auth_refresh_latency_seconds End-to-end latency of POST /auth/refresh (WorkOS call + DB lookup + re-enrol). Labelled by `result` so `histogram_quantile(...)` can isolate successful-refresh tail latency from WorkOS slow-fails — the `workos_error` series is a real network round-trip that can itself be slow (mirrors the shape used in OpenTelemetry semconv for outgoing client request duration). Also labelled by `source` (mirrors AUTH_REFRESH_TOTAL) so the SSR-time middleware path can be measured separately from the client-side route-handler path — they have meaningfully different tail latency profiles (middleware blocks SSR, route_handler runs after first paint). # TYPE auth_refresh_latency_seconds histogram auth_refresh_latency_seconds_bucket{le="0.05",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="success",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="success",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="success",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="success",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="success",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="workos_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="workos_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="workos_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="workos_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="workos_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="session_ended",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="session_ended",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="session_ended",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="session_ended",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="session_ended",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="user_not_found",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="user_not_found",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="user_not_found",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="user_not_found",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="user_not_found",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="internal_error",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="internal_error",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="internal_error",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="internal_error",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="internal_error",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="coalesced_replay",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="coalesced_replay",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="coalesced_replay",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="coalesced_replay",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="coalesced_replay",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="workos_transient",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="workos_transient",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="workos_transient",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="workos_transient",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="workos_transient",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_count{result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_sum{result="coalesce_timeout",source="middleware"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_count{result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_sum{result="coalesce_timeout",source="route_handler"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_count{result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_sum{result="coalesce_timeout",source="office_addin"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_count{result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_sum{result="coalesce_timeout",source="desktop_waiting"} 0.0 auth_refresh_latency_seconds_bucket{le="0.05",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.1",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.25",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="0.5",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="1.0",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="2.5",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="5.0",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="10.0",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_bucket{le="+Inf",result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_count{result="coalesce_timeout",source="unknown"} 0.0 auth_refresh_latency_seconds_sum{result="coalesce_timeout",source="unknown"} 0.0 # HELP auth_refresh_latency_seconds_created End-to-end latency of POST /auth/refresh (WorkOS call + DB lookup + re-enrol). Labelled by `result` so `histogram_quantile(...)` can isolate successful-refresh tail latency from WorkOS slow-fails — the `workos_error` series is a real network round-trip that can itself be slow (mirrors the shape used in OpenTelemetry semconv for outgoing client request duration). Also labelled by `source` (mirrors AUTH_REFRESH_TOTAL) so the SSR-time middleware path can be measured separately from the client-side route-handler path — they have meaningfully different tail latency profiles (middleware blocks SSR, route_handler runs after first paint). # TYPE auth_refresh_latency_seconds_created gauge auth_refresh_latency_seconds_created{result="success",source="middleware"} 1.7869676368300533e+09 auth_refresh_latency_seconds_created{result="success",source="route_handler"} 1.7869676368300812e+09 auth_refresh_latency_seconds_created{result="success",source="office_addin"} 1.7869676368301046e+09 auth_refresh_latency_seconds_created{result="success",source="desktop_waiting"} 1.7869676368301437e+09 auth_refresh_latency_seconds_created{result="success",source="unknown"} 1.786967636830191e+09 auth_refresh_latency_seconds_created{result="workos_error",source="middleware"} 1.786967636830215e+09 auth_refresh_latency_seconds_created{result="workos_error",source="route_handler"} 1.7869676368302367e+09 auth_refresh_latency_seconds_created{result="workos_error",source="office_addin"} 1.7869676368302777e+09 auth_refresh_latency_seconds_created{result="workos_error",source="desktop_waiting"} 1.7869676368303058e+09 auth_refresh_latency_seconds_created{result="workos_error",source="unknown"} 1.7869676368303268e+09 auth_refresh_latency_seconds_created{result="session_ended",source="middleware"} 1.786967636830347e+09 auth_refresh_latency_seconds_created{result="session_ended",source="route_handler"} 1.7869676368303757e+09 auth_refresh_latency_seconds_created{result="session_ended",source="office_addin"} 1.7869676368304067e+09 auth_refresh_latency_seconds_created{result="session_ended",source="desktop_waiting"} 1.7869676368304276e+09 auth_refresh_latency_seconds_created{result="session_ended",source="unknown"} 1.7869676368304746e+09 auth_refresh_latency_seconds_created{result="user_not_found",source="middleware"} 1.7869676368305075e+09 auth_refresh_latency_seconds_created{result="user_not_found",source="route_handler"} 1.7869676368305287e+09 auth_refresh_latency_seconds_created{result="user_not_found",source="office_addin"} 1.7869676368305516e+09 auth_refresh_latency_seconds_created{result="user_not_found",source="desktop_waiting"} 1.7869676368305736e+09 auth_refresh_latency_seconds_created{result="user_not_found",source="unknown"} 1.7869676368305945e+09 auth_refresh_latency_seconds_created{result="internal_error",source="middleware"} 1.7869676368306146e+09 auth_refresh_latency_seconds_created{result="internal_error",source="route_handler"} 1.7869676368306334e+09 auth_refresh_latency_seconds_created{result="internal_error",source="office_addin"} 1.7869676368306534e+09 auth_refresh_latency_seconds_created{result="internal_error",source="desktop_waiting"} 1.7869676368306735e+09 auth_refresh_latency_seconds_created{result="internal_error",source="unknown"} 1.7869676368307652e+09 auth_refresh_latency_seconds_created{result="coalesced_replay",source="middleware"} 1.7869676368307881e+09 auth_refresh_latency_seconds_created{result="coalesced_replay",source="route_handler"} 1.7869676368308103e+09 auth_refresh_latency_seconds_created{result="coalesced_replay",source="office_addin"} 1.786967636830854e+09 auth_refresh_latency_seconds_created{result="coalesced_replay",source="desktop_waiting"} 1.7869676368308752e+09 auth_refresh_latency_seconds_created{result="coalesced_replay",source="unknown"} 1.7869676368308954e+09 auth_refresh_latency_seconds_created{result="workos_transient",source="middleware"} 1.7869676368309143e+09 auth_refresh_latency_seconds_created{result="workos_transient",source="route_handler"} 1.7869676368309348e+09 auth_refresh_latency_seconds_created{result="workos_transient",source="office_addin"} 1.7869676368309546e+09 auth_refresh_latency_seconds_created{result="workos_transient",source="desktop_waiting"} 1.7869676368309739e+09 auth_refresh_latency_seconds_created{result="workos_transient",source="unknown"} 1.786967636830993e+09 auth_refresh_latency_seconds_created{result="coalesce_timeout",source="middleware"} 1.786967636831012e+09 auth_refresh_latency_seconds_created{result="coalesce_timeout",source="route_handler"} 1.786967636831032e+09 auth_refresh_latency_seconds_created{result="coalesce_timeout",source="office_addin"} 1.7869676368310506e+09 auth_refresh_latency_seconds_created{result="coalesce_timeout",source="desktop_waiting"} 1.7869676368310702e+09 auth_refresh_latency_seconds_created{result="coalesce_timeout",source="unknown"} 1.7869676368310902e+09 # HELP auth_refresh_workos_rejection_total Confirmed WorkOS 4xx rejections of POST /auth/refresh, split by reason — the breakdown the opaque `auth_refresh_total{result="workos_error"}` bucket could not give. `reuse_detected` is the literal fingerprint of the 2026-06-11 cross-tenant forced-logout class (refresh-token reuse-detection). The exact WorkOS error/code literals for reuse vs revoke are unconfirmed in prod, so unknowns bucket to `other_4xx` and the raw workos error/code/description ride the Loki reject log (MML-14115) so the allow-list can be refined from real data. Only confirmed 4xx rejects increment here — transient 5xx/429 (kept-session 503s) and `user_not_found` do not. # TYPE auth_refresh_workos_rejection_total counter auth_refresh_workos_rejection_total{reason="token_expired",source="middleware"} 0.0 auth_refresh_workos_rejection_total{reason="session_ended",source="middleware"} 0.0 auth_refresh_workos_rejection_total{reason="reuse_detected",source="middleware"} 0.0 auth_refresh_workos_rejection_total{reason="invalid_grant",source="middleware"} 0.0 auth_refresh_workos_rejection_total{reason="session_revoked",source="middleware"} 0.0 auth_refresh_workos_rejection_total{reason="other_4xx",source="middleware"} 0.0 auth_refresh_workos_rejection_total{reason="token_expired",source="route_handler"} 0.0 auth_refresh_workos_rejection_total{reason="session_ended",source="route_handler"} 0.0 auth_refresh_workos_rejection_total{reason="reuse_detected",source="route_handler"} 0.0 auth_refresh_workos_rejection_total{reason="invalid_grant",source="route_handler"} 0.0 auth_refresh_workos_rejection_total{reason="session_revoked",source="route_handler"} 0.0 auth_refresh_workos_rejection_total{reason="other_4xx",source="route_handler"} 0.0 auth_refresh_workos_rejection_total{reason="token_expired",source="office_addin"} 0.0 auth_refresh_workos_rejection_total{reason="session_ended",source="office_addin"} 0.0 auth_refresh_workos_rejection_total{reason="reuse_detected",source="office_addin"} 0.0 auth_refresh_workos_rejection_total{reason="invalid_grant",source="office_addin"} 0.0 auth_refresh_workos_rejection_total{reason="session_revoked",source="office_addin"} 0.0 auth_refresh_workos_rejection_total{reason="other_4xx",source="office_addin"} 0.0 auth_refresh_workos_rejection_total{reason="token_expired",source="desktop_waiting"} 0.0 auth_refresh_workos_rejection_total{reason="session_ended",source="desktop_waiting"} 0.0 auth_refresh_workos_rejection_total{reason="reuse_detected",source="desktop_waiting"} 0.0 auth_refresh_workos_rejection_total{reason="invalid_grant",source="desktop_waiting"} 0.0 auth_refresh_workos_rejection_total{reason="session_revoked",source="desktop_waiting"} 0.0 auth_refresh_workos_rejection_total{reason="other_4xx",source="desktop_waiting"} 0.0 auth_refresh_workos_rejection_total{reason="token_expired",source="unknown"} 0.0 auth_refresh_workos_rejection_total{reason="session_ended",source="unknown"} 0.0 auth_refresh_workos_rejection_total{reason="reuse_detected",source="unknown"} 0.0 auth_refresh_workos_rejection_total{reason="invalid_grant",source="unknown"} 0.0 auth_refresh_workos_rejection_total{reason="session_revoked",source="unknown"} 0.0 auth_refresh_workos_rejection_total{reason="other_4xx",source="unknown"} 0.0 # HELP auth_refresh_workos_rejection_created Confirmed WorkOS 4xx rejections of POST /auth/refresh, split by reason — the breakdown the opaque `auth_refresh_total{result="workos_error"}` bucket could not give. `reuse_detected` is the literal fingerprint of the 2026-06-11 cross-tenant forced-logout class (refresh-token reuse-detection). The exact WorkOS error/code literals for reuse vs revoke are unconfirmed in prod, so unknowns bucket to `other_4xx` and the raw workos error/code/description ride the Loki reject log (MML-14115) so the allow-list can be refined from real data. Only confirmed 4xx rejects increment here — transient 5xx/429 (kept-session 503s) and `user_not_found` do not. # TYPE auth_refresh_workos_rejection_created gauge auth_refresh_workos_rejection_created{reason="token_expired",source="middleware"} 1.7869676368311117e+09 auth_refresh_workos_rejection_created{reason="session_ended",source="middleware"} 1.786967636831119e+09 auth_refresh_workos_rejection_created{reason="reuse_detected",source="middleware"} 1.7869676368311255e+09 auth_refresh_workos_rejection_created{reason="invalid_grant",source="middleware"} 1.7869676368311315e+09 auth_refresh_workos_rejection_created{reason="session_revoked",source="middleware"} 1.7869676368311372e+09 auth_refresh_workos_rejection_created{reason="other_4xx",source="middleware"} 1.786967636831143e+09 auth_refresh_workos_rejection_created{reason="token_expired",source="route_handler"} 1.7869676368311493e+09 auth_refresh_workos_rejection_created{reason="session_ended",source="route_handler"} 1.7869676368311555e+09 auth_refresh_workos_rejection_created{reason="reuse_detected",source="route_handler"} 1.7869676368311615e+09 auth_refresh_workos_rejection_created{reason="invalid_grant",source="route_handler"} 1.7869676368311672e+09 auth_refresh_workos_rejection_created{reason="session_revoked",source="route_handler"} 1.7869676368311732e+09 auth_refresh_workos_rejection_created{reason="other_4xx",source="route_handler"} 1.786967636831179e+09 auth_refresh_workos_rejection_created{reason="token_expired",source="office_addin"} 1.786967636831185e+09 auth_refresh_workos_rejection_created{reason="session_ended",source="office_addin"} 1.7869676368311908e+09 auth_refresh_workos_rejection_created{reason="reuse_detected",source="office_addin"} 1.786967636831197e+09 auth_refresh_workos_rejection_created{reason="invalid_grant",source="office_addin"} 1.7869676368312027e+09 auth_refresh_workos_rejection_created{reason="session_revoked",source="office_addin"} 1.7869676368312087e+09 auth_refresh_workos_rejection_created{reason="other_4xx",source="office_addin"} 1.7869676368312144e+09 auth_refresh_workos_rejection_created{reason="token_expired",source="desktop_waiting"} 1.78696763683122e+09 auth_refresh_workos_rejection_created{reason="session_ended",source="desktop_waiting"} 1.7869676368312259e+09 auth_refresh_workos_rejection_created{reason="reuse_detected",source="desktop_waiting"} 1.7869676368312318e+09 auth_refresh_workos_rejection_created{reason="invalid_grant",source="desktop_waiting"} 1.786967636831237e+09 auth_refresh_workos_rejection_created{reason="session_revoked",source="desktop_waiting"} 1.7869676368312435e+09 auth_refresh_workos_rejection_created{reason="other_4xx",source="desktop_waiting"} 1.7869676368312492e+09 auth_refresh_workos_rejection_created{reason="token_expired",source="unknown"} 1.786967636831255e+09 auth_refresh_workos_rejection_created{reason="session_ended",source="unknown"} 1.7869676368312607e+09 auth_refresh_workos_rejection_created{reason="reuse_detected",source="unknown"} 1.7869676368312664e+09 auth_refresh_workos_rejection_created{reason="invalid_grant",source="unknown"} 1.786967636831272e+09 auth_refresh_workos_rejection_created{reason="session_revoked",source="unknown"} 1.7869676368312778e+09 auth_refresh_workos_rejection_created{reason="other_4xx",source="unknown"} 1.7869676368312836e+09 # HELP auth_refresh_coalesce_total Single-flight + grace-cache coalescer outcomes for POST /auth/refresh (src/auth/refresh_coalesce.py), by the role this caller played. The #8016 coalescer shipped with zero instrumentation; a rising `redis_fail_open` means single-flight has silently degraded to pre-#8016 direct-exchange behaviour and the rotation-race / reuse-detection bug can recur. This is a mechanics diagnostic, NOT a partition of every refresh: `leader_exchanged` counts the caller that won the lock and ran the WorkOS exchange; a leader whose exchange WorkOS rejected or 5xx'd is accounted by auth_refresh_total / auth_refresh_workos_rejection_total, not here. `cache_write_failed` is an additive sub-event that can co-occur with a leader outcome. # TYPE auth_refresh_coalesce_total counter auth_refresh_coalesce_total{role_outcome="redis_fail_open",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="grace_hit",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="negative_sentinel_replay",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="leader_exchanged",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_replayed",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_timeout",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="cache_write_failed",source="middleware"} 0.0 auth_refresh_coalesce_total{role_outcome="redis_fail_open",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="grace_hit",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="negative_sentinel_replay",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="leader_exchanged",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_replayed",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_timeout",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="cache_write_failed",source="route_handler"} 0.0 auth_refresh_coalesce_total{role_outcome="redis_fail_open",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="grace_hit",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="negative_sentinel_replay",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="leader_exchanged",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_replayed",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_timeout",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="cache_write_failed",source="office_addin"} 0.0 auth_refresh_coalesce_total{role_outcome="redis_fail_open",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="grace_hit",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="negative_sentinel_replay",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="leader_exchanged",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_replayed",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_timeout",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="cache_write_failed",source="desktop_waiting"} 0.0 auth_refresh_coalesce_total{role_outcome="redis_fail_open",source="unknown"} 0.0 auth_refresh_coalesce_total{role_outcome="grace_hit",source="unknown"} 0.0 auth_refresh_coalesce_total{role_outcome="negative_sentinel_replay",source="unknown"} 0.0 auth_refresh_coalesce_total{role_outcome="leader_exchanged",source="unknown"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_replayed",source="unknown"} 0.0 auth_refresh_coalesce_total{role_outcome="follower_timeout",source="unknown"} 0.0 auth_refresh_coalesce_total{role_outcome="cache_write_failed",source="unknown"} 0.0 # HELP auth_refresh_coalesce_created Single-flight + grace-cache coalescer outcomes for POST /auth/refresh (src/auth/refresh_coalesce.py), by the role this caller played. The #8016 coalescer shipped with zero instrumentation; a rising `redis_fail_open` means single-flight has silently degraded to pre-#8016 direct-exchange behaviour and the rotation-race / reuse-detection bug can recur. This is a mechanics diagnostic, NOT a partition of every refresh: `leader_exchanged` counts the caller that won the lock and ran the WorkOS exchange; a leader whose exchange WorkOS rejected or 5xx'd is accounted by auth_refresh_total / auth_refresh_workos_rejection_total, not here. `cache_write_failed` is an additive sub-event that can co-occur with a leader outcome. # TYPE auth_refresh_coalesce_created gauge auth_refresh_coalesce_created{role_outcome="redis_fail_open",source="middleware"} 1.7869676368312912e+09 auth_refresh_coalesce_created{role_outcome="grace_hit",source="middleware"} 1.7869676368312979e+09 auth_refresh_coalesce_created{role_outcome="negative_sentinel_replay",source="middleware"} 1.7869676368313038e+09 auth_refresh_coalesce_created{role_outcome="leader_exchanged",source="middleware"} 1.7869676368313098e+09 auth_refresh_coalesce_created{role_outcome="follower_replayed",source="middleware"} 1.7869676368313158e+09 auth_refresh_coalesce_created{role_outcome="follower_timeout",source="middleware"} 1.7869676368313212e+09 auth_refresh_coalesce_created{role_outcome="cache_write_failed",source="middleware"} 1.7869676368313274e+09 auth_refresh_coalesce_created{role_outcome="redis_fail_open",source="route_handler"} 1.786967636831334e+09 auth_refresh_coalesce_created{role_outcome="grace_hit",source="route_handler"} 1.7869676368313406e+09 auth_refresh_coalesce_created{role_outcome="negative_sentinel_replay",source="route_handler"} 1.7869676368313465e+09 auth_refresh_coalesce_created{role_outcome="leader_exchanged",source="route_handler"} 1.7869676368313522e+09 auth_refresh_coalesce_created{role_outcome="follower_replayed",source="route_handler"} 1.7869676368313582e+09 auth_refresh_coalesce_created{role_outcome="follower_timeout",source="route_handler"} 1.786967636831364e+09 auth_refresh_coalesce_created{role_outcome="cache_write_failed",source="route_handler"} 1.7869676368313692e+09 auth_refresh_coalesce_created{role_outcome="redis_fail_open",source="office_addin"} 1.7869676368313751e+09 auth_refresh_coalesce_created{role_outcome="grace_hit",source="office_addin"} 1.7869676368313804e+09 auth_refresh_coalesce_created{role_outcome="negative_sentinel_replay",source="office_addin"} 1.786967636831386e+09 auth_refresh_coalesce_created{role_outcome="leader_exchanged",source="office_addin"} 1.786967636831392e+09 auth_refresh_coalesce_created{role_outcome="follower_replayed",source="office_addin"} 1.7869676368313978e+09 auth_refresh_coalesce_created{role_outcome="follower_timeout",source="office_addin"} 1.7869676368314035e+09 auth_refresh_coalesce_created{role_outcome="cache_write_failed",source="office_addin"} 1.7869676368314092e+09 auth_refresh_coalesce_created{role_outcome="redis_fail_open",source="desktop_waiting"} 1.7869676368314145e+09 auth_refresh_coalesce_created{role_outcome="grace_hit",source="desktop_waiting"} 1.786967636831421e+09 auth_refresh_coalesce_created{role_outcome="negative_sentinel_replay",source="desktop_waiting"} 1.7869676368314264e+09 auth_refresh_coalesce_created{role_outcome="leader_exchanged",source="desktop_waiting"} 1.7869676368314319e+09 auth_refresh_coalesce_created{role_outcome="follower_replayed",source="desktop_waiting"} 1.7869676368314376e+09 auth_refresh_coalesce_created{role_outcome="follower_timeout",source="desktop_waiting"} 1.7869676368314433e+09 auth_refresh_coalesce_created{role_outcome="cache_write_failed",source="desktop_waiting"} 1.7869676368314803e+09 auth_refresh_coalesce_created{role_outcome="redis_fail_open",source="unknown"} 1.786967636831497e+09 auth_refresh_coalesce_created{role_outcome="grace_hit",source="unknown"} 1.786967636831529e+09 auth_refresh_coalesce_created{role_outcome="negative_sentinel_replay",source="unknown"} 1.786967636831536e+09 auth_refresh_coalesce_created{role_outcome="leader_exchanged",source="unknown"} 1.786967636831542e+09 auth_refresh_coalesce_created{role_outcome="follower_replayed",source="unknown"} 1.7869676368315482e+09 auth_refresh_coalesce_created{role_outcome="follower_timeout",source="unknown"} 1.786967636831554e+09 auth_refresh_coalesce_created{role_outcome="cache_write_failed",source="unknown"} 1.78696763683156e+09 # HELP auth_refresh_org_resync_total Org-membership re-sync outcome on POST /auth/refresh — the 1h-cadence heartbeat that propagates WorkOS-side org/role changes into our DB. `db_fallback` (a WorkOS org-API blip where we keep the DB org rather than null it and bounce the user to onboarding) was log-only before this; `no_org_id` (WorkOS returned no organization_id) was entirely silent. A rising `db_fallback` / `no_org_id` is the leading signal for the spurious-onboarding bounce class (MML-11088). # TYPE auth_refresh_org_resync_total counter auth_refresh_org_resync_total{outcome="workos_synced"} 0.0 auth_refresh_org_resync_total{outcome="db_fallback"} 0.0 auth_refresh_org_resync_total{outcome="no_org_id"} 0.0 # HELP auth_refresh_org_resync_created Org-membership re-sync outcome on POST /auth/refresh — the 1h-cadence heartbeat that propagates WorkOS-side org/role changes into our DB. `db_fallback` (a WorkOS org-API blip where we keep the DB org rather than null it and bounce the user to onboarding) was log-only before this; `no_org_id` (WorkOS returned no organization_id) was entirely silent. A rising `db_fallback` / `no_org_id` is the leading signal for the spurious-onboarding bounce class (MML-11088). # TYPE auth_refresh_org_resync_created gauge auth_refresh_org_resync_created{outcome="workos_synced"} 1.7869676368315666e+09 auth_refresh_org_resync_created{outcome="db_fallback"} 1.7869676368315725e+09 auth_refresh_org_resync_created{outcome="no_org_id"} 1.7869676368315778e+09 # HELP auth_refresh_follower_wait_seconds Time a coalescer follower blocked waiting for the leader to publish a result before replaying it / giving up (refresh_coalesce.py `_await_leader`). Leading indicator for `coalesce_timeout` 503s: the distribution creeping toward the 10s `_FOLLOWER_WAIT_SECONDS` ceiling means leaders are running slow (WorkOS / DB stress) and followers are about to start timing out. Observed on every follower path — replay, sentinel, or timeout. # TYPE auth_refresh_follower_wait_seconds histogram auth_refresh_follower_wait_seconds_bucket{le="0.1",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.25",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.5",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="1.0",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="2.0",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="3.0",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="5.0",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="7.5",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="10.0",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="+Inf",source="middleware"} 0.0 auth_refresh_follower_wait_seconds_count{source="middleware"} 0.0 auth_refresh_follower_wait_seconds_sum{source="middleware"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.1",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.25",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.5",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="1.0",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="2.0",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="3.0",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="5.0",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="7.5",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="10.0",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="+Inf",source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_count{source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_sum{source="route_handler"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.1",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.25",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.5",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="1.0",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="2.0",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="3.0",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="5.0",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="7.5",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="10.0",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="+Inf",source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_count{source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_sum{source="office_addin"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.1",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.25",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.5",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="1.0",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="2.0",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="3.0",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="5.0",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="7.5",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="10.0",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="+Inf",source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_count{source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_sum{source="desktop_waiting"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.1",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.25",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="0.5",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="1.0",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="2.0",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="3.0",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="5.0",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="7.5",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="10.0",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_bucket{le="+Inf",source="unknown"} 0.0 auth_refresh_follower_wait_seconds_count{source="unknown"} 0.0 auth_refresh_follower_wait_seconds_sum{source="unknown"} 0.0 # HELP auth_refresh_follower_wait_seconds_created Time a coalescer follower blocked waiting for the leader to publish a result before replaying it / giving up (refresh_coalesce.py `_await_leader`). Leading indicator for `coalesce_timeout` 503s: the distribution creeping toward the 10s `_FOLLOWER_WAIT_SECONDS` ceiling means leaders are running slow (WorkOS / DB stress) and followers are about to start timing out. Observed on every follower path — replay, sentinel, or timeout. # TYPE auth_refresh_follower_wait_seconds_created gauge auth_refresh_follower_wait_seconds_created{source="middleware"} 1.786967636831611e+09 auth_refresh_follower_wait_seconds_created{source="route_handler"} 1.786967636831636e+09 auth_refresh_follower_wait_seconds_created{source="office_addin"} 1.7869676368316572e+09 auth_refresh_follower_wait_seconds_created{source="desktop_waiting"} 1.7869676368316774e+09 auth_refresh_follower_wait_seconds_created{source="unknown"} 1.7869676368316984e+09 # HELP auth_refresh_inflight POST /auth/refresh handlers currently executing on THIS process. `.inc()` at handler entry, `.dec()` in the finally. Per-pod — sum across replicas for a fleet view. Two uses: (1) a live concurrency signal for the refresh stampede a post-deploy RT-reuse storm produces; (2) SIGTERM accounting — the lifespan shutdown logs this value so refreshes killed mid-flight by a rolling deploy are counted instead of vanishing (completion-only logging hid exactly this in the 2026-06-11 incident). # TYPE auth_refresh_inflight gauge auth_refresh_inflight 0.0 # HELP auth_login_latency_seconds End-to-end latency of GET /auth (WorkOS code exchange + org-membership lookup + DB user/org/collection writes). Buckets reach 30 s because first-touch logins do significantly more work than refreshes (user create, default collection seed, org enrol). **Success-only observation:** only the `result='success'` branch observes into this histogram. Error paths (`workos_error`, `no_org`, `internal_error`) are fast-fail validation or 401-redirect cases where latency isn't a useful SLO signal; including them would pollute `histogram_quantile` with near-instant samples and obscure real successful-login tail latency. Use `auth_login_total{result=...}` for error-rate dashboards. # TYPE auth_login_latency_seconds histogram auth_login_latency_seconds_bucket{le="0.1"} 0.0 auth_login_latency_seconds_bucket{le="0.25"} 0.0 auth_login_latency_seconds_bucket{le="0.5"} 0.0 auth_login_latency_seconds_bucket{le="1.0"} 0.0 auth_login_latency_seconds_bucket{le="2.5"} 0.0 auth_login_latency_seconds_bucket{le="5.0"} 0.0 auth_login_latency_seconds_bucket{le="10.0"} 0.0 auth_login_latency_seconds_bucket{le="30.0"} 0.0 auth_login_latency_seconds_bucket{le="+Inf"} 0.0 auth_login_latency_seconds_count 0.0 auth_login_latency_seconds_sum 0.0 # HELP auth_login_latency_seconds_created End-to-end latency of GET /auth (WorkOS code exchange + org-membership lookup + DB user/org/collection writes). Buckets reach 30 s because first-touch logins do significantly more work than refreshes (user create, default collection seed, org enrol). **Success-only observation:** only the `result='success'` branch observes into this histogram. Error paths (`workos_error`, `no_org`, `internal_error`) are fast-fail validation or 401-redirect cases where latency isn't a useful SLO signal; including them would pollute `histogram_quantile` with near-instant samples and obscure real successful-login tail latency. Use `auth_login_total{result=...}` for error-rate dashboards. # TYPE auth_login_latency_seconds_created gauge auth_login_latency_seconds_created 1.7869676368294873e+09 # HELP auth_jwt_verify_latency_seconds Latency of verify_jwt by routing path. Fires on every authenticated request, so this is the canonical view of how much time auth contributes to API tail latency. The user path includes any JWKS fetch (rare; PyJWKClient caches internally), service/desktop paths are local RSA verifies. # TYPE auth_jwt_verify_latency_seconds histogram # HELP auth_jwks_cache_miss_total WorkOS JWKS fetch events (first fetch, reinitialization, or key rotation) # TYPE auth_jwks_cache_miss_total counter auth_jwks_cache_miss_total 0.0 # HELP auth_jwks_cache_miss_created WorkOS JWKS fetch events (first fetch, reinitialization, or key rotation) # TYPE auth_jwks_cache_miss_created gauge auth_jwks_cache_miss_created 1.786967636829546e+09 # HELP auth_login_total GET /auth (OAuth code exchange) outcomes — first-touch login volume, distinct from refresh churn. `workos_transient` is the WorkOS-outage split (5xx / network failure → 503 so the client retries); `seat_limited` is an intentional 403 denial for a new billable member; `internal_error` covers anything else that escapes the explicit paths. The total is success + workos_error + workos_transient + no_org + seat_limited + internal_error. # TYPE auth_login_total counter auth_login_total{result="success"} 0.0 auth_login_total{result="workos_error"} 0.0 auth_login_total{result="workos_transient"} 0.0 auth_login_total{result="no_org"} 0.0 auth_login_total{result="seat_limited"} 0.0 auth_login_total{result="internal_error"} 0.0 # HELP auth_login_created GET /auth (OAuth code exchange) outcomes — first-touch login volume, distinct from refresh churn. `workos_transient` is the WorkOS-outage split (5xx / network failure → 503 so the client retries); `seat_limited` is an intentional 403 denial for a new billable member; `internal_error` covers anything else that escapes the explicit paths. The total is success + workos_error + workos_transient + no_org + seat_limited + internal_error. # TYPE auth_login_created gauge auth_login_created{result="success"} 1.7869676368296704e+09 auth_login_created{result="workos_error"} 1.7869676368296826e+09 auth_login_created{result="workos_transient"} 1.7869676368296914e+09 auth_login_created{result="no_org"} 1.786967636829698e+09 auth_login_created{result="seat_limited"} 1.7869676368297048e+09 auth_login_created{result="internal_error"} 1.7869676368297105e+09 # HELP auth_refresh_last_success_timestamp_seconds Unix timestamp of the most recent successful POST /auth/refresh on THIS process. The series is per-pod — each replica only advances its own gauge when it handles a refresh, so a quiet pod will look stale even when the fleet is healthy. Aggregate across pods with `max(...)` in Prometheus to get a true population-wide staleness signal. Alert predicate: `time() - max(...) by (env) > access_token_ttl AND max(...) by (env) > 0 AND sum(rate(auth_refresh_total[10m])) > 0` — the rate gate prevents false pages in low-traffic tenants where 'stale' means 'nobody tried recently', not 'refresh broken'. The `> 0` guard is required because an empty label-less Gauge emits a 0 sample on first scrape, and `time() - 0` permanently exceeds any reasonable TTL — without the guard, every cold rollout would page. Populated for free by real traffic — no synthetic probe required. # TYPE auth_refresh_last_success_timestamp_seconds gauge auth_refresh_last_success_timestamp_seconds 0.0 # HELP auth_active_users Distinct WorkOS user ids that have successfully authenticated within the rolling `window`. Counts USERS, not sessions: a user with two browser tabs plus the desktop client open contributes once per platform, not three times. `platform` is one of `PlatformLabel` — `web` (Bearer-header WorkOS access tokens), `office_addin` (WorkOS access token in the `modelml_addin_*` cookie), `desktop` (Tauri shell DTs) — plus a synthetic `platform="all"` series carrying the cross-platform deduped count. A user on both web and desktop counts as ONE in `platform="all"` but as ONE in each of `platform="web"` and `platform="desktop"` (so `sum(auth_active_users{platform!="all"})` would double-count; use `platform="all"` for the 'how many real users right now?' number). Source-of-truth is a Redis ZSET written fire-and-forget on every successful verify in `get_current_user` / `require_desktop_jwt_user` (dual-write to the per-platform key and the `all` key so dedup happens at insert time); this gauge is updated every 30 s by the `active_users` collector task registered in `_app_lifespan`. The ZSET is shared across all API pods on the same Redis, so a user hitting two pods is still counted once — every pod emits the same gauge value, and PromQL aggregation should use `max(auth_active_users{...})` to dedupe replica samples. Intended use: release-readiness ('how exposed are we right now?') and outage blast-radius ('how many real users are affected?') from the short windows; daily- and weekly-active counts from `24h` and `7d`. Every window is ROLLING, not calendar — `24h` is the trailing day, not midnight-to-midnight — and each reads low until the collector has been running for that window's length, since the ZSET only knows what it has observed. Still NOT a billing signal, and not a substitute for warehouse analytics: this counts distinct users that authenticated, which is a narrower thing than a product-defined active user. # TYPE auth_active_users gauge auth_active_users{platform="web",window="5m"} 0.0 auth_active_users{platform="web",window="15m"} 0.0 auth_active_users{platform="web",window="60m"} 0.0 auth_active_users{platform="web",window="24h"} 0.0 auth_active_users{platform="web",window="7d"} 0.0 auth_active_users{platform="desktop",window="5m"} 0.0 auth_active_users{platform="desktop",window="15m"} 0.0 auth_active_users{platform="desktop",window="60m"} 0.0 auth_active_users{platform="desktop",window="24h"} 0.0 auth_active_users{platform="desktop",window="7d"} 0.0 auth_active_users{platform="office_addin",window="5m"} 0.0 auth_active_users{platform="office_addin",window="15m"} 0.0 auth_active_users{platform="office_addin",window="60m"} 0.0 auth_active_users{platform="office_addin",window="24h"} 0.0 auth_active_users{platform="office_addin",window="7d"} 0.0 auth_active_users{platform="all",window="5m"} 0.0 auth_active_users{platform="all",window="15m"} 0.0 auth_active_users{platform="all",window="60m"} 0.0 auth_active_users{platform="all",window="24h"} 0.0 auth_active_users{platform="all",window="7d"} 0.0 # HELP dt_mint_total Desktop-token mint outcomes from _mint_desktop_token # TYPE dt_mint_total counter dt_mint_total{reason="success"} 0.0 # HELP dt_mint_created Desktop-token mint outcomes from _mint_desktop_token # TYPE dt_mint_created gauge dt_mint_created{reason="success"} 1.7869676368318324e+09 # HELP auth_forced_logout_total Involuntary session terminations — the user was bounced to re-auth without asking. The first-class signal for re-login storms (MML-14113): before this, they were only inferable from `auth_refresh_total{result="workos_error"}` + `invalid_grant` logs, and the 2026-06-11 cross-tenant incident was only ever reconstructed by hand. Phase 7's ship go/no-go is data-gated on this counter. All three labels are closed enums (no PII, no high-cardinality ids — those go in the Loki log line emitted alongside each increment), so the series count is bounded at |source| × |trigger| × |platform|. # TYPE auth_forced_logout_total counter auth_forced_logout_total{platform="web",source="office_addin",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="desktop",source="office_addin",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="office_addin",source="office_addin",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="web",source="office_addin",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="desktop",source="office_addin",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="office_addin",source="office_addin",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="web",source="office_addin",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="desktop",source="office_addin",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="office_addin",source="office_addin",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="web",source="office_addin",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="desktop",source="office_addin",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="office_addin",source="office_addin",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="web",source="web",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="desktop",source="web",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="office_addin",source="web",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="web",source="web",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="desktop",source="web",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="office_addin",source="web",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="web",source="web",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="desktop",source="web",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="office_addin",source="web",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="web",source="web",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="desktop",source="web",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="office_addin",source="web",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="web",source="desktop",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="desktop",source="desktop",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="office_addin",source="desktop",trigger="401_refresh_rejected"} 0.0 auth_forced_logout_total{platform="web",source="desktop",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="desktop",source="desktop",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="office_addin",source="desktop",trigger="503_given_up"} 0.0 auth_forced_logout_total{platform="web",source="desktop",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="desktop",source="desktop",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="office_addin",source="desktop",trigger="no_cookie"} 0.0 auth_forced_logout_total{platform="web",source="desktop",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="desktop",source="desktop",trigger="verify_failed"} 0.0 auth_forced_logout_total{platform="office_addin",source="desktop",trigger="verify_failed"} 0.0 # HELP auth_forced_logout_created Involuntary session terminations — the user was bounced to re-auth without asking. The first-class signal for re-login storms (MML-14113): before this, they were only inferable from `auth_refresh_total{result="workos_error"}` + `invalid_grant` logs, and the 2026-06-11 cross-tenant incident was only ever reconstructed by hand. Phase 7's ship go/no-go is data-gated on this counter. All three labels are closed enums (no PII, no high-cardinality ids — those go in the Loki log line emitted alongside each increment), so the series count is bounded at |source| × |trigger| × |platform|. # TYPE auth_forced_logout_created gauge auth_forced_logout_created{platform="web",source="office_addin",trigger="401_refresh_rejected"} 1.786967636831847e+09 auth_forced_logout_created{platform="desktop",source="office_addin",trigger="401_refresh_rejected"} 1.7869676368318598e+09 auth_forced_logout_created{platform="office_addin",source="office_addin",trigger="401_refresh_rejected"} 1.7869676368318732e+09 auth_forced_logout_created{platform="web",source="office_addin",trigger="503_given_up"} 1.7869676368318872e+09 auth_forced_logout_created{platform="desktop",source="office_addin",trigger="503_given_up"} 1.7869676368319008e+09 auth_forced_logout_created{platform="office_addin",source="office_addin",trigger="503_given_up"} 1.786967636831913e+09 auth_forced_logout_created{platform="web",source="office_addin",trigger="no_cookie"} 1.7869676368319242e+09 auth_forced_logout_created{platform="desktop",source="office_addin",trigger="no_cookie"} 1.7869676368319345e+09 auth_forced_logout_created{platform="office_addin",source="office_addin",trigger="no_cookie"} 1.7869676368319445e+09 auth_forced_logout_created{platform="web",source="office_addin",trigger="verify_failed"} 1.786967636831955e+09 auth_forced_logout_created{platform="desktop",source="office_addin",trigger="verify_failed"} 1.786967636831965e+09 auth_forced_logout_created{platform="office_addin",source="office_addin",trigger="verify_failed"} 1.7869676368319752e+09 auth_forced_logout_created{platform="web",source="web",trigger="401_refresh_rejected"} 1.7869676368319855e+09 auth_forced_logout_created{platform="desktop",source="web",trigger="401_refresh_rejected"} 1.7869676368319964e+09 auth_forced_logout_created{platform="office_addin",source="web",trigger="401_refresh_rejected"} 1.7869676368320065e+09 auth_forced_logout_created{platform="web",source="web",trigger="503_given_up"} 1.7869676368320167e+09 auth_forced_logout_created{platform="desktop",source="web",trigger="503_given_up"} 1.786967636832027e+09 auth_forced_logout_created{platform="office_addin",source="web",trigger="503_given_up"} 1.7869676368320372e+09 auth_forced_logout_created{platform="web",source="web",trigger="no_cookie"} 1.7869676368320475e+09 auth_forced_logout_created{platform="desktop",source="web",trigger="no_cookie"} 1.7869676368320572e+09 auth_forced_logout_created{platform="office_addin",source="web",trigger="no_cookie"} 1.7869676368320673e+09 auth_forced_logout_created{platform="web",source="web",trigger="verify_failed"} 1.7869676368320787e+09 auth_forced_logout_created{platform="desktop",source="web",trigger="verify_failed"} 1.7869676368320892e+09 auth_forced_logout_created{platform="office_addin",source="web",trigger="verify_failed"} 1.7869676368320987e+09 auth_forced_logout_created{platform="web",source="desktop",trigger="401_refresh_rejected"} 1.786967636832109e+09 auth_forced_logout_created{platform="desktop",source="desktop",trigger="401_refresh_rejected"} 1.7869676368321187e+09 auth_forced_logout_created{platform="office_addin",source="desktop",trigger="401_refresh_rejected"} 1.7869676368321288e+09 auth_forced_logout_created{platform="web",source="desktop",trigger="503_given_up"} 1.7869676368321393e+09 auth_forced_logout_created{platform="desktop",source="desktop",trigger="503_given_up"} 1.7869676368321495e+09 auth_forced_logout_created{platform="office_addin",source="desktop",trigger="503_given_up"} 1.7869676368321595e+09 auth_forced_logout_created{platform="web",source="desktop",trigger="no_cookie"} 1.7869676368321698e+09 auth_forced_logout_created{platform="desktop",source="desktop",trigger="no_cookie"} 1.7869676368321798e+09 auth_forced_logout_created{platform="office_addin",source="desktop",trigger="no_cookie"} 1.7869676368321905e+09 auth_forced_logout_created{platform="web",source="desktop",trigger="verify_failed"} 1.7869676368322005e+09 auth_forced_logout_created{platform="desktop",source="desktop",trigger="verify_failed"} 1.78696763683221e+09 auth_forced_logout_created{platform="office_addin",source="desktop",trigger="verify_failed"} 1.78696763683222e+09 # HELP auth_refresh_client_timeout_total Frontend server-side refresh fetches aborted at the client-side timeout (AUTH_REFRESH_TIMEOUT_MS, frontend/src/auth/constants.ts), reported via the unauthenticated POST /auth/telemetry/refresh-timeout beacon. A dead-socket abort never reaches the refresh handler, so this beacon is the only signal separating 'backend slow' (compare auth_refresh_latency_seconds tail vs the client budget) from 'backend unreachable' — and the data for tuning the client timeout. # TYPE auth_refresh_client_timeout_total counter auth_refresh_client_timeout_total{source="middleware"} 0.0 auth_refresh_client_timeout_total{source="route_handler"} 0.0 auth_refresh_client_timeout_total{source="desktop_exchange"} 0.0 auth_refresh_client_timeout_total{source="desktop_waiting"} 0.0 # HELP auth_refresh_client_timeout_created Frontend server-side refresh fetches aborted at the client-side timeout (AUTH_REFRESH_TIMEOUT_MS, frontend/src/auth/constants.ts), reported via the unauthenticated POST /auth/telemetry/refresh-timeout beacon. A dead-socket abort never reaches the refresh handler, so this beacon is the only signal separating 'backend slow' (compare auth_refresh_latency_seconds tail vs the client budget) from 'backend unreachable' — and the data for tuning the client timeout. # TYPE auth_refresh_client_timeout_created gauge auth_refresh_client_timeout_created{source="middleware"} 1.786967636831584e+09 auth_refresh_client_timeout_created{source="route_handler"} 1.7869676368315897e+09 auth_refresh_client_timeout_created{source="desktop_exchange"} 1.7869676368315947e+09 auth_refresh_client_timeout_created{source="desktop_waiting"} 1.7869676368316e+09 # HELP agent_v2_turns_total Agent v2 runs by terminal outcome. # TYPE agent_v2_turns_total counter # HELP agent_v2_model_turns_total Agent v2 terminal runs by effective model and outcome. # TYPE agent_v2_model_turns_total counter # HELP agent_v2_turn_first_visible_seconds Seconds from request creation to the first user-visible Agent v2 progress. # TYPE agent_v2_turn_first_visible_seconds histogram # HELP agent_v2_turn_answer_start_seconds Seconds from request creation to the first token of the Agent v2 final answer — when a chat user can start reading. Observed only for turns that ended with a visible answer, on surfaces with a live reader. # TYPE agent_v2_turn_answer_start_seconds histogram agent_v2_turn_answer_start_seconds_bucket{le="0.5",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="1.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="2.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="3.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="5.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="10.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="15.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="20.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="30.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="45.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="60.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="90.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="120.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="180.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="300.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="600.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="1200.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="3600.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="10800.0",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_bucket{le="+Inf",mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_count{mode="thread",platform="web"} 0.0 agent_v2_turn_answer_start_seconds_sum{mode="thread",platform="web"} 0.0 # HELP agent_v2_turn_answer_start_seconds_created Seconds from request creation to the first token of the Agent v2 final answer — when a chat user can start reading. Observed only for turns that ended with a visible answer, on surfaces with a live reader. # TYPE agent_v2_turn_answer_start_seconds_created gauge agent_v2_turn_answer_start_seconds_created{mode="thread",platform="web"} 1.7869676400996864e+09 # HELP agent_v2_turn_duration_seconds Seconds from request creation to the terminal Agent v2 outcome. Counts only turns that reached a terminal in-process, so it excludes the orphaned turns AGENT_V2_TURN_FAILURES_TOTAL records from the SSE tail. # TYPE agent_v2_turn_duration_seconds histogram agent_v2_turn_duration_seconds_bucket{le="0.5",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="1.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="2.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="3.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="5.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="10.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="15.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="20.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="30.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="45.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="60.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="90.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="120.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="180.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="300.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="600.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="1200.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="3600.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="10800.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="+Inf",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_count{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_sum{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_duration_seconds_bucket{le="0.5",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="1.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="2.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="3.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="5.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="10.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="15.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="20.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="30.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="45.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="60.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="90.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="120.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="180.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="300.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="600.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="1200.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="3600.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="10800.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="+Inf",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_count{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_sum{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_duration_seconds_bucket{le="0.5",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="1.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="2.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="3.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="5.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="10.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="15.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="20.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="30.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="45.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="60.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="90.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="120.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="180.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="300.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="600.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="1200.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="3600.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="10800.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_bucket{le="+Inf",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_count{mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_duration_seconds_sum{mode="thread",platform="web",status="cancelled"} 0.0 # HELP agent_v2_turn_duration_seconds_created Seconds from request creation to the terminal Agent v2 outcome. Counts only turns that reached a terminal in-process, so it excludes the orphaned turns AGENT_V2_TURN_FAILURES_TOTAL records from the SSE tail. # TYPE agent_v2_turn_duration_seconds_created gauge agent_v2_turn_duration_seconds_created{mode="thread",platform="web",status="success"} 1.7869676400997288e+09 agent_v2_turn_duration_seconds_created{mode="thread",platform="web",status="failed"} 1.7869676400997622e+09 agent_v2_turn_duration_seconds_created{mode="thread",platform="web",status="cancelled"} 1.786967640099793e+09 # HELP agent_v2_turn_iterations Agent loop iterations one Agent v2 turn consumed. Buckets are roughly geometric and top out at the 400-iteration budget itself, which is effectively never reached — so this measures thrash rather than budget exhaustion. Covers executor-staged turns only. Absent: grid cells (they record outcomes through their own metrics object, which never reaches this staging) and any turn that did not return from the agent loop, whether it raised inside the loop or failed before reaching it. Prefer this histogram's own _count as the denominator; agent_v2_turns_total is a wider population, so an all-mode ratio against it understates, though within a single mode the two do track. # TYPE agent_v2_turn_iterations histogram agent_v2_turn_iterations_bucket{le="1.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="2.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="3.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="5.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="8.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="13.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="21.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="34.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="55.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="89.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="144.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="240.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="400.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="+Inf",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_count{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_sum{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_iterations_bucket{le="1.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="2.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="3.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="5.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="8.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="13.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="21.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="34.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="55.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="89.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="144.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="240.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="400.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="+Inf",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_count{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_sum{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_iterations_bucket{le="1.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="2.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="3.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="5.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="8.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="13.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="21.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="34.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="55.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="89.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="144.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="240.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="400.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_bucket{le="+Inf",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_count{mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_iterations_sum{mode="thread",platform="web",status="cancelled"} 0.0 # HELP agent_v2_turn_iterations_created Agent loop iterations one Agent v2 turn consumed. Buckets are roughly geometric and top out at the 400-iteration budget itself, which is effectively never reached — so this measures thrash rather than budget exhaustion. Covers executor-staged turns only. Absent: grid cells (they record outcomes through their own metrics object, which never reaches this staging) and any turn that did not return from the agent loop, whether it raised inside the loop or failed before reaching it. Prefer this histogram's own _count as the denominator; agent_v2_turns_total is a wider population, so an all-mode ratio against it understates, though within a single mode the two do track. # TYPE agent_v2_turn_iterations_created gauge agent_v2_turn_iterations_created{mode="thread",platform="web",status="success"} 1.7869676400998235e+09 agent_v2_turn_iterations_created{mode="thread",platform="web",status="failed"} 1.7869676400998495e+09 agent_v2_turn_iterations_created{mode="thread",platform="web",status="cancelled"} 1.7869676400998743e+09 # HELP agent_v2_turn_tool_calls Tool calls one Agent v2 turn made. Distinct from AGENT_V2_TOOL_CALLS_TOTAL, which counts every call across all surfaces including subagents and grid cells, so the two do not reconcile. Covers executor-staged turns only. Absent: grid cells (they record outcomes through their own metrics object, which never reaches this staging) and any turn that did not return from the agent loop, whether it raised inside the loop or failed before reaching it. Prefer this histogram's own _count as the denominator; agent_v2_turns_total is a wider population, so an all-mode ratio against it understates, though within a single mode the two do track. # TYPE agent_v2_turn_tool_calls histogram agent_v2_turn_tool_calls_bucket{le="1.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="2.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="3.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="5.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="8.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="13.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="21.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="34.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="55.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="89.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="144.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="240.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="400.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="650.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="1000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="1600.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="+Inf",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_count{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_sum{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_tool_calls_bucket{le="1.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="2.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="3.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="5.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="8.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="13.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="21.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="34.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="55.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="89.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="144.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="240.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="400.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="650.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="1000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="1600.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="+Inf",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_count{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_sum{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_tool_calls_bucket{le="1.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="2.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="3.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="5.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="8.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="13.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="21.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="34.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="55.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="89.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="144.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="240.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="400.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="650.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="1000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="1600.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_bucket{le="+Inf",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_count{mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_tool_calls_sum{mode="thread",platform="web",status="cancelled"} 0.0 # HELP agent_v2_turn_tool_calls_created Tool calls one Agent v2 turn made. Distinct from AGENT_V2_TOOL_CALLS_TOTAL, which counts every call across all surfaces including subagents and grid cells, so the two do not reconcile. Covers executor-staged turns only. Absent: grid cells (they record outcomes through their own metrics object, which never reaches this staging) and any turn that did not return from the agent loop, whether it raised inside the loop or failed before reaching it. Prefer this histogram's own _count as the denominator; agent_v2_turns_total is a wider population, so an all-mode ratio against it understates, though within a single mode the two do track. # TYPE agent_v2_turn_tool_calls_created gauge agent_v2_turn_tool_calls_created{mode="thread",platform="web",status="success"} 1.7869676400998988e+09 agent_v2_turn_tool_calls_created{mode="thread",platform="web",status="failed"} 1.7869676400999265e+09 agent_v2_turn_tool_calls_created{mode="thread",platform="web",status="cancelled"} 1.786967640099954e+09 # HELP agent_v2_turn_output_tokens Output tokens one Agent v2 turn produced, summed across every model call in the turn. Reasoning tokens are already inside this figure — they are a subset of output, not an addition — so never sum the two. Covers executor-staged turns only. Absent: grid cells (they record outcomes through their own metrics object, which never reaches this staging) and any turn that did not return from the agent loop, whether it raised inside the loop or failed before reaching it. Prefer this histogram's own _count as the denominator; agent_v2_turns_total is a wider population, so an all-mode ratio against it understates, though within a single mode the two do track. # TYPE agent_v2_turn_output_tokens histogram agent_v2_turn_output_tokens_bucket{le="100.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="250.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="500.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="1000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="2000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="4000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="8000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="16000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="32000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="64000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="128000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="256000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="512000.0",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="+Inf",mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_count{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_sum{mode="thread",platform="web",status="success"} 0.0 agent_v2_turn_output_tokens_bucket{le="100.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="250.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="500.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="1000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="2000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="4000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="8000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="16000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="32000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="64000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="128000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="256000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="512000.0",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="+Inf",mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_count{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_sum{mode="thread",platform="web",status="failed"} 0.0 agent_v2_turn_output_tokens_bucket{le="100.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="250.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="500.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="1000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="2000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="4000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="8000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="16000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="32000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="64000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="128000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="256000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="512000.0",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_bucket{le="+Inf",mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_count{mode="thread",platform="web",status="cancelled"} 0.0 agent_v2_turn_output_tokens_sum{mode="thread",platform="web",status="cancelled"} 0.0 # HELP agent_v2_turn_output_tokens_created Output tokens one Agent v2 turn produced, summed across every model call in the turn. Reasoning tokens are already inside this figure — they are a subset of output, not an addition — so never sum the two. Covers executor-staged turns only. Absent: grid cells (they record outcomes through their own metrics object, which never reaches this staging) and any turn that did not return from the agent loop, whether it raised inside the loop or failed before reaching it. Prefer this histogram's own _count as the denominator; agent_v2_turns_total is a wider population, so an all-mode ratio against it understates, though within a single mode the two do track. # TYPE agent_v2_turn_output_tokens_created gauge agent_v2_turn_output_tokens_created{mode="thread",platform="web",status="success"} 1.7869676401000094e+09 agent_v2_turn_output_tokens_created{mode="thread",platform="web",status="failed"} 1.7869676401000335e+09 agent_v2_turn_output_tokens_created{mode="thread",platform="web",status="cancelled"} 1.7869676401000571e+09 # HELP agent_v2_auto_router_duration_seconds End-to-end seconds spent routing an Agent v2 Auto turn. # TYPE agent_v2_auto_router_duration_seconds histogram # HELP agent_v2_auto_router_model_selections_total Completed Agent v2 Auto model selections and switches by outcome. # TYPE agent_v2_auto_router_model_selections_total counter # HELP agent_v2_auto_router_reasoning_selections_total Completed Agent v2 Auto routes by applied reasoning level and outcome. # TYPE agent_v2_auto_router_reasoning_selections_total counter # HELP agent_v2_active_turns Agent v2 runs currently executing. # TYPE agent_v2_active_turns gauge # HELP agent_v2_tool_calls_total Terminal Agent v2 tool calls by tool, execution outcome, and mode. # TYPE agent_v2_tool_calls_total counter # HELP agent_v2_tool_call_duration_seconds Seconds from Agent v2 tool dispatch until its raw terminal result. # TYPE agent_v2_tool_call_duration_seconds histogram # HELP agent_v2_addin_script_outcome_total Office add-in tool calls by host app, tool, and outcome. outcome ok = add-in returned a value; null_result = success envelope with no value ({"result": null} — the script ran but the agent got no data back); runtime_error = add-in-side script failure; environmental = cell-edit-mode / host-runtime timeout (not the model's fault); revision_stale = the write was refused because the user moved the workbook under it, which is normal co-editing and not a failure; revision_stale_exhausted = too many refusals in a row, so the turn stopped redrafting and handed back to the user. error_class buckets the failure (empty on ok/null_result; the refusal reason on revision_stale). # TYPE agent_v2_addin_script_outcome_total counter # HELP agent_v2_addin_revision_stale_recovered_total Office add-in writes that landed after one or more stale-revision refusals, by host app, tool, and how many refusals preceded the write. # TYPE agent_v2_addin_revision_stale_recovered_total counter # HELP agent_v2_code_exec_total exec_command runs by sandbox pool and outcome. sandbox_mode python = V1 general PythonLTS pool, document_editing = V2 .NET document-editing pool; status unavailable = sandbox never started. # TYPE agent_v2_code_exec_total counter agent_v2_code_exec_total{sandbox_mode="python",status="success"} 0.0 agent_v2_code_exec_total{sandbox_mode="python",status="error"} 0.0 agent_v2_code_exec_total{sandbox_mode="python",status="timeout"} 0.0 agent_v2_code_exec_total{sandbox_mode="python",status="unavailable"} 0.0 agent_v2_code_exec_total{sandbox_mode="document_editing",status="success"} 0.0 agent_v2_code_exec_total{sandbox_mode="document_editing",status="error"} 0.0 agent_v2_code_exec_total{sandbox_mode="document_editing",status="timeout"} 0.0 agent_v2_code_exec_total{sandbox_mode="document_editing",status="unavailable"} 0.0 # HELP agent_v2_code_exec_created exec_command runs by sandbox pool and outcome. sandbox_mode python = V1 general PythonLTS pool, document_editing = V2 .NET document-editing pool; status unavailable = sandbox never started. # TYPE agent_v2_code_exec_created gauge agent_v2_code_exec_created{sandbox_mode="python",status="success"} 1.7869676400991666e+09 agent_v2_code_exec_created{sandbox_mode="python",status="error"} 1.786967640099173e+09 agent_v2_code_exec_created{sandbox_mode="python",status="timeout"} 1.786967640099179e+09 agent_v2_code_exec_created{sandbox_mode="python",status="unavailable"} 1.7869676400991852e+09 agent_v2_code_exec_created{sandbox_mode="document_editing",status="success"} 1.7869676400991917e+09 agent_v2_code_exec_created{sandbox_mode="document_editing",status="error"} 1.7869676400991976e+09 agent_v2_code_exec_created{sandbox_mode="document_editing",status="timeout"} 1.7869676400992038e+09 agent_v2_code_exec_created{sandbox_mode="document_editing",status="unavailable"} 1.7869676400992112e+09 # HELP agent_v2_exec_command_failure_total exec_command runs that exited non-zero, by coarse stderr class and whether a remediation hint was appended to the tool result. Scope is deliberately narrower than agent_v2_code_exec_total: timeouts and unavailable sandboxes never reach the hint path, so they are absent here and this counter divides cleanly into a hint-coverage rate. error_class no_stderr = exited non-zero with nothing on stderr, so the model had only an exit code to work from. # TYPE agent_v2_exec_command_failure_total counter agent_v2_exec_command_failure_total{error_class="no_stderr",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="no_stderr",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="sandbox_infra",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="sandbox_infra",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="oxml_schema",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="oxml_schema",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="officecli_args",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="officecli_args",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="shell_syntax",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="shell_syntax",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="python_syntax",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="python_syntax",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="path_or_module",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="path_or_module",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="python_runtime",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="python_runtime",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="other",hinted="true",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="other",hinted="false",sandbox_mode="python"} 0.0 agent_v2_exec_command_failure_total{error_class="no_stderr",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="no_stderr",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="sandbox_infra",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="sandbox_infra",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="oxml_schema",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="oxml_schema",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="officecli_args",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="officecli_args",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="shell_syntax",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="shell_syntax",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="python_syntax",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="python_syntax",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="path_or_module",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="path_or_module",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="python_runtime",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="python_runtime",hinted="false",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="other",hinted="true",sandbox_mode="document_editing"} 0.0 agent_v2_exec_command_failure_total{error_class="other",hinted="false",sandbox_mode="document_editing"} 0.0 # HELP agent_v2_exec_command_failure_created exec_command runs that exited non-zero, by coarse stderr class and whether a remediation hint was appended to the tool result. Scope is deliberately narrower than agent_v2_code_exec_total: timeouts and unavailable sandboxes never reach the hint path, so they are absent here and this counter divides cleanly into a hint-coverage rate. error_class no_stderr = exited non-zero with nothing on stderr, so the model had only an exit code to work from. # TYPE agent_v2_exec_command_failure_created gauge agent_v2_exec_command_failure_created{error_class="no_stderr",hinted="true",sandbox_mode="python"} 1.7869676400992675e+09 agent_v2_exec_command_failure_created{error_class="no_stderr",hinted="false",sandbox_mode="python"} 1.7869676400992763e+09 agent_v2_exec_command_failure_created{error_class="sandbox_infra",hinted="true",sandbox_mode="python"} 1.7869676400992835e+09 agent_v2_exec_command_failure_created{error_class="sandbox_infra",hinted="false",sandbox_mode="python"} 1.7869676400992901e+09 agent_v2_exec_command_failure_created{error_class="oxml_schema",hinted="true",sandbox_mode="python"} 1.7869676400992963e+09 agent_v2_exec_command_failure_created{error_class="oxml_schema",hinted="false",sandbox_mode="python"} 1.7869676400993028e+09 agent_v2_exec_command_failure_created{error_class="officecli_args",hinted="true",sandbox_mode="python"} 1.7869676400993087e+09 agent_v2_exec_command_failure_created{error_class="officecli_args",hinted="false",sandbox_mode="python"} 1.786967640099315e+09 agent_v2_exec_command_failure_created{error_class="shell_syntax",hinted="true",sandbox_mode="python"} 1.7869676400993214e+09 agent_v2_exec_command_failure_created{error_class="shell_syntax",hinted="false",sandbox_mode="python"} 1.786967640099328e+09 agent_v2_exec_command_failure_created{error_class="python_syntax",hinted="true",sandbox_mode="python"} 1.7869676400993347e+09 agent_v2_exec_command_failure_created{error_class="python_syntax",hinted="false",sandbox_mode="python"} 1.7869676400993419e+09 agent_v2_exec_command_failure_created{error_class="path_or_module",hinted="true",sandbox_mode="python"} 1.7869676400993483e+09 agent_v2_exec_command_failure_created{error_class="path_or_module",hinted="false",sandbox_mode="python"} 1.7869676400993545e+09 agent_v2_exec_command_failure_created{error_class="python_runtime",hinted="true",sandbox_mode="python"} 1.7869676400993612e+09 agent_v2_exec_command_failure_created{error_class="python_runtime",hinted="false",sandbox_mode="python"} 1.7869676400993676e+09 agent_v2_exec_command_failure_created{error_class="other",hinted="true",sandbox_mode="python"} 1.786967640099374e+09 agent_v2_exec_command_failure_created{error_class="other",hinted="false",sandbox_mode="python"} 1.786967640099381e+09 agent_v2_exec_command_failure_created{error_class="no_stderr",hinted="true",sandbox_mode="document_editing"} 1.786967640099388e+09 agent_v2_exec_command_failure_created{error_class="no_stderr",hinted="false",sandbox_mode="document_editing"} 1.7869676400993946e+09 agent_v2_exec_command_failure_created{error_class="sandbox_infra",hinted="true",sandbox_mode="document_editing"} 1.786967640099401e+09 agent_v2_exec_command_failure_created{error_class="sandbox_infra",hinted="false",sandbox_mode="document_editing"} 1.786967640099408e+09 agent_v2_exec_command_failure_created{error_class="oxml_schema",hinted="true",sandbox_mode="document_editing"} 1.7869676400994155e+09 agent_v2_exec_command_failure_created{error_class="oxml_schema",hinted="false",sandbox_mode="document_editing"} 1.786967640099422e+09 agent_v2_exec_command_failure_created{error_class="officecli_args",hinted="true",sandbox_mode="document_editing"} 1.7869676400994282e+09 agent_v2_exec_command_failure_created{error_class="officecli_args",hinted="false",sandbox_mode="document_editing"} 1.7869676400994341e+09 agent_v2_exec_command_failure_created{error_class="shell_syntax",hinted="true",sandbox_mode="document_editing"} 1.7869676400994406e+09 agent_v2_exec_command_failure_created{error_class="shell_syntax",hinted="false",sandbox_mode="document_editing"} 1.7869676400994651e+09 agent_v2_exec_command_failure_created{error_class="python_syntax",hinted="true",sandbox_mode="document_editing"} 1.7869676400994747e+09 agent_v2_exec_command_failure_created{error_class="python_syntax",hinted="false",sandbox_mode="document_editing"} 1.786967640099481e+09 agent_v2_exec_command_failure_created{error_class="path_or_module",hinted="true",sandbox_mode="document_editing"} 1.7869676400994878e+09 agent_v2_exec_command_failure_created{error_class="path_or_module",hinted="false",sandbox_mode="document_editing"} 1.7869676400994945e+09 agent_v2_exec_command_failure_created{error_class="python_runtime",hinted="true",sandbox_mode="document_editing"} 1.786967640099501e+09 agent_v2_exec_command_failure_created{error_class="python_runtime",hinted="false",sandbox_mode="document_editing"} 1.786967640099507e+09 agent_v2_exec_command_failure_created{error_class="other",hinted="true",sandbox_mode="document_editing"} 1.786967640099514e+09 agent_v2_exec_command_failure_created{error_class="other",hinted="false",sandbox_mode="document_editing"} 1.7869676400995204e+09 # HELP agent_v2_code_exec_duration_seconds Wall-clock seconds for one exec_command run, by sandbox pool. # TYPE agent_v2_code_exec_duration_seconds histogram # HELP agent_v2_exec_commands_per_turn exec_command calls within one completed agent run that used code execution at least once; runs with zero exec calls are not observed. # TYPE agent_v2_exec_commands_per_turn histogram # HELP agent_v2_code_sandbox_stage_skipped_total Workspace files dropped from a code-sandbox transfer because their path exceeds ACA's /files depth limit. direction upload = stage into the session, download = pull back to the workspace. # TYPE agent_v2_code_sandbox_stage_skipped_total counter agent_v2_code_sandbox_stage_skipped_total{direction="upload",reason="path_too_deep"} 0.0 agent_v2_code_sandbox_stage_skipped_total{direction="download",reason="path_too_deep"} 0.0 # HELP agent_v2_code_sandbox_stage_skipped_created Workspace files dropped from a code-sandbox transfer because their path exceeds ACA's /files depth limit. direction upload = stage into the session, download = pull back to the workspace. # TYPE agent_v2_code_sandbox_stage_skipped_created gauge agent_v2_code_sandbox_stage_skipped_created{direction="upload",reason="path_too_deep"} 1.7869676400995278e+09 agent_v2_code_sandbox_stage_skipped_created{direction="download",reason="path_too_deep"} 1.7869676400995345e+09 # HELP agent_v2_result_cache_lookups_total Shadow Agent v2 result-cache lookups by mode and outcome. # TYPE agent_v2_result_cache_lookups_total counter # HELP agent_v2_result_cache_writes_total Shadow Agent v2 result-cache writes by mode, outcome, and replay policy. # TYPE agent_v2_result_cache_writes_total counter # HELP agent_v2_result_cache_operation_duration_seconds Time spent on shadow Agent v2 result-cache and dependency operations. # TYPE agent_v2_result_cache_operation_duration_seconds histogram # HELP agent_v2_result_cache_age_seconds Age of valid shadow Agent v2 result-cache candidates. # TYPE agent_v2_result_cache_age_seconds histogram # HELP agent_v2_result_cache_payload_bytes Serialized shadow Agent v2 result-cache metadata size. # TYPE agent_v2_result_cache_payload_bytes histogram # HELP agent_v2_result_cache_avoided_tokens_total Tokens that a valid shadow Agent v2 result-cache hit would have avoided. # TYPE agent_v2_result_cache_avoided_tokens_total counter # HELP agent_v2_result_cache_comparisons_total Comparison of valid cached candidates with freshly executed Agent v2 results. # TYPE agent_v2_result_cache_comparisons_total counter # HELP agent_v2_result_cache_tool_policies_total Executed Agent v2 tools observed by result-replay policy. # TYPE agent_v2_result_cache_tool_policies_total counter # HELP agent_v2_result_cache_eligibility_blockers_total Agent v2 result-cache eligibility blockers, counted once per tool and run. # TYPE agent_v2_result_cache_eligibility_blockers_total counter # HELP agent_v2_prompt_cache_input_tokens_total Agent v2 input tokens split into fresh, cache-read, and cache-write buckets. # TYPE agent_v2_prompt_cache_input_tokens_total counter # HELP agent_v2_prompt_cache_calls_total Completed Agent v2 agent-loop calls by prompt-cache outcome. # TYPE agent_v2_prompt_cache_calls_total counter # HELP agent_v2_prompt_cache_usage_unavailable_total Agent v2 model attempts omitted from cache-token accounting. # TYPE agent_v2_prompt_cache_usage_unavailable_total counter # HELP agent_v2_model_output_tokens_total Agent v2 model output tokens reported by the provider. # TYPE agent_v2_model_output_tokens_total counter # HELP agent_v2_model_reasoning_tokens_total Agent v2 reasoning tokens; these are a subset of output tokens. # TYPE agent_v2_model_reasoning_tokens_total counter # HELP agent_v2_turn_degraded_total Agent v2 success turns that still delivered a final answer but hit a budget or safety rail — or self-reported friction that degraded or blocked the answer — compromising its quality. Counted once per degraded success turn. # TYPE agent_v2_turn_degraded_total counter # HELP agent_v2_turn_degradation_events_total Distinct degradation reasons on degraded Agent v2 success turns. # TYPE agent_v2_turn_degradation_events_total counter agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="salvage"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="truncation_continuation"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="truncated_placeholder"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="turn_end_reminder"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="final_answer_nudge"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="mixed_turn_truncation"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="empty_output_retry"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="mcp_tool_schema_demoted"} 0.0 agent_v2_turn_degradation_events_total{mode="thread",platform="web",reason="agent_reported_friction"} 0.0 # HELP agent_v2_turn_degradation_events_created Distinct degradation reasons on degraded Agent v2 success turns. # TYPE agent_v2_turn_degradation_events_created gauge agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="salvage"} 1.7869676400995417e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="truncation_continuation"} 1.7869676400995486e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="truncated_placeholder"} 1.7869676400995557e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="turn_end_reminder"} 1.786967640099562e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="final_answer_nudge"} 1.7869676400995684e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="mixed_turn_truncation"} 1.7869676400995746e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="empty_output_retry"} 1.7869676400995812e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="mcp_tool_schema_demoted"} 1.786967640099588e+09 agent_v2_turn_degradation_events_created{mode="thread",platform="web",reason="agent_reported_friction"} 1.7869676400995939e+09 # HELP agent_v2_turn_failures_total Failed Agent v2 turns by failure reason. # TYPE agent_v2_turn_failures_total counter agent_v2_turn_failures_total{mode="thread",platform="web",reason="empty_output"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="output_truncated"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="max_iterations"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="timed_out"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="provider_unavailable"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="rate_limited"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="request_too_large"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="transport_exhausted"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="orphaned"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="context_overflow"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="invalid_model"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="provider_bad_request"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="database_error"} 0.0 agent_v2_turn_failures_total{mode="thread",platform="web",reason="other"} 0.0 # HELP agent_v2_turn_failures_created Failed Agent v2 turns by failure reason. # TYPE agent_v2_turn_failures_created gauge agent_v2_turn_failures_created{mode="thread",platform="web",reason="empty_output"} 1.7869676401000812e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="output_truncated"} 1.7869676401000888e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="max_iterations"} 1.7869676401000965e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="timed_out"} 1.7869676401001034e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="provider_unavailable"} 1.7869676401001098e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="rate_limited"} 1.7869676401001165e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="request_too_large"} 1.7869676401001232e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="transport_exhausted"} 1.7869676401001296e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="orphaned"} 1.7869676401001356e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="context_overflow"} 1.7869676401001418e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="invalid_model"} 1.786967640100148e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="provider_bad_request"} 1.7869676401001544e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="database_error"} 1.7869676401001608e+09 agent_v2_turn_failures_created{mode="thread",platform="web",reason="other"} 1.7869676401001673e+09 # HELP agent_v2_degraded_operations_total Non-critical agent-runtime operations that raised and were degraded (the turn still produced a response), by stage and component. # TYPE agent_v2_degraded_operations_total counter # HELP agent_v2_turn_capability_total Turns that exercised each product capability, counted once per turn per capability, so the ratio against agent_v2_turns_total is the share of turns reaching for that capability. Counts the attempt rather than the outcome: a failed save still means the user reached for authoring. # TYPE agent_v2_turn_capability_total counter agent_v2_turn_capability_total{capability="skill_use",mode="thread",platform="web"} 0.0 agent_v2_turn_capability_total{capability="skill_authoring",mode="thread",platform="web"} 0.0 agent_v2_turn_capability_total{capability="memory_write",mode="thread",platform="web"} 0.0 agent_v2_turn_capability_total{capability="subagent",mode="thread",platform="web"} 0.0 agent_v2_turn_capability_total{capability="custom_agent",mode="thread",platform="web"} 0.0 agent_v2_turn_capability_total{capability="custom_agent_authoring",mode="thread",platform="web"} 0.0 agent_v2_turn_capability_total{capability="mcp",mode="thread",platform="web"} 0.0 # HELP agent_v2_turn_capability_created Turns that exercised each product capability, counted once per turn per capability, so the ratio against agent_v2_turns_total is the share of turns reaching for that capability. Counts the attempt rather than the outcome: a failed save still means the user reached for authoring. # TYPE agent_v2_turn_capability_created gauge agent_v2_turn_capability_created{capability="skill_use",mode="thread",platform="web"} 1.786967640099633e+09 agent_v2_turn_capability_created{capability="skill_authoring",mode="thread",platform="web"} 1.7869676400996397e+09 agent_v2_turn_capability_created{capability="memory_write",mode="thread",platform="web"} 1.7869676400996456e+09 agent_v2_turn_capability_created{capability="subagent",mode="thread",platform="web"} 1.786967640099652e+09 agent_v2_turn_capability_created{capability="custom_agent",mode="thread",platform="web"} 1.7869676400996592e+09 agent_v2_turn_capability_created{capability="custom_agent_authoring",mode="thread",platform="web"} 1.7869676400996656e+09 agent_v2_turn_capability_created{capability="mcp",mode="thread",platform="web"} 1.7869676400996723e+09 # HELP agent_v2_user_message_oversize_blocked_total Inline user messages rejected for exceeding the agent_v2 message cap. # TYPE agent_v2_user_message_oversize_blocked_total counter agent_v2_user_message_oversize_blocked_total{location="frontend"} 0.0 agent_v2_user_message_oversize_blocked_total{location="backend"} 0.0 # HELP agent_v2_user_message_oversize_blocked_created Inline user messages rejected for exceeding the agent_v2 message cap. # TYPE agent_v2_user_message_oversize_blocked_created gauge agent_v2_user_message_oversize_blocked_created{location="frontend"} 1.7869676400990484e+09 agent_v2_user_message_oversize_blocked_created{location="backend"} 1.786967640099055e+09 # HELP agent_v2_tool_output_truncated_total Tool outputs truncated inline after persisting the full output. # TYPE agent_v2_tool_output_truncated_total counter # HELP agent_v2_tool_output_persisted_total Full tool outputs persisted for recovery, by destination. `workspace` means the archive declined or failed and the legacy workspace path was used instead. Stage `compaction_arguments` is a call's arguments rather than its output. # TYPE agent_v2_tool_output_persisted_total counter agent_v2_tool_output_persisted_total{destination="archive",stage="execution"} 0.0 agent_v2_tool_output_persisted_total{destination="workspace",stage="execution"} 0.0 agent_v2_tool_output_persisted_total{destination="archive",stage="compaction"} 0.0 agent_v2_tool_output_persisted_total{destination="workspace",stage="compaction"} 0.0 # HELP agent_v2_tool_output_persisted_created Full tool outputs persisted for recovery, by destination. `workspace` means the archive declined or failed and the legacy workspace path was used instead. Stage `compaction_arguments` is a call's arguments rather than its output. # TYPE agent_v2_tool_output_persisted_created gauge agent_v2_tool_output_persisted_created{destination="archive",stage="execution"} 1.7869676400990624e+09 agent_v2_tool_output_persisted_created{destination="workspace",stage="execution"} 1.786967640099069e+09 agent_v2_tool_output_persisted_created{destination="archive",stage="compaction"} 1.7869676400990753e+09 agent_v2_tool_output_persisted_created{destination="workspace",stage="compaction"} 1.7869676400990815e+09 # HELP agent_v2_compaction_runs_total Background compaction runs by outcome; conflict means the history_revision guard lost a race to a normal turn persist. # TYPE agent_v2_compaction_runs_total counter agent_v2_compaction_runs_total{outcome="skipped_locked"} 0.0 agent_v2_compaction_runs_total{outcome="not_needed"} 0.0 agent_v2_compaction_runs_total{outcome="success"} 0.0 agent_v2_compaction_runs_total{outcome="conflict"} 0.0 agent_v2_compaction_runs_total{outcome="error"} 0.0 agent_v2_compaction_runs_total{outcome="blocked_workspace_quota"} 0.0 # HELP agent_v2_compaction_runs_created Background compaction runs by outcome; conflict means the history_revision guard lost a race to a normal turn persist. # TYPE agent_v2_compaction_runs_created gauge agent_v2_compaction_runs_created{outcome="skipped_locked"} 1.7869676400990877e+09 agent_v2_compaction_runs_created{outcome="not_needed"} 1.7869676400990932e+09 agent_v2_compaction_runs_created{outcome="success"} 1.786967640099099e+09 agent_v2_compaction_runs_created{outcome="conflict"} 1.7869676400991044e+09 agent_v2_compaction_runs_created{outcome="error"} 1.78696764009911e+09 agent_v2_compaction_runs_created{outcome="blocked_workspace_quota"} 1.7869676400991154e+09 # HELP agent_v2_compaction_step_tokens_saved_total Estimated context tokens saved by compaction step. # TYPE agent_v2_compaction_step_tokens_saved_total counter agent_v2_compaction_step_tokens_saved_total{step="tool_outputs"} 0.0 agent_v2_compaction_step_tokens_saved_total{step="images"} 0.0 agent_v2_compaction_step_tokens_saved_total{step="summary"} 0.0 # HELP agent_v2_compaction_step_tokens_saved_created Estimated context tokens saved by compaction step. # TYPE agent_v2_compaction_step_tokens_saved_created gauge agent_v2_compaction_step_tokens_saved_created{step="tool_outputs"} 1.786967640099121e+09 agent_v2_compaction_step_tokens_saved_created{step="images"} 1.7869676400991268e+09 agent_v2_compaction_step_tokens_saved_created{step="summary"} 1.7869676400991323e+09 # HELP agent_v2_compaction_degenerate_overflow_total Compaction runs that could not project below the target after the full cascade. # TYPE agent_v2_compaction_degenerate_overflow_total counter agent_v2_compaction_degenerate_overflow_total 0.0 # HELP agent_v2_compaction_degenerate_overflow_created Compaction runs that could not project below the target after the full cascade. # TYPE agent_v2_compaction_degenerate_overflow_created gauge agent_v2_compaction_degenerate_overflow_created 1.7869676400985837e+09 # HELP agent_v2_native_web_search_citations_total URL citations emitted by provider-hosted web search (OpenAI ``web_search`` / Anthropic ``web_search_20250305``). # TYPE agent_v2_native_web_search_citations_total counter agent_v2_native_web_search_citations_total{provider="openai"} 0.0 agent_v2_native_web_search_citations_total{provider="anthropic"} 0.0 # HELP agent_v2_native_web_search_citations_created URL citations emitted by provider-hosted web search (OpenAI ``web_search`` / Anthropic ``web_search_20250305``). # TYPE agent_v2_native_web_search_citations_created gauge agent_v2_native_web_search_citations_created{provider="openai"} 1.786967640099138e+09 agent_v2_native_web_search_citations_created{provider="anthropic"} 1.7869676400991437e+09 # HELP agent_v2_web_search_backend_selected_total Which web-search backend served a turn — labelled by the WebSearchBackend value (parallel / native / openai_proxy). Incremented once per ``prepare_agent`` call on turns where the web_search toolkit family is in scope. # TYPE agent_v2_web_search_backend_selected_total counter agent_v2_web_search_backend_selected_total{backend="parallel"} 0.0 agent_v2_web_search_backend_selected_total{backend="native"} 0.0 agent_v2_web_search_backend_selected_total{backend="openai_proxy"} 0.0 # HELP agent_v2_web_search_backend_selected_created Which web-search backend served a turn — labelled by the WebSearchBackend value (parallel / native / openai_proxy). Incremented once per ``prepare_agent`` call on turns where the web_search toolkit family is in scope. # TYPE agent_v2_web_search_backend_selected_created gauge agent_v2_web_search_backend_selected_created{backend="parallel"} 1.7869676400991492e+09 agent_v2_web_search_backend_selected_created{backend="native"} 1.7869676400991547e+09 agent_v2_web_search_backend_selected_created{backend="openai_proxy"} 1.78696764009916e+09 # HELP agent_v2_turn_fragility_events_total Recovered wobbles during an Agent v2 turn, by reason. The infra/transport/provider reasons fire once per recovery event, so a single turn can contribute several (e.g. three mid-stream transport retries); agent_reported_friction is model-self-reported and fires once per turn. # TYPE agent_v2_turn_fragility_events_total counter agent_v2_turn_fragility_events_total{mode="thread",platform="web",reason="midstream_retry"} 0.0 agent_v2_turn_fragility_events_total{mode="thread",platform="web",reason="initial_model_fallback"} 0.0 agent_v2_turn_fragility_events_total{mode="thread",platform="web",reason="lock_heartbeat_blip"} 0.0 agent_v2_turn_fragility_events_total{mode="thread",platform="web",reason="agent_reported_friction"} 0.0 # HELP agent_v2_turn_fragility_events_created Recovered wobbles during an Agent v2 turn, by reason. The infra/transport/provider reasons fire once per recovery event, so a single turn can contribute several (e.g. three mid-stream transport retries); agent_reported_friction is model-self-reported and fires once per turn. # TYPE agent_v2_turn_fragility_events_created gauge agent_v2_turn_fragility_events_created{mode="thread",platform="web",reason="midstream_retry"} 1.7869676400996006e+09 agent_v2_turn_fragility_events_created{mode="thread",platform="web",reason="initial_model_fallback"} 1.786967640099607e+09 agent_v2_turn_fragility_events_created{mode="thread",platform="web",reason="lock_heartbeat_blip"} 1.7869676400996132e+09 agent_v2_turn_fragility_events_created{mode="thread",platform="web",reason="agent_reported_friction"} 1.7869676400996196e+09 # HELP agent_v2_turn_fragile_total Agent v2 turns that recovered from at least one fragility event, counted once per turn regardless of the final outcome (a fragile turn that still failed is meaningful). Partly model-self-reported via agent_reported_friction, not purely runtime-observed. # TYPE agent_v2_turn_fragile_total counter agent_v2_turn_fragile_total{mode="thread",platform="web"} 0.0 # HELP agent_v2_turn_fragile_created Agent v2 turns that recovered from at least one fragility event, counted once per turn regardless of the final outcome (a fragile turn that still failed is meaningful). Partly model-self-reported via agent_reported_friction, not purely runtime-observed. # TYPE agent_v2_turn_fragile_created gauge agent_v2_turn_fragile_created{mode="thread",platform="web"} 1.786967640099626e+09 # HELP agent_v2_map_reduce_runs_total Map-reduce runs by terminal shape. refused = the map-reduce never dispatched a unit (flag, unit cap, merge pipeline, role delivery, spawn budget, claim clash, pre-dispatch steer); merged = its units reduced to one artifact; unmerged = it ran but the merge or the coverage assertion refused it. The reason a refusal names is prose and stays in the log body. # TYPE agent_v2_map_reduce_runs_total counter agent_v2_map_reduce_runs_total{outcome="merged"} 0.0 agent_v2_map_reduce_runs_total{outcome="unmerged"} 0.0 agent_v2_map_reduce_runs_total{outcome="refused"} 0.0 # HELP agent_v2_map_reduce_runs_created Map-reduce runs by terminal shape. refused = the map-reduce never dispatched a unit (flag, unit cap, merge pipeline, role delivery, spawn budget, claim clash, pre-dispatch steer); merged = its units reduced to one artifact; unmerged = it ran but the merge or the coverage assertion refused it. The reason a refusal names is prose and stays in the log body. # TYPE agent_v2_map_reduce_runs_created gauge agent_v2_map_reduce_runs_created{outcome="merged"} 1.7869676401001732e+09 agent_v2_map_reduce_runs_created{outcome="unmerged"} 1.7869676401001794e+09 agent_v2_map_reduce_runs_created{outcome="refused"} 1.7869676401001854e+09 # HELP agent_v2_map_reduce_units_total Map-reduce work units by the ledger state they settled at, counted once per unit per map-reduce. outstanding is expected to stay zero — a non-zero rate means the barrier released over a unit that never reported an outcome. # TYPE agent_v2_map_reduce_units_total counter agent_v2_map_reduce_units_total{state="abandoned"} 0.0 agent_v2_map_reduce_units_total{state="cancelled"} 0.0 agent_v2_map_reduce_units_total{state="degraded"} 0.0 agent_v2_map_reduce_units_total{state="failed"} 0.0 agent_v2_map_reduce_units_total{state="succeeded"} 0.0 agent_v2_map_reduce_units_total{state="outstanding"} 0.0 # HELP agent_v2_map_reduce_units_created Map-reduce work units by the ledger state they settled at, counted once per unit per map-reduce. outstanding is expected to stay zero — a non-zero rate means the barrier released over a unit that never reported an outcome. # TYPE agent_v2_map_reduce_units_created gauge agent_v2_map_reduce_units_created{state="abandoned"} 1.7869676401001909e+09 agent_v2_map_reduce_units_created{state="cancelled"} 1.7869676401001964e+09 agent_v2_map_reduce_units_created{state="degraded"} 1.7869676401002018e+09 agent_v2_map_reduce_units_created{state="failed"} 1.7869676401002073e+09 agent_v2_map_reduce_units_created{state="succeeded"} 1.7869676401002128e+09 agent_v2_map_reduce_units_created{state="outstanding"} 1.7869676401002178e+09 # HELP agent_v2_map_reduce_unit_dispatches_total Map-reduce unit attempts that reached a worker. One per unit plus one per retry, so dividing by agent_v2_map_reduce_units_total gives the retry rate. # TYPE agent_v2_map_reduce_unit_dispatches_total counter agent_v2_map_reduce_unit_dispatches_total 0.0 # HELP agent_v2_map_reduce_unit_dispatches_created Map-reduce unit attempts that reached a worker. One per unit plus one per retry, so dividing by agent_v2_map_reduce_units_total gives the retry rate. # TYPE agent_v2_map_reduce_unit_dispatches_created gauge agent_v2_map_reduce_unit_dispatches_created 1.7869676400987365e+09 # HELP agent_v2_map_reduce_duration_seconds Wall-clock seconds a map-reduce held its parent turn: windowed dispatch, the barrier and the merge. outcome refused is never observed — a map-reduce that dispatched nothing took no measurable time. # TYPE agent_v2_map_reduce_duration_seconds histogram agent_v2_map_reduce_duration_seconds_bucket{le="30.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="60.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="120.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="300.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="600.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="1200.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="1800.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="5520.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="10200.0",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="+Inf",outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_count{outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_sum{outcome="merged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="30.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="60.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="120.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="300.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="600.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="1200.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="1800.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="5520.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="10200.0",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_bucket{le="+Inf",outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_count{outcome="unmerged"} 0.0 agent_v2_map_reduce_duration_seconds_sum{outcome="unmerged"} 0.0 # HELP agent_v2_map_reduce_duration_seconds_created Wall-clock seconds a map-reduce held its parent turn: windowed dispatch, the barrier and the merge. outcome refused is never observed — a map-reduce that dispatched nothing took no measurable time. # TYPE agent_v2_map_reduce_duration_seconds_created gauge agent_v2_map_reduce_duration_seconds_created{outcome="merged"} 1.786967640100227e+09 agent_v2_map_reduce_duration_seconds_created{outcome="unmerged"} 1.7869676401002474e+09 # HELP agent_v2_map_reduce_merge_duration_seconds Seconds spent reducing a map-reduce's unit artifacts to one merged artifact, including the digest re-check and parquet decode of every input. Observed on every attempted merge, so a refusal's cost is measured too. # TYPE agent_v2_map_reduce_merge_duration_seconds histogram agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.05",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.1",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.25",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.5",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="1.0",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="2.5",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="5.0",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="10.0",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="30.0",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="60.0",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="300.0",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="+Inf",result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_count{result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_sum{result="merged"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.05",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.1",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.25",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.5",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="1.0",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="2.5",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="5.0",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="10.0",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="30.0",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="60.0",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="300.0",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="+Inf",result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_count{result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_sum{result="refused"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.05",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.1",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.25",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="0.5",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="1.0",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="2.5",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="5.0",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="10.0",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="30.0",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="60.0",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="300.0",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_bucket{le="+Inf",result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_count{result="coverage_failed"} 0.0 agent_v2_map_reduce_merge_duration_seconds_sum{result="coverage_failed"} 0.0 # HELP agent_v2_map_reduce_merge_duration_seconds_created Seconds spent reducing a map-reduce's unit artifacts to one merged artifact, including the digest re-check and parquet decode of every input. Observed on every attempted merge, so a refusal's cost is measured too. # TYPE agent_v2_map_reduce_merge_duration_seconds_created gauge agent_v2_map_reduce_merge_duration_seconds_created{result="merged"} 1.7869676401002667e+09 agent_v2_map_reduce_merge_duration_seconds_created{result="refused"} 1.7869676401002893e+09 agent_v2_map_reduce_merge_duration_seconds_created{result="coverage_failed"} 1.786967640100311e+09 # HELP agent_v2_map_reduce_coverage_assertions_total assert_coverage evaluations on map-reduces whose merge pipeline named one. fail means a claimed unit was neither in the merged output nor excluded by a coded rule, which fails the map-reduce. # TYPE agent_v2_map_reduce_coverage_assertions_total counter agent_v2_map_reduce_coverage_assertions_total{result="pass"} 0.0 agent_v2_map_reduce_coverage_assertions_total{result="fail"} 0.0 # HELP agent_v2_map_reduce_coverage_assertions_created assert_coverage evaluations on map-reduces whose merge pipeline named one. fail means a claimed unit was neither in the merged output nor excluded by a coded rule, which fails the map-reduce. # TYPE agent_v2_map_reduce_coverage_assertions_created gauge agent_v2_map_reduce_coverage_assertions_created{result="pass"} 1.7869676401003315e+09 agent_v2_map_reduce_coverage_assertions_created{result="fail"} 1.7869676401003377e+09 # HELP agent_v2_map_reduce_unsettled_total Map-reduces whose barrier released over units their own branch never settled, by reason — a deadline the map-reduce or one of its units missed, or a user instruction that arrived mid-map-reduce. Both can hold for one map-reduce, and neither is a success or a failure on its own. # TYPE agent_v2_map_reduce_unsettled_total counter agent_v2_map_reduce_unsettled_total{reason="deadline_missed"} 0.0 agent_v2_map_reduce_unsettled_total{reason="interrupted"} 0.0 # HELP agent_v2_map_reduce_unsettled_created Map-reduces whose barrier released over units their own branch never settled, by reason — a deadline the map-reduce or one of its units missed, or a user instruction that arrived mid-map-reduce. Both can hold for one map-reduce, and neither is a success or a failure on its own. # TYPE agent_v2_map_reduce_unsettled_created gauge agent_v2_map_reduce_unsettled_created{reason="deadline_missed"} 1.7869676401003435e+09 agent_v2_map_reduce_unsettled_created{reason="interrupted"} 1.7869676401003492e+09 # HELP agent_v2_map_reduce_ledger_contended_total Ledger row writes that exhausted their compare-and-set retries under contention and returned no state. The outcome is dropped and the unit falls to abandoned, recoverable via multi_agent_map_reduce_status. A byte-exact CAS over three attempts makes this rare, so any sustained rate is a real signal rather than noise. # TYPE agent_v2_map_reduce_ledger_contended_total counter agent_v2_map_reduce_ledger_contended_total 0.0 # HELP agent_v2_map_reduce_ledger_contended_created Ledger row writes that exhausted their compare-and-set retries under contention and returned no state. The outcome is dropped and the unit falls to abandoned, recoverable via multi_agent_map_reduce_status. A byte-exact CAS over three attempts makes this rare, so any sustained rate is a real signal rather than noise. # TYPE agent_v2_map_reduce_ledger_contended_created gauge agent_v2_map_reduce_ledger_contended_created 1.786967640098795e+09 # HELP agent_v2_map_reduce_advance_failures_total Advances that raised while topping up the window or running the merge (e.g. a Redis blip). The progress flag is restored so the next iteration boundary or the minute reconcile sweep retries, so a one-off is harmless but a sustained rate is a run at risk of never merging — page on it. # TYPE agent_v2_map_reduce_advance_failures_total counter agent_v2_map_reduce_advance_failures_total 0.0 # HELP agent_v2_map_reduce_advance_failures_created Advances that raised while topping up the window or running the merge (e.g. a Redis blip). The progress flag is restored so the next iteration boundary or the minute reconcile sweep retries, so a one-off is harmless but a sustained rate is a run at risk of never merging — page on it. # TYPE agent_v2_map_reduce_advance_failures_created gauge agent_v2_map_reduce_advance_failures_created 1.7869676400988045e+09 # HELP agent_v2_agent_reported_friction_total Friction the model reported about itself via ``report_agent_friction``, by category and impact. Counts one per report, so it does NOT reconcile with agent_v2_turn_degradation_events_total / agent_v2_turn_fragility_events_total, which count once per turn per reason (a turn may file up to three reports). Model self-assessment rather than an observed runtime rail, so its volume moves when the tool's prompt is tuned. # TYPE agent_v2_agent_reported_friction_total counter agent_v2_agent_reported_friction_total{category="tool_error",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="tool_error",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="tool_error",impact="blocked"} 0.0 agent_v2_agent_reported_friction_total{category="unexpected_output",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="unexpected_output",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="unexpected_output",impact="blocked"} 0.0 agent_v2_agent_reported_friction_total{category="missing_capability",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="missing_capability",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="missing_capability",impact="blocked"} 0.0 agent_v2_agent_reported_friction_total{category="conflicting_context",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="conflicting_context",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="conflicting_context",impact="blocked"} 0.0 agent_v2_agent_reported_friction_total{category="data_quality",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="data_quality",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="data_quality",impact="blocked"} 0.0 agent_v2_agent_reported_friction_total{category="performance",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="performance",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="performance",impact="blocked"} 0.0 agent_v2_agent_reported_friction_total{category="other",impact="recovered"} 0.0 agent_v2_agent_reported_friction_total{category="other",impact="degraded"} 0.0 agent_v2_agent_reported_friction_total{category="other",impact="blocked"} 0.0 # HELP agent_v2_agent_reported_friction_created Friction the model reported about itself via ``report_agent_friction``, by category and impact. Counts one per report, so it does NOT reconcile with agent_v2_turn_degradation_events_total / agent_v2_turn_fragility_events_total, which count once per turn per reason (a turn may file up to three reports). Model self-assessment rather than an observed runtime rail, so its volume moves when the tool's prompt is tuned. # TYPE agent_v2_agent_reported_friction_created gauge agent_v2_agent_reported_friction_created{category="tool_error",impact="recovered"} 1.7869676400989122e+09 agent_v2_agent_reported_friction_created{category="tool_error",impact="degraded"} 1.7869676400989223e+09 agent_v2_agent_reported_friction_created{category="tool_error",impact="blocked"} 1.7869676400989294e+09 agent_v2_agent_reported_friction_created{category="unexpected_output",impact="recovered"} 1.786967640098936e+09 agent_v2_agent_reported_friction_created{category="unexpected_output",impact="degraded"} 1.7869676400989428e+09 agent_v2_agent_reported_friction_created{category="unexpected_output",impact="blocked"} 1.786967640098949e+09 agent_v2_agent_reported_friction_created{category="missing_capability",impact="recovered"} 1.7869676400989552e+09 agent_v2_agent_reported_friction_created{category="missing_capability",impact="degraded"} 1.7869676400989616e+09 agent_v2_agent_reported_friction_created{category="missing_capability",impact="blocked"} 1.7869676400989678e+09 agent_v2_agent_reported_friction_created{category="conflicting_context",impact="recovered"} 1.786967640098974e+09 agent_v2_agent_reported_friction_created{category="conflicting_context",impact="degraded"} 1.7869676400989802e+09 agent_v2_agent_reported_friction_created{category="conflicting_context",impact="blocked"} 1.7869676400989864e+09 agent_v2_agent_reported_friction_created{category="data_quality",impact="recovered"} 1.7869676400989926e+09 agent_v2_agent_reported_friction_created{category="data_quality",impact="degraded"} 1.786967640098999e+09 agent_v2_agent_reported_friction_created{category="data_quality",impact="blocked"} 1.7869676400990052e+09 agent_v2_agent_reported_friction_created{category="performance",impact="recovered"} 1.7869676400990114e+09 agent_v2_agent_reported_friction_created{category="performance",impact="degraded"} 1.7869676400990171e+09 agent_v2_agent_reported_friction_created{category="performance",impact="blocked"} 1.786967640099023e+09 agent_v2_agent_reported_friction_created{category="other",impact="recovered"} 1.7869676400990295e+09 agent_v2_agent_reported_friction_created{category="other",impact="degraded"} 1.7869676400990357e+09 agent_v2_agent_reported_friction_created{category="other",impact="blocked"} 1.7869676400990415e+09 # HELP db_pool_connects_total New physical DBAPI connections opened by this process's pools. Steady-state traffic should be served from the pool, so a sustained rate here means connections are being created and discarded rather than reused — either overflow churn (demand above pool_size, each new connection paying a full TCP + TLS + auth round trip) or pool_recycle turnover. Per-process, so sum across replicas for a fleet view. # TYPE db_pool_connects_total counter db_pool_connects_total{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default"} 0.0 db_pool_connects_total{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query"} 0.0 db_pool_connects_total{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query"} 0.0 db_pool_connects_total{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default"} 169.0 db_pool_connects_total{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query"} 0.0 db_pool_connects_total{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query"} 0.0 # HELP db_pool_connects_created New physical DBAPI connections opened by this process's pools. Steady-state traffic should be served from the pool, so a sustained rate here means connections are being created and discarded rather than reused — either overflow churn (demand above pool_size, each new connection paying a full TCP + TLS + auth round trip) or pool_recycle turnover. Per-process, so sum across replicas for a fleet view. # TYPE db_pool_connects_created gauge db_pool_connects_created{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default"} 1.786967653451963e+09 db_pool_connects_created{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query"} 1.7869676534520078e+09 db_pool_connects_created{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query"} 1.786967653452037e+09 db_pool_connects_created{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default"} 1.7869676573170135e+09 db_pool_connects_created{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query"} 1.7869676573170514e+09 db_pool_connects_created{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query"} 1.7869676573170772e+09 # HELP db_pool_connections Connections currently held by a pool, by state. checked_out are in use by application code right now; checked_in are idle and immediately reusable. Compare checked_out against db_pool_limit{kind="size"} + db_pool_limit{kind="max_overflow"}: reaching that ceiling makes a further checkout wait for a connection to be returned and raise sqlalchemy.exc.TimeoutError if none is, so saturation shows up as both latency and errors. Sampled at scrape time, so a burst shorter than the scrape interval can be missed — db_pool_connects_total is the cumulative companion signal. # TYPE db_pool_connections gauge db_pool_connections{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default",state="checked_out"} 0.0 db_pool_connections{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default",state="checked_in"} 0.0 db_pool_connections{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query",state="checked_out"} 0.0 db_pool_connections{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query",state="checked_in"} 0.0 db_pool_connections{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query",state="checked_out"} 0.0 db_pool_connections{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query",state="checked_in"} 0.0 db_pool_connections{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default",state="checked_out"} 0.0 db_pool_connections{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="default",state="checked_in"} 52.0 db_pool_connections{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query",state="checked_out"} 0.0 db_pool_connections{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="moderately_slow_query",state="checked_in"} 0.0 db_pool_connections{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query",state="checked_out"} 0.0 db_pool_connections{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",session_type="slow_query",state="checked_in"} 0.0 # HELP db_pool_limit Configured pool bounds: size is the number of connections kept open, max_overflow the extra connections allowed above it before checkouts wait. Emitted as a metric rather than assumed by dashboards so a settings change moves the saturation threshold without a panel edit. # TYPE db_pool_limit gauge db_pool_limit{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="size",session_type="default"} 100.0 db_pool_limit{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="max_overflow",session_type="default"} 300.0 db_pool_limit{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="size",session_type="moderately_slow_query"} 100.0 db_pool_limit{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="max_overflow",session_type="moderately_slow_query"} 300.0 db_pool_limit{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="size",session_type="slow_query"} 100.0 db_pool_limit{application="app",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="max_overflow",session_type="slow_query"} 300.0 db_pool_limit{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="size",session_type="default"} 100.0 db_pool_limit{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="max_overflow",session_type="default"} 300.0 db_pool_limit{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="size",session_type="moderately_slow_query"} 100.0 db_pool_limit{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="max_overflow",session_type="moderately_slow_query"} 300.0 db_pool_limit{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="size",session_type="slow_query"} 100.0 db_pool_limit{application="temporal-worker",database="postgres",host="db-ftpartners.postgres.database.azure.com",kind="max_overflow",session_type="slow_query"} 300.0 # HELP redis_pool_failures_total Redis pool acquisitions that failed, by how the failure surfaced. acquire_fail means no connection could be obtained and the caller's request died, so a user saw an error; mid_use means an established connection failed while checked out, which callers of `redis_client()` swallow, so it is invisible to the user. Cumulative companion to redis_pool_connections: the gauge is sampled at scrape time and can miss a burst shorter than the scrape interval, this cannot. Per-process, so sum across replicas for a fleet view. # TYPE redis_pool_failures_total counter redis_pool_failures_total{outcome="acquire_fail",pool="app"} 0.0 redis_pool_failures_total{outcome="mid_use",pool="app"} 0.0 redis_pool_failures_total{outcome="acquire_fail",pool="notifications"} 0.0 redis_pool_failures_total{outcome="mid_use",pool="notifications"} 0.0 # HELP redis_pool_failures_created Redis pool acquisitions that failed, by how the failure surfaced. acquire_fail means no connection could be obtained and the caller's request died, so a user saw an error; mid_use means an established connection failed while checked out, which callers of `redis_client()` swallow, so it is invisible to the user. Cumulative companion to redis_pool_connections: the gauge is sampled at scrape time and can miss a burst shorter than the scrape interval, this cannot. Per-process, so sum across replicas for a fleet view. # TYPE redis_pool_failures_created gauge redis_pool_failures_created{outcome="acquire_fail",pool="app"} 1.7869676376486585e+09 redis_pool_failures_created{outcome="mid_use",pool="app"} 1.7869676376486707e+09 redis_pool_failures_created{outcome="acquire_fail",pool="notifications"} 1.7869676376486876e+09 redis_pool_failures_created{outcome="mid_use",pool="notifications"} 1.7869676376486952e+09 # HELP redis_pool_connections Connections held by a redis-py pool, by state. in_use are checked out by application code right now; available are idle and immediately reusable. Compare in_use against redis_pool_limit{kind="max_connections"}: this is a plain ConnectionPool, not BlockingConnectionPool, so reaching the limit raises ConnectionError("Too many connections") immediately rather than queueing — saturation shows up as errors, not latency. A connection checked out and never released stays counted here, so a monotonic climb that never recovers overnight is a leak, whereas load tracks the working day. A pool's series appear when that pool is first built, not at process start: a lazily-built pool such as data_ops is absent until first use, so absent means not yet used in this process rather than instrumentation broken — scope an absent() wiring check to pool="app". # TYPE redis_pool_connections gauge redis_pool_connections{pool="app",state="in_use"} 0.0 redis_pool_connections{pool="app",state="available"} 2.0 redis_pool_connections{pool="notifications",state="in_use"} 0.0 redis_pool_connections{pool="notifications",state="available"} 0.0 # HELP redis_pool_limit Configured ceiling on connections this process's pool will open. Emitted as a metric rather than assumed by dashboards so a settings change moves the saturation threshold without a panel edit. This is per process, not per deployment: alert on max across pods, never sum, or one saturated pod hides behind healthy siblings. # TYPE redis_pool_limit gauge redis_pool_limit{kind="max_connections",pool="app"} 500.0 redis_pool_limit{kind="max_connections",pool="notifications"} 32.0 # HELP sse_streams_in_flight Server-sent-event streams currently open in this process, by surface. Attributes Redis pool occupancy to a feature: a surface whose stream count falls while redis_pool_connections stays elevated is not returning its connections. Per-process, so alert on max across pods, not sum. # TYPE sse_streams_in_flight gauge sse_streams_in_flight{surface="notifications"} 0.0 sse_streams_in_flight{surface="chat"} 0.0 sse_streams_in_flight{surface="microsites"} 0.0 sse_streams_in_flight{surface="autocheck"} 0.0 # HELP temporal_activity_execution_latency Histogram of activity execution latencies # TYPE temporal_activity_execution_latency histogram temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="50"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="100"} 23 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="250"} 26 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="500"} 26 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1000"} 26 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="2500"} 33 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="5000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="10000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="20000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="30000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="60000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="120000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="300000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="600000"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="+Inf"} 35 temporal_activity_execution_latency_sum{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 22733 temporal_activity_execution_latency_count{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="50"} 0 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="100"} 0 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="250"} 0 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="500"} 0 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1000"} 0 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="2500"} 1 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="5000"} 3 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="10000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="20000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="30000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="60000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="120000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="300000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="600000"} 4 temporal_activity_execution_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="+Inf"} 4 temporal_activity_execution_latency_sum{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 13467 temporal_activity_execution_latency_count{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="50"} 0 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="100"} 47 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="250"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="500"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="2500"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="5000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="10000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="20000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="30000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="60000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="120000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="300000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="600000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="+Inf"} 63 temporal_activity_execution_latency_sum{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 6311 temporal_activity_execution_latency_count{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="50"} 0 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="100"} 48 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="250"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="500"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="2500"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="5000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="10000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="20000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="30000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="60000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="120000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="300000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="600000"} 63 temporal_activity_execution_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="+Inf"} 63 temporal_activity_execution_latency_sum{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 5944 temporal_activity_execution_latency_count{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="50"} 1 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="250"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="2500"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="5000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="20000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="30000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="60000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="120000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="300000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="600000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 2 temporal_activity_execution_latency_sum{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 104 temporal_activity_execution_latency_count{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="50"} 0 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 0 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="250"} 0 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="2500"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="5000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="20000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="30000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="60000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="120000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="300000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="600000"} 2 temporal_activity_execution_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 2 temporal_activity_execution_latency_sum{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 745 temporal_activity_execution_latency_count{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="50"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="100"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="250"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="500"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="2500"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="5000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="10000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="20000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="30000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="60000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="120000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="300000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="600000"} 4 temporal_activity_execution_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="+Inf"} 4 temporal_activity_execution_latency_sum{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 2 temporal_activity_execution_latency_count{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_activity_poll_no_task Count of activity task queue poll timeouts (no new task) # TYPE temporal_activity_poll_no_task counter temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 246 temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="live_data"} 498 temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="meetings"} 625 temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-high-mem"} 375 temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-mid-mem"} 498 temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 568 temporal_activity_poll_no_task{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot"} 625 # HELP temporal_activity_schedule_to_start_latency Histogram of activity schedule-to-start latencies # TYPE temporal_activity_schedule_to_start_latency histogram temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="100"} 0 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="500"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="1000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="5000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="10000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="30000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="60000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="120000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="300000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="600000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="1200000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="1800000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="3600000"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="+Inf"} 4 temporal_activity_schedule_to_start_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 878 temporal_activity_schedule_to_start_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 4 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="100"} 0 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="500"} 157 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="1000"} 168 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="5000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="10000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="30000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="60000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="120000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="300000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="600000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="1200000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="1800000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="3600000"} 169 temporal_activity_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="+Inf"} 169 temporal_activity_schedule_to_start_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 45577 temporal_activity_schedule_to_start_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 169 # HELP temporal_activity_succeed_endtoend_latency Histogram of activity execution latencies for successful activities # TYPE temporal_activity_succeed_endtoend_latency histogram temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="500"} 11 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1000"} 24 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="2500"} 30 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="10000"} 35 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="+Inf"} 35 temporal_activity_succeed_endtoend_latency_sum{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 39863 temporal_activity_succeed_endtoend_latency_count{activity_type="collections.description.refresh",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="500"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1000"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="2500"} 1 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="10000"} 4 temporal_activity_succeed_endtoend_latency_bucket{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="+Inf"} 4 temporal_activity_succeed_endtoend_latency_sum{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 18014 temporal_activity_succeed_endtoend_latency_count{activity_type="collections.description.sweep",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="500"} 29 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1000"} 59 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="2500"} 63 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="10000"} 63 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="+Inf"} 63 temporal_activity_succeed_endtoend_latency_sum{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 35330 temporal_activity_succeed_endtoend_latency_count{activity_type="email_capture.poll_gmail",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="500"} 26 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1000"} 62 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="2500"} 63 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="10000"} 63 temporal_activity_succeed_endtoend_latency_bucket{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="+Inf"} 63 temporal_activity_succeed_endtoend_latency_sum{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 35666 temporal_activity_succeed_endtoend_latency_count{activity_type="email_capture.poll_outlook",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 1 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="2500"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10000"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 2 temporal_activity_succeed_endtoend_latency_sum{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 996 temporal_activity_succeed_endtoend_latency_count{activity_type="flipline.reconcile_stuck_running",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 1 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="2500"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10000"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 2 temporal_activity_succeed_endtoend_latency_sum{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 1350 temporal_activity_succeed_endtoend_latency_count{activity_type="flipline.sweep_orphaned_blobs",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="50"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="100"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="500"} 0 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1000"} 4 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="2500"} 4 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="10000"} 4 temporal_activity_succeed_endtoend_latency_bucket{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="+Inf"} 4 temporal_activity_succeed_endtoend_latency_sum{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 2965 temporal_activity_succeed_endtoend_latency_count{activity_type="templates.migration.migrate_batch",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_activity_task_received Count of activity task queue poll successes # TYPE temporal_activity_task_received counter temporal_activity_task_received{activity_type="collections.description.refresh",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_activity_task_received{activity_type="collections.description.sweep",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_activity_task_received{activity_type="email_capture.poll_gmail",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_activity_task_received{activity_type="email_capture.poll_outlook",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_activity_task_received{activity_type="flipline.reconcile_stuck_running",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_activity_task_received{activity_type="flipline.sweep_orphaned_blobs",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_activity_task_received{activity_type="templates.migration.migrate_batch",eager="false",namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_long_request Count of long-poll request successes by rpc name # TYPE temporal_long_request counter temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline"} 254 temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data"} 505 temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings"} 636 temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem"} 378 temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem"} 502 temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared"} 748 temporal_long_request{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot"} 632 temporal_long_request{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline"} 625 temporal_long_request{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data"} 622 temporal_long_request{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings"} 625 temporal_long_request{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared"} 833 temporal_long_request{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot"} 622 # HELP temporal_long_request_latency Histogram of client long-poll request latencies # TYPE temporal_long_request_latency histogram temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="50"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="100"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="500"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="1000"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="2500"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="10000"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="+Inf"} 252 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline"} 14758054 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="flipline"} 252 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="50"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="100"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="500"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="1000"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="2500"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="10000"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="+Inf"} 501 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data"} 29506051 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="live_data"} 501 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="50"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="100"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="500"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="1000"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="2500"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="10000"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="+Inf"} 631 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings"} 37031311 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings"} 631 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="50"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="100"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="1000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="2500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="10000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem",le="+Inf"} 375 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem"} 22212768 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-high-mem"} 375 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="50"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="100"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="1000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="2500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="10000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",le="+Inf"} 498 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem"} 29506014 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="meetings-mid-mem"} 498 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="50"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="100"} 6 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="500"} 14 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="1000"} 22 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="2500"} 45 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="10000"} 120 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="+Inf"} 743 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared"} 37099284 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="shared"} 743 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="50"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="100"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="500"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="1000"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="2500"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="10000"} 2 temporal_long_request_latency_bucket{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="+Inf"} 627 temporal_long_request_latency_sum{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot"} 37034279 temporal_long_request_latency_count{namespace="default",operation="PollActivityTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot"} 627 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="50"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="100"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="500"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="1000"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="2500"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="10000"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline",le="+Inf"} 620 temporal_long_request_latency_sum{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline"} 36867195 temporal_long_request_latency_count{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="flipline"} 620 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="50"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="100"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="1000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="2500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="10000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data",le="+Inf"} 617 temporal_long_request_latency_sum{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data"} 36924733 temporal_long_request_latency_count{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="live_data"} 617 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="50"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="100"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="500"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="1000"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="2500"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="10000"} 3 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings",le="+Inf"} 620 temporal_long_request_latency_sum{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings"} 36936630 temporal_long_request_latency_count{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="meetings"} 620 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="50"} 1 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="100"} 1 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="500"} 70 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="1000"} 122 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="2500"} 192 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="10000"} 206 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared",le="+Inf"} 828 temporal_long_request_latency_sum{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared"} 37116425 temporal_long_request_latency_count{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="shared"} 828 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="50"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="100"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="1000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="2500"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="10000"} 0 temporal_long_request_latency_bucket{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot",le="+Inf"} 617 temporal_long_request_latency_sum{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot"} 36931287 temporal_long_request_latency_count{namespace="default",operation="PollWorkflowTaskQueue",service_name="temporal-core-sdk",task_queue="teams-bot"} 617 # HELP temporal_num_pollers Current number of active pollers per queue type # TYPE temporal_num_pollers gauge temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="flipline"} 2 temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="live_data"} 4 temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="meetings"} 5 temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="meetings-high-mem"} 3 temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="meetings-mid-mem"} 4 temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="shared"} 5 temporal_num_pollers{namespace="default",poller_type="activity_task",service_name="temporal-core-sdk",task_queue="teams-bot"} 5 temporal_num_pollers{namespace="default",poller_type="sticky_workflow_task",service_name="temporal-core-sdk",task_queue="flipline"} 4 temporal_num_pollers{namespace="default",poller_type="sticky_workflow_task",service_name="temporal-core-sdk",task_queue="live_data"} 4 temporal_num_pollers{namespace="default",poller_type="sticky_workflow_task",service_name="temporal-core-sdk",task_queue="meetings"} 4 temporal_num_pollers{namespace="default",poller_type="sticky_workflow_task",service_name="temporal-core-sdk",task_queue="shared"} 4 temporal_num_pollers{namespace="default",poller_type="sticky_workflow_task",service_name="temporal-core-sdk",task_queue="teams-bot"} 4 temporal_num_pollers{namespace="default",poller_type="workflow_task",service_name="temporal-core-sdk",task_queue="flipline"} 1 temporal_num_pollers{namespace="default",poller_type="workflow_task",service_name="temporal-core-sdk",task_queue="live_data"} 1 temporal_num_pollers{namespace="default",poller_type="workflow_task",service_name="temporal-core-sdk",task_queue="meetings"} 1 temporal_num_pollers{namespace="default",poller_type="workflow_task",service_name="temporal-core-sdk",task_queue="shared"} 1 temporal_num_pollers{namespace="default",poller_type="workflow_task",service_name="temporal-core-sdk",task_queue="teams-bot"} 1 # HELP temporal_request Count of client request successes by rpc name # TYPE temporal_request counter temporal_request{operation="GetSystemInfo",service_name="temporal-core-sdk"} 2 temporal_request{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk"} 6 temporal_request{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk"} 8 temporal_request{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk"} 6 temporal_request{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk"} 173 temporal_request{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk"} 344 temporal_request{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk"} 1 temporal_request{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared"} 35 # HELP temporal_request_failure Count of client request failures by rpc name # TYPE temporal_request_failure counter temporal_request_failure{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",status_code="ALREADY_EXISTS"} 6 temporal_request_failure{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",status_code="UNIMPLEMENTED"} 1 # HELP temporal_request_latency Histogram of client request latencies # TYPE temporal_request_latency histogram temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="50"} 2 temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="100"} 2 temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="500"} 2 temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="1000"} 2 temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="2500"} 2 temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="10000"} 2 temporal_request_latency_bucket{operation="GetSystemInfo",service_name="temporal-core-sdk",le="+Inf"} 2 temporal_request_latency_sum{operation="GetSystemInfo",service_name="temporal-core-sdk"} 2 temporal_request_latency_count{operation="GetSystemInfo",service_name="temporal-core-sdk"} 2 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="50"} 0 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="100"} 6 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="500"} 6 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="1000"} 6 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="2500"} 6 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="10000"} 6 temporal_request_latency_bucket{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk",le="+Inf"} 6 temporal_request_latency_sum{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk"} 434 temporal_request_latency_count{namespace="default",operation="CreateSchedule",service_name="temporal-core-sdk"} 6 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="50"} 4 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="100"} 4 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="500"} 8 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="1000"} 8 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="2500"} 8 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="10000"} 8 temporal_request_latency_bucket{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk",le="+Inf"} 8 temporal_request_latency_sum{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk"} 550 temporal_request_latency_count{namespace="default",operation="DescribeNamespace",service_name="temporal-core-sdk"} 8 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="50"} 0 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="100"} 2 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="500"} 4 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="1000"} 6 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="2500"} 6 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="10000"} 6 temporal_request_latency_bucket{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk",le="+Inf"} 6 temporal_request_latency_sum{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk"} 1631 temporal_request_latency_count{namespace="default",operation="RecordActivityTaskHeartbeat",service_name="temporal-core-sdk"} 6 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="50"} 0 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="100"} 0 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="500"} 168 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="1000"} 173 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="2500"} 173 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="10000"} 173 temporal_request_latency_bucket{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk",le="+Inf"} 173 temporal_request_latency_sum{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk"} 38762 temporal_request_latency_count{namespace="default",operation="RespondActivityTaskCompleted",service_name="temporal-core-sdk"} 173 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="50"} 0 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="100"} 0 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="500"} 333 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="1000"} 344 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="2500"} 344 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="10000"} 344 temporal_request_latency_bucket{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk",le="+Inf"} 344 temporal_request_latency_sum{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk"} 76779 temporal_request_latency_count{namespace="default",operation="RespondWorkflowTaskCompleted",service_name="temporal-core-sdk"} 344 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="50"} 1 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="100"} 1 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="500"} 1 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="1000"} 1 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="2500"} 1 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="10000"} 1 temporal_request_latency_bucket{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk",le="+Inf"} 1 temporal_request_latency_sum{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk"} 0 temporal_request_latency_count{namespace="default",operation="ShutdownWorker",service_name="temporal-core-sdk"} 1 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="50"} 0 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="100"} 0 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="500"} 29 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="1000"} 34 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="2500"} 35 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="10000"} 35 temporal_request_latency_bucket{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared",le="+Inf"} 35 temporal_request_latency_sum{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared"} 13081 temporal_request_latency_count{namespace="default",operation="StartWorkflowExecution",service_name="temporal-core-sdk",task_queue="shared"} 35 # HELP temporal_sticky_cache_hit Count of times the workflow cache was used for a new workflow task # TYPE temporal_sticky_cache_hit counter temporal_sticky_cache_hit{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 4 temporal_sticky_cache_hit{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_sticky_cache_hit{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_sticky_cache_hit{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_sticky_cache_hit{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_sticky_cache_hit{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_sticky_cache_size Current number of cached workflows # TYPE temporal_sticky_cache_size gauge temporal_sticky_cache_size{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 0 temporal_sticky_cache_size{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 0 # HELP temporal_worker_start Count of the number of initialized workers # TYPE temporal_worker_start counter temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="live_data"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="meetings"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-high-mem"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-mid-mem"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot"} 1 temporal_worker_start{namespace="default",service_name="temporal-core-sdk",task_queue="temporal-sys/worker-commands/default/3ca04e1b-641e-49af-bdd1-f28a4b6feab5"} 1 # HELP temporal_worker_task_slots_available Current number of available slots per task type # TYPE temporal_worker_task_slots_available gauge temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",worker_type="ActivityWorker"} 2 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",worker_type="LocalActivityWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",worker_type="WorkflowWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="live_data",worker_type="ActivityWorker"} 4 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="live_data",worker_type="LocalActivityWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="live_data",worker_type="WorkflowWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="meetings",worker_type="ActivityWorker"} 50 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="meetings",worker_type="LocalActivityWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="meetings",worker_type="WorkflowWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-high-mem",worker_type="ActivityWorker"} 3 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",worker_type="ActivityWorker"} 4 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="shared",worker_type="ActivityWorker"} 20 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="shared",worker_type="LocalActivityWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="shared",worker_type="WorkflowWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot",worker_type="ActivityWorker"} 20 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot",worker_type="LocalActivityWorker"} 100 temporal_worker_task_slots_available{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot",worker_type="WorkflowWorker"} 100 # HELP temporal_worker_task_slots_used Current number of used slots per task type # TYPE temporal_worker_task_slots_used gauge temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",worker_type="ActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",worker_type="LocalActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",worker_type="WorkflowWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="live_data",worker_type="ActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="live_data",worker_type="LocalActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="live_data",worker_type="WorkflowWorker"} 1 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="meetings",worker_type="ActivityWorker"} 1 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="meetings",worker_type="LocalActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="meetings",worker_type="WorkflowWorker"} 1 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-high-mem",worker_type="ActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="meetings-mid-mem",worker_type="ActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="shared",worker_type="ActivityWorker"} 1 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="shared",worker_type="LocalActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="shared",worker_type="WorkflowWorker"} 1 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot",worker_type="ActivityWorker"} 1 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot",worker_type="LocalActivityWorker"} 0 temporal_worker_task_slots_used{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot",worker_type="WorkflowWorker"} 1 # HELP temporal_workflow_completed Count of successfully completed workflows # TYPE temporal_workflow_completed counter temporal_workflow_completed{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_workflow_completed{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_workflow_completed{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_workflow_completed{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_workflow_completed{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_workflow_completed{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_workflow_endtoend_latency Histogram of total workflow execution latencies # TYPE temporal_workflow_endtoend_latency histogram temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="2000"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="5000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="30000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="60000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="120000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="300000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="600000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1800000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="3600000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="30600000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="86400000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 2 temporal_workflow_endtoend_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 5117 temporal_workflow_endtoend_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="100"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1000"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="2000"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="5000"} 1 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="10000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="30000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="60000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="120000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="300000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="600000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1800000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="3600000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="30600000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="86400000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="+Inf"} 4 temporal_workflow_endtoend_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 22495 temporal_workflow_endtoend_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="100"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1000"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1500"} 9 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="2000"} 17 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="5000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="10000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="30000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="60000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="120000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="300000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="600000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1800000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="3600000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="30600000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="86400000"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="+Inf"} 35 temporal_workflow_endtoend_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 86223 temporal_workflow_endtoend_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="100"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1000"} 11 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1500"} 36 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="2000"} 53 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="5000"} 61 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="10000"} 62 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="30000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="60000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="120000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="300000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="600000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1800000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="3600000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="30600000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="86400000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="+Inf"} 63 temporal_workflow_endtoend_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 114079 temporal_workflow_endtoend_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="100"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1000"} 8 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1500"} 31 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="2000"} 48 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="5000"} 61 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="10000"} 61 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="30000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="60000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="120000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="300000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="600000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1800000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="3600000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="30600000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="86400000"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="+Inf"} 63 temporal_workflow_endtoend_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 127875 temporal_workflow_endtoend_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="100"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="500"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1000"} 0 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1500"} 1 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="2000"} 2 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="5000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="10000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="30000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="60000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="120000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="300000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="600000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1800000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="3600000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="30600000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="86400000"} 4 temporal_workflow_endtoend_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="+Inf"} 4 temporal_workflow_endtoend_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 9030 temporal_workflow_endtoend_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_workflow_task_execution_latency Histogram of workflow task execution (not replay) latencies # TYPE temporal_workflow_task_execution_latency histogram temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1"} 5 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="20"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="50"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="200"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 6 temporal_workflow_task_execution_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 7 temporal_workflow_task_execution_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 6 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="10"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="20"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="50"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="100"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="200"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="500"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1000"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="+Inf"} 8 temporal_workflow_task_execution_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 8 temporal_workflow_task_execution_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1"} 61 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="10"} 68 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="20"} 68 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="50"} 68 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="100"} 68 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="200"} 68 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="500"} 69 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1000"} 70 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="+Inf"} 70 temporal_workflow_task_execution_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 1256 temporal_workflow_task_execution_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 70 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1"} 109 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="10"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="20"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="50"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="100"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="200"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="500"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1000"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="+Inf"} 126 temporal_workflow_task_execution_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 142 temporal_workflow_task_execution_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1"} 112 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="10"} 125 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="20"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="50"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="100"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="200"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="500"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1000"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="+Inf"} 126 temporal_workflow_task_execution_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 155 temporal_workflow_task_execution_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 126 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1"} 7 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="10"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="20"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="50"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="100"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="200"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="500"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1000"} 8 temporal_workflow_task_execution_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="+Inf"} 8 temporal_workflow_task_execution_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 9 temporal_workflow_task_execution_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 8 # HELP temporal_workflow_task_queue_poll_empty Count of workflow task queue poll timeouts (no new task) # TYPE temporal_workflow_task_queue_poll_empty counter temporal_workflow_task_queue_poll_empty{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 611 temporal_workflow_task_queue_poll_empty{namespace="default",service_name="temporal-core-sdk",task_queue="live_data"} 617 temporal_workflow_task_queue_poll_empty{namespace="default",service_name="temporal-core-sdk",task_queue="meetings"} 617 temporal_workflow_task_queue_poll_empty{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 489 temporal_workflow_task_queue_poll_empty{namespace="default",service_name="temporal-core-sdk",task_queue="teams-bot"} 617 # HELP temporal_workflow_task_queue_poll_succeed Count of workflow task queue poll successes # TYPE temporal_workflow_task_queue_poll_succeed counter temporal_workflow_task_queue_poll_succeed{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 6 temporal_workflow_task_queue_poll_succeed{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 338 # HELP temporal_workflow_task_replay_latency Histogram of workflow task replay latencies # TYPE temporal_workflow_task_replay_latency histogram temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="10"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="20"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="50"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="100"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="200"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="500"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="1000"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow",le="+Inf"} 4 temporal_workflow_task_replay_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 0 temporal_workflow_task_replay_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",workflow_type="FliplineReconcileWorkflow"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="10"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="20"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="50"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="100"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="200"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="500"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="1000"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow",le="+Inf"} 4 temporal_workflow_task_replay_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 0 temporal_workflow_task_replay_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionSweepWorkflow"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="10"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="20"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="50"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="100"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="200"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="500"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="1000"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow",le="+Inf"} 35 temporal_workflow_task_replay_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 0 temporal_workflow_task_replay_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="CollectionDescriptionWorkflow"} 35 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="10"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="20"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="50"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="100"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="200"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="500"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="1000"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow",le="+Inf"} 63 temporal_workflow_task_replay_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 0 temporal_workflow_task_replay_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="GmailCaptureWorkflow"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="10"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="20"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="50"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="100"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="200"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="500"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="1000"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow",le="+Inf"} 63 temporal_workflow_task_replay_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 0 temporal_workflow_task_replay_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="OutlookCaptureWorkflow"} 63 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="10"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="20"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="50"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="100"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="200"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="500"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="1000"} 4 temporal_workflow_task_replay_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow",le="+Inf"} 4 temporal_workflow_task_replay_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 0 temporal_workflow_task_replay_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared",workflow_type="TemplateMigrationTickWorkflow"} 4 # HELP temporal_workflow_task_schedule_to_start_latency Histogram of workflow task schedule-to-start latencies # TYPE temporal_workflow_task_schedule_to_start_latency histogram temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="100"} 0 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="500"} 5 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="1000"} 6 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="5000"} 6 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="10000"} 6 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="100000"} 6 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="1000000"} 6 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="flipline",le="+Inf"} 6 temporal_workflow_task_schedule_to_start_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 1360 temporal_workflow_task_schedule_to_start_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="flipline"} 6 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="100"} 0 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="500"} 298 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="1000"} 331 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="5000"} 336 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="10000"} 336 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="100000"} 338 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="1000000"} 338 temporal_workflow_task_schedule_to_start_latency_bucket{namespace="default",service_name="temporal-core-sdk",task_queue="shared",le="+Inf"} 338 temporal_workflow_task_schedule_to_start_latency_sum{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 139204 temporal_workflow_task_schedule_to_start_latency_count{namespace="default",service_name="temporal-core-sdk",task_queue="shared"} 338 # HELP temporal_sdk_proxy_up 1 when the loopback fetch of the Temporal SDK exporter returned metrics, 0 when it failed # TYPE temporal_sdk_proxy_up gauge temporal_sdk_proxy_up 1