Apache Airflow environment metrics in CloudWatch

DAG

Indicates a specific Apache Airflow DAG name.

DAG Filename

Indicates a specific Apache Airflow DAG file name.

Function

This dimension is used to improve the grouping of metrics in CloudWatch.

Job

Indicates an Apache Airflow job run by the scheduler. Always has a value of Job.

Operator

Indicates a specific Apache Airflow operator.

Pool

Indicates a specific Apache Airflow worker pool.

Task

Indicates a specific Apache Airflow task.

HostName

Indicates the hostname for a specific running Apache Airflow process.

SLAMissed

sla_missed

Count

Function, Scheduler

FailedSLACallback

sla_callback_notification_failure

Count

Function, Scheduler

Updates

dataset.updates

Count

Function, Scheduler

Orphaned

dataset.orphaned

Count

Function, Scheduler

FailedCeleryTaskExecution

celery.execute_command.failure

Count

Function, Celery

FilePathQueueUpdateCount

dag_processing.file_path_queue_update_count

Count

Function, Scheduler

CriticalSectionBusy

scheduler.critical_section_busy

Count

Function, Scheduler

DagBagSize

dagbag_size

Count

Function, DAG Processing

DagCallbackExceptions

dag.callback_exceptions

Count

DAG, All

FailedSLAEmailAttempts

sla_email_notification_failure

Count

Function, Scheduler

TaskInstanceFinished

ti.finish.{dag_id}.{task_id}.{state}

Count

DAG, {dag_id}

Task, {task_id}

State, {state}

JobEnd

{job_name}_end

Count

Job, {job_name}

JobHeartbeatFailure

{job_name}_heartbeat_failure

Count

Job, {job_name}

JobStart

{job_name}_start

Count

Job, {job_name}

ManagerStalls

dag_processing.manager_stalls

Count

Function, DAG Processing

OperatorFailures

operator_failures_{operator_name}

Count

Operator, {operator_name}

OperatorSuccesses

operator_successes_{operator_name}

Count

Operator, {operator_name}

OtherCallbackCount

dag_processing.other_callback_count

Count

Function, Scheduler

Processes

dag_processing.processes

Count

Function, DAG Processing

SchedulerHeartbeat

scheduler_heartbeat

Count

Function, Scheduler

StartedTaskInstances

ti.start.{dag_id}.{task_id}

Count

DAG, All

Task, All

SlaCallbackCount

dag_processing.sla_callback_count

Count

Function, Scheduler

TasksKilledExternally

scheduler.tasks.killed_externally

Count

Function, Scheduler

TaskTimeoutError

celery.task_timeout_error

Count

Function, Celery

TaskInstanceCreatedUsingOperator

task_instance_created-{operator_name}

Count

Operator, {operator_name}

TaskInstancePreviouslySucceeded

previously_succeeded

Count

DAG, All

Task, All

TaskInstanceFailures

ti_failures

Count

DAG, All

Task, All

TaskInstanceSuccesses

ti_successes

Count

DAG, All

Task, All

TaskRemovedFromDAG

task_removed_from_dag.{dag_id}

Count

DAG, {dag_id}

TaskRestoredToDAG

task_restored_to_dag.{dag_id}

Count

DAG, {dag_id}

TriggersSucceeded

triggers.succeeded

Count

Function, Trigger

TriggersFailed

triggers.failed

Count

Function, Trigger

TriggersBlockedMainThread

triggers.blocked_main_thread

Count

Function, Trigger

TriggerHeartbeat

triggerer_heartbeat

Count

Function, Triggerer

TaskInstanceCreatedUsingOperator

airflow.task_instance_created_{operator_name}

Count

Operator, {operator_name}

ZombiesKilled

zombies_killed

Count

DAG, All

Task, All

DAGFileRefreshError

dag_file_refresh_error

Count

Function, DAG Processing

ImportErrors

dag_processing.import_errors

Count

Function, DAG Processing

ExceptionFailures

smart_sensor_operator.exception_failures

Count

Function, Smart Sensor Operator

ExecutedTasks

smart_sensor_operator.executed_tasks

Count

Function, Smart Sensor Operator

InfraFailures

smart_sensor_operator.infra_failures

Count

Function, Smart Sensor Operator

LoadedTasks

smart_sensor_operator.loaded_tasks

Count

Function, Smart Sensor Operator

TotalParseTime

dag_processing.total_parse_time

Seconds

Function, DAG Processing

TriggeredDagRuns

dataset.triggered_dagruns

Count

Function, Scheduler

TriggersRunning

triggers.running.{hostname}

Count

Function, Trigger

HostName, {hostname}

PoolDeferredSlots

pool.deferred_slots.{pool_name}

Count

Pool, {pool_name}

DAGFileProcessingLastRunSecondsAgo

dag_processing.last_run.seconds_ago.{dag_filename}

Seconds

DAG Filename, {dag_filename}

OpenSlots

executor.open_slots

Count

Function, Executor

OrphanedTasksAdopted

scheduler.orphaned_tasks.adopted

Count

Function, Scheduler

OrphanedTasksCleared

scheduler.orphaned_tasks.cleared

Count

Function, Scheduler

PokedExceptions

smart_sensor_operator.poked_exception

Count

Function, Smart Sensor Operator

PokedSuccess

smart_sensor_operator.poked_success

Count

Function, Smart Sensor Operator

PokedTasks

smart_sensor_operator.poked_tasks

Count

Function, Smart Sensor Operator

PoolFailures

pool.open_slots.{pool_name}

Count

Pool, {pool_name}

PoolStarvingTasks

pool.starving_tasks.{pool_name}

Count

Pool, {pool_name}

PoolOpenSlots

pool.open_slots.{pool_name}

Count

Pool, {pool_name}

PoolQueuedSlots

pool.queued_slots.{pool_name}

Count

Pool, {pool_name}

PoolRunningSlots

pool.running_slots.{pool_name}

Count

Pool, {pool_name}

ProcessorTimeouts

dag_processing.processor_timeouts

Count

Function, DAG Processing

QueuedTasks

executor.queued_tasks

Count

Function, Executor

RunningTasks

executor.running_tasks

Count

Function, Executor

TasksExecutable

scheduler.tasks.executable

Count

Function, Scheduler

TasksPending

scheduler.tasks.pending

Count

Function, Scheduler

TasksRunning

scheduler.tasks.running

Count

Function, Scheduler

TasksStarving

scheduler.tasks.starving

Count

Function, Scheduler

TasksWithoutDagRun

scheduler.tasks.without_dagrun

Count

Function, Scheduler

DAGFileProcessingLastNumOfDbQueries

dag_processing.last_num_of_db_queries.{dag_filename}

Count

DAG Filename, {dag_filename}

PoolScheduledSlots

pool.scheduled_slots.{pool_name}

Count

Pool, {pool_name}

TaskCpuUsage

cpu.usage.{dag_id}.{task_id}

Percent

DAG, {dag_id}

Task, {task_id}

TaskMemoryUsage

mem.usage.{dag_id}.{task_id}

Percent

DAG, {dag_id}

Task, {task_id}

CollectDBDags

collect_db_dags

Milliseconds

Function, DAG Processing

CriticalSectionDuration

scheduler.critical_section_duration

Milliseconds

Function, Scheduler

CriticalSectionQueryDuration

scheduler.critical_section_query_duration

Milliseconds

Function, Scheduler

DAGDependencyCheck

dagrun.dependency-check.{dag_id}

Milliseconds

DAG, {dag_id}

DAGDurationFailed

dagrun.duration.failed.{dag_id}

Milliseconds

DAG, {dag_id}

DAGDurationSuccess

dagrun.duration.success.{dag_id}

Milliseconds

DAG, {dag_id}

DAGFileProcessingLastDuration

dag_processing.last_duration.{dag_filename}

Seconds

DAG Filename, {dag_filename}

DAGScheduleDelay

dagrun.schedule_delay.{dag_id}

Milliseconds

DAG, {dag_id}

FirstTaskSchedulingDelay

dagrun.{dag_id}.first_task_scheduling_delay

Milliseconds

DAG, {dag_id}

SchedulerLoopDuration

scheduler.scheduler_loop_duration

Milliseconds

Function, Scheduler

TaskInstanceDuration

dag.{dag_id}.{task_id}.duration

Milliseconds

DAG, {dag_id}

Task, {task_id}

TaskInstanceQueuedDuration

dag.{dag_id}.{task_id}.queued_duration

Milliseconds

DAG, {dag_id}

Task, {task_id}

TaskInstanceScheduledDuration

dag.{dag_id}.{task_id}.scheduled_duration

Milliseconds

DAG, {dag_id}

Task, {task_id}