#1134 add data-sources tag to active DAGs

dags/assets_pull.py

@@ -437,6 +437,7 @@ def pull_traffic_signal():
     default_args=DEFAULT_ARGS,
     max_active_runs=1,
     template_searchpath=[os.path.join(AIRFLOW_ROOT, 'assets/rlc/airflow/tasks')],
+    tags=["bdit_data-sources", "data_pull", "traffic_signals"],
     schedule='0 4 * * 1-5')
     # minutes past each hour | Hours (0-23) | Days of the month (1-31) | Months (1-12) | Days of the week (0-7, Sunday represented as either/both 0 and 7)
 

dags/citywide_tti_aggregate.py

@@ -48,7 +48,7 @@
     default_args=default_args, 
     schedule=None, # gets triggered by HERE dag
     doc_md = doc_md,
-    tags=["HERE"],
+    tags=["HERE", "aggregation"],
     catchup=False
 )
 

dags/ecocounter_check.py

@@ -47,7 +47,7 @@
     schedule='5 10 * * MON', # Run at 10:05 AM on Monday
     catchup=True,
     template_searchpath=os.path.join(repo_path, 'volumes/ecocounter/data_checks'),
-    tags=["ecocounter", "data_checks"],
+    tags=["ecocounter", "data_checks", "weekly"],
     doc_md=DOC_MD
 )
 def ecocounter_check_dag():

dags/gcc_layers_pull.py

@@ -24,7 +24,7 @@ def create_gcc_puller_dag(dag_id, default_args, name, conn_id):
         dag_id=dag_id,
         default_args=default_args,
         catchup=False,
-        tags=['gcc', name],
+        tags=["bdit_data-sources", "gcc", name, "quarterly"],
         schedule='0 7 1 */3 *' #'@quarterly'
     )
     def gcc_layers_dag():

dags/log_cleanup.py

@@ -62,6 +62,7 @@ def create_dag(filepath, doc, start_date, schedule_interval):
       # Prevent the same DAG from running concurrently more than once.
       max_active_runs=1,
       schedule=schedule_interval,
+      tags=['bdit_data-sources', 'maintenance'],
       # This allows us to simplify `create_bash_task` below.
       template_searchpath=AIRFLOW_TASKS
     )

dags/miovision_check.py

@@ -47,7 +47,7 @@
     schedule='0 4 * * MON', # Run at 4 AM on Monday
     catchup=False,
     template_searchpath=os.path.join(repo_path,'volumes/miovision/sql/data_checks'),
-    tags=["miovision", "data_checks"],
+    tags=["miovision", "data_checks", "weekly"],
     doc_md=DOC_MD
 )
 def miovision_check_dag():

dags/tti_aggregate.py

@@ -45,7 +45,7 @@
     default_args=default_args, 
     schedule=None, # gets triggered by HERE dag
     doc_md = doc_md,
-    tags=["HERE"],
+    tags=["HERE", "aggregation"],
     catchup=False
 )
 

dags/vds_check.py

@@ -48,7 +48,7 @@
     schedule='0 5 * * FRI', # Run at 5 AM on Friday
     catchup=False,
     template_searchpath=os.path.join(repo_path,'volumes/vds/sql/select'),
-    tags=["vds", "data_checks"],
+    tags=["bdit_data-sources", "vds", "data_checks", "weekly"],
     doc_md=DOC_MD
 )
 def vds_check_dag():

dags/vds_pull_vdsdata.py

@@ -46,7 +46,7 @@
         os.path.join(repo_path,'dags/sql')
     ],
     doc_md=DOC_MD,
-    tags=['vds', 'vdsdata', 'data_checks', 'pull', 'detector_inventory'],
+    tags=["bdit_data-sources", 'vds', 'vdsdata', 'data_checks', 'data_pull', 'detector_inventory'],
     schedule='0 4 * * *' #daily at 4am
 )
 def vdsdata_dag():

dags/vds_pull_vdsvehicledata.py

@@ -44,7 +44,7 @@
         os.path.join(repo_path,'dags/sql')
     ],
     doc_md=DOC_MD,
-    tags=['vds', 'vdsvehicledata', 'data_checks', 'pull'],
+    tags=["bdit_data-sources", 'vds', 'vdsvehicledata', 'data_checks', 'data_pull'],
     schedule='5 4 * * *' #daily at 4:05am
 )
 def vdsvehicledata_dag():