Saved first artifact using Jupyter notebook and custom plugins

usnistgov · Feb 4, 2025 · d56d4f3 · d56d4f3
1 parent b0b7a25
commit d56d4f3
Show file tree

Hide file tree

Showing 9 changed files with 17,502 additions and 0 deletions.
diff --git a/examples/data-science-demo/data_science_demo.html b/examples/data-science-demo/data_science_demo.html
diff --git a/examples/data-science-demo/data_science_demo_plugins.ipynb b/examples/data-science-demo/data_science_demo_plugins.ipynb
diff --git a/examples/data-science-demo/data_science_experiment.ipynb b/examples/data-science-demo/data_science_experiment.ipynb
diff --git a/examples/data-science-demo/src/fit_estimator.yml b/examples/data-science-demo/src/fit_estimator.yml
@@ -0,0 +1,106 @@
+types:
+# to be developed
+  my_type_1:
+  my_type_2:
+
+parameters:
+  experiment_id: 'placeholder_id'
+  job_id: 'placeholder_id'
+  estimator_degree: 3
+
+
+tasks:
+  #load_artifacts_for_job:
+  #  plugin: dioptra_custom.data_science_example.plugins.load_artifacts_for_job
+  #  inputs:
+  #    - name: job_id
+  #      type: string
+  #      required: true
+  retrieve_first_artifact_by_job_id:
+    plugin: dioptra_custom.data_science_example.plugins.retrieve_first_artifact_by_job_id
+    inputs:
+      - name: experiment_id
+        type: string
+        required: true
+      - name: job_id
+        type: string
+        required: true
+    outputs:
+      df: any
+
+  split_df:
+    plugin: dioptra_custom.data_science_example.plugins.split_df
+    inputs: 
+    - name: df
+      type: any
+      required: true
+    - name: estimator_degree
+      type: integer
+      required: true
+    - name: x_vars
+      type: any
+      required: false
+    - name: y_var
+      type: string
+      required: false
+    - name: OOD_percent
+      type: number
+      required: false
+    - name: test_size
+      type: number
+      required: false
+    outputs:
+      splits_dict: any
+
+  fit_linear_model:
+    plugin: dioptra_custom.data_science_example.plugins.fit_linear_model
+    inputs:
+    - name: splits_dict
+      type: any
+      required: true
+    outputs:
+      model: any
+
+  evaluate_linear_model:
+    plugin: dioptra_custom.data_science_example.plugins.evaluate_linear_model 
+    inputs:
+    - name: splits_dict
+      type: any
+      required: true
+    - name: model
+      type: any
+      required: true
+    output:
+      metrics: any
+
+
+
+graph:    
+  load:
+    #load_artifacts_for_job: 
+    #  job_id: $job_id
+    retrieve_first_artifact_by_job_id:
+      experiment_id: $experiment_id
+      job_id: $job_id
+
+  split:
+    split_df:
+      df: $load.df
+      estimator_degree: $estimator_degree
+    dependencies:
+      - load
+
+  train:
+    fit_linear_model:
+      splits_dict: $split.splits_dict
+    dependencies:
+      - split
+
+  eval:
+    evaluate_linear_model:
+      splits_dict: $split.splits_dict
+      model: $train.model
+    dependencies:
+      - train
+
+
diff --git a/examples/data-science-demo/src/generate_sim_data.yml b/examples/data-science-demo/src/generate_sim_data.yml
@@ -0,0 +1,81 @@
+types:
+# to be developed
+  my_type_1:
+  my_type_2:
+
+parameters:
+  poly_degree: 3
+  sample_size: 100
+  df_file_name: 'simulated_df.csv'
+  df_file_format: 'csv'
+
+tasks:
+  gen_poly_func:
+    plugin: dioptra_custom.data_science_example.plugins.gen_poly_func
+    inputs:
+      - name: poly_degree
+        type: integer
+        required: true
+      - name: min_x
+        type: integer
+        required: false
+      - name: max_x
+        type: integer
+        required: false
+      - name: y_range_max
+        type: integer
+        required: false
+    outputs:
+      - polynomial_dict: any
+
+
+  simulate_data_from_poly:
+    plugin: dioptra_custom.data_science_example.plugins.simulate_data_from_poly
+    inputs:
+      - name: polynomial_dict
+        type: any
+        required: true
+      - name: sample_size
+        type: integer
+        required: true
+      - name: noise_mean
+        type: number
+        required: false
+      - name: noise_var
+        type: number
+        required: false
+    outputs:
+      sim_data_df: any
+
+  upload_df:
+    plugin: dioptra_custom.data_science_example.plugins.upload_df
+    inputs:
+      - name: df
+        type: any
+        requried: true
+      - name: file_name
+        type: string
+        required: false
+      - name: file_format
+        type: string
+        required: false
+
+
+graph:    
+  polynomial:
+    gen_poly_func:
+      poly_degree: $poly_degree    
+
+  data:
+    simulate_data_from_poly:
+      polynomial_dict: $polynomial.polynomial_dict
+      sample_size: $sample_size
+    dependencies:
+      - polynomial
+
+  save:
+    upload_df:
+      df: $data.sim_data_df
+      file_name: $df_file_name
+      file_format: $df_file_format
+