AGILE and INTEGRAL Text Conversions

.gitignore

@@ -2,3 +2,4 @@
 .vscode
 __pycache__
 .DS_Store
+dist

gcn_classic_text_to_json/__main__.py

@@ -1,4 +1,12 @@
+from gcn_classic_text_to_json.notices.alexis import conversion as alexis_conversion
 from gcn_classic_text_to_json.notices.calet import conversion as calet_conversion
+from gcn_classic_text_to_json.notices.gecam import conversion as gecam_conversion
+from gcn_classic_text_to_json.notices.sk_sn import conversion as sk_sn_conversion
+from gcn_classic_text_to_json.notices.snews import conversion as snews_conversion
 
 if __name__ == "__main__":
+    alexis_conversion.create_all_alexis_jsons()
     calet_conversion.create_all_calet_jsons()
+    gecam_conversion.create_all_gecam_jsons()
+    sk_sn_conversion.create_all_sk_sn_jsons()
+    snews_conversion.create_all_snews_jsons()

gcn_classic_text_to_json/conversion.py

@@ -18,8 +18,10 @@
     "Dec",
 ]
 
+invalid_trigger_dates = ["(yy-mm-dd)", "(yy/mm/dd)", "(yyyy/mm/dd)"]
 
-def parse_trigger_links(link, prefix):
+
+def parse_trigger_links(link, prefix, regex_string):
     """Returns a list of trigger_links present in `link`.
 
     The function parses through each row and hyperlink present in the html
@@ -30,7 +32,9 @@ def parse_trigger_links(link, prefix):
     link: string
         The webpage with the trigger links listed.
     prefix: string
-        The prefix to be added to the incomplete link
+        The prefix to be added to the incomplete link.
+    regex_string: string
+        Regex string to search for while looking through links.
 
     Returns
     -------
@@ -42,7 +46,7 @@ def parse_trigger_links(link, prefix):
     soup = BeautifulSoup(data, "html.parser")
 
     links = set({})
-    for incomplete_link in soup.find_all("a", attrs={"href": re.compile("other")}):
+    for incomplete_link in soup.find_all("a", attrs={"href": re.compile(regex_string)}):
         link = prefix + incomplete_link.get("href")
         links.add(link)
     return links
@@ -94,15 +98,17 @@ def text_to_json(notice, keywords_dict):
         notice_id = keywords_dict["standard"]["id"]
         id_data = notice[notice_id]
         id = id_data.split()[0]
-        output["id"] = int(id)
+        output["id"] = [int(id)]
 
     if "trigger_time" in keywords_dict["standard"]:
         notice_date, notice_time = keywords_dict["standard"]["trigger_time"]
         trigger_date_data = notice[notice_date].split()
 
         trigger_date = trigger_date_data[-1]
-        if trigger_date == "(yy-mm-dd)":
+        if trigger_date in invalid_trigger_dates:
             trigger_date = trigger_date_data[-2]
+        if len(trigger_date) == 8:
+            trigger_date = "20" + trigger_date
 
         trigger_time_data = notice[notice_time]
         trigger_time_start_idx = trigger_time_data.find("{")
@@ -113,24 +119,30 @@ def text_to_json(notice, keywords_dict):
         trigger_datetime = f"{trigger_date.replace('/', '-', 2)}T{trigger_time}Z"
         output["trigger_time"] = trigger_datetime
 
-    if "ra_dec" in keywords_dict["standard"]:
-        notice_ra, notice_dec = keywords_dict["standard"]["ra_dec"]
-
-        ra_data = notice[notice_ra]
-        output["ra"] = float(ra_data.split()[0][:-1])
-
-        dec_data = notice[notice_dec]
-        output["dec"] = float(dec_data.split()[0][:-1])
-
-    for json_keyword, notice_keyword_tuple in keywords_dict["additional"].items():
-        notice_keyword, keyword_type = notice_keyword_tuple
-        if notice.get(notice_keyword) is not None:
-            val = notice[notice_keyword].split()[0]
-            if keyword_type == "int":
-                val = int(val)
-            elif keyword_type == "float":
-                val = float(val)
-            output[json_keyword] = val
+    if "ra" in keywords_dict["standard"]:
+        notice_ra = keywords_dict["standard"]["ra"]
+        ra_data = notice[notice_ra].split()
+
+        if ra_data[0] != "Undefined":
+            output["ra"] = float(ra_data[0][:-1])
+
+    if "dec" in keywords_dict["standard"]:
+        notice_dec = keywords_dict["standard"]["dec"]
+        dec_data = notice[notice_dec].split()
+
+        if dec_data[0] != "Undefined":
+            output["dec"] = float(dec_data[0][:-1])
+
+    if "additional" in keywords_dict:
+        for json_keyword, notice_keyword_tuple in keywords_dict["additional"].items():
+            notice_keyword, keyword_type = notice_keyword_tuple
+            if notice.get(notice_keyword) is not None and notice.get(notice_keyword):
+                val = notice[notice_keyword].split()[0]
+                if keyword_type == "int":
+                    val = int(val)
+                elif keyword_type == "float":
+                    val = float(val)
+                output[json_keyword] = val
 
     output["additional_info"] = notice["COMMENTS"]
 

gcn_classic_text_to_json/notices/agile/README.md

@@ -0,0 +1,25 @@
+# AGILE Text Conversion
+
+Parses through all webpages with AGILE text notices and creates a JSON with GCN schema keywords. Creates a `agile_jsons` directory inside an `output` directory and saves jsons as `AGILE_{serial_number}.json` where serial_number is a random iterating number with no association to the notices.
+
+### Uses the following fields from the core schema for text notice fields
+- `id` → TRIGGER_NUM
+- `ra` → GRB_RA (Ground Only)
+- `dec` → GRB_DEC (Ground Only)
+- `ra_dec_error` → GRB_ERROR (Ground Only)
+- `alert_datetime` → NOTICE_DATE
+- `trigger_time` → GRB_DATE, GRB_TIME
+- `latitude`, `longitude` → SC_LON_LAT (MCAL Only)
+
+### Defines the following new fields for the text notice fields
+- `n_events` → GRB_TOTAL_COUNTS (MCAL Only)
+- `n_events_x`, `n_events_y` → GRB_INTEN (Ground Only)
+- `snr_x`, `snr_y` → GRB_SIGNIF (Ground Only)
+- `events_snr` → GRB_SIGNIF (MCAL Only)
+- `lightcurve_url` → LIGHT_CURVE (MCAL Only)
+- `peak_events` → PEAK_COUNTS (MCAL Only)
+- `peak_snr` → PEAK_SIGNIF (MCAL Only)
+- `bkg_events` → BACKGROUND (MCAL Only)
+- `triggering_interval` → DATA_TIME_SCALE (MCAL Only)
+- `events_energy_range` → ENERGY_RANGE (MCAL Only)
+- `trigger_logic` → TRIGGER_LOGIC

gcn_classic_text_to_json/notices/agile/__main__.py

@@ -0,0 +1,4 @@
+from . import conversion
+
+if __name__ == "__main__":
+    conversion.create_all_agile_jsons()

gcn_classic_text_to_json/notices/agile/conversion.py

@@ -0,0 +1,157 @@
+import email
+import json
+import os
+
+import requests
+
+from ... import conversion
+
+input_ground = {
+    "standard": {
+        "id": "TRIGGER_NUM",
+        "alert_datetime": "NOTICE_DATE",
+        "trigger_time": ["GRB_DATE", "GRB_TIME"],
+        "ra": "GRB_RA",
+        "dec": "GRB_DEC",
+    },
+    "additional": {
+        "ra_dec_error": ("GRB_ERROR", "float"),
+        "n_events_x": ("GRB_INTEN", "float"),
+        "snr_x": ("GRB_SIGNIF", "float"),
+    },
+}
+
+input_mcal = {
+    "standard": {
+        "id": "TRIGGER_NUM",
+        "alert_datetime": "NOTICE_DATE",
+        "trigger_time": ["GRB_DATE", "GRB_TIME"],
+    },
+    "additional": {
+        "n_events": ("GRB_TOTAL_COUNTS", "int"),
+        "events_snr": ("GRB_SIGNIF", "float"),
+        "peak_events": ("PEAK_COUNTS", "int"),
+        "peak_snr": ("PEAK_SIGNIF", "float"),
+        "bkg_events": ("BACKGROUND", "int"),
+        "triggering_interval": ("DATA_TIME_SCALE", "float"),
+        "duration": ("INTEG_TIME", "float"),
+        "lightcurve_url": ("LIGHT_CURVE", "string"),
+    },
+}
+
+triggering_intervals = ["sub-ms", "1ms", "16ms", "64ms", "256ms", "1024ms", "8192ms"]
+
+
+def text_to_json_agile(notice, input, notice_type):
+    """Function calls text_to_json and then adds additional fields depending on the notice type.
+
+    Parameters
+    -----------
+    notice: dict
+        The text notice that is being parsed.
+    input: dict
+        The mapping between text notices keywords and GCN schema keywords.
+    notice_type: string
+        If the AGILE notice is a Ground or MCAL notice.
+
+    Returns
+    -------
+    dictionary
+        A dictionary compliant with the associated schema for the mission."""
+    output_dict = conversion.text_to_json(notice, input)
+
+    output_dict["$schema"] = (
+        "https://gcn.nasa.gov/schema/main/gcn/notices/classic/agile/alert.schema.json"
+    )
+    output_dict["mission"] = "AGILE"
+    output_dict["notice_type"] = notice_type
+
+    if notice_type == "Ground":
+        output_dict["systematic_included"] = True
+        output_dict["n_events_y"] = float(notice["GRB_INTEN"].split()[1])
+        output_dict["snr_y"] = float(notice["GRB_SIGNIF"].split()[1])
+        output_dict["ra_dec_error"] /= 60
+    elif notice_type == "MCAL":
+        lon_lat_data = notice["SC_LON_LAT"].split(",")
+        output_dict["longitude"] = float(lon_lat_data[0])
+        output_dict["latitude"] = float(lon_lat_data[1].split()[0])
+
+        erange_data = notice["ENERGY_RANGE"].split()
+        output_dict["events_energy_range"] = [int(erange_data[0]), int(erange_data[2])]
+
+        triggering_data = notice["TRIGGER_LOGIC"].split(",")
+        trigger_logic = {}
+        for trigger_interval in triggering_intervals:
+            if trigger_interval in triggering_data:
+                trigger_logic[trigger_interval] = True
+            else:
+                trigger_logic[trigger_interval] = False
+        output_dict["trigger_logic"] = trigger_logic
+
+    return output_dict
+
+
+def create_agile_jsons_one_webpage(link, notice_type, output_path, sernum):
+    """Parses through a single webpage and creates JSONs for all the triggers in that webpage
+
+    Parameters
+    ----------
+    link: string
+        The web address to the table with all the triggers for `notice_type`
+    notice_type: string
+        The particular type of notices stored in `link`
+    output_path: string
+        The path to which to save the JSONs
+    sernum: int
+        A random iterating serial number with no relation to the data to stored the JSONs
+
+    Returns
+    -------
+    sernum: int
+        Increments sernum and returns it for the next function call"""
+    prefix = "https://gcn.gsfc.nasa.gov/"
+    search_string = "other/.*agile"
+    links_set = conversion.parse_trigger_links(link, prefix, search_string)
+    links_list = list(links_set)
+
+    for link in links_list:
+        data = requests.get(link).text
+
+        start_idx = data.find("\n") + 1
+        while True:
+            end_idx = data.find("\n \n ", start_idx)
+            notice_message = email.message_from_string(data[start_idx:end_idx].strip())
+            comment = "\n".join(notice_message.get_all("COMMENTS"))
+            notice_dict = dict(notice_message)
+            notice_dict["COMMENTS"] = comment
+
+            if notice_type == "Ground":
+                output = text_to_json_agile(notice_dict, input_ground, "Ground")
+            elif notice_type == "MCAL":
+                output = text_to_json_agile(notice_dict, input_mcal, "MCAL")
+
+            with open(f"{output_path}AGILE_{sernum}.json", "w") as f:
+                json.dump(output, f)
+
+            sernum += 1
+            temp_start_idx = data.find("///////////", end_idx)
+            start_idx = data.find("\n", temp_start_idx)
+            if temp_start_idx == -1:
+                break
+
+    return sernum
+
+
+def create_all_agile_jsons():
+    """Creates a `agile_jsons` directory inside an `output` directory and fills it with the json for all AGILE triggers."""
+    output_path = "./output/agile_jsons/"
+    if not os.path.exists(output_path):
+        os.makedirs(output_path)
+
+    sernum = 1
+    sernum = create_agile_jsons_one_webpage(
+        "https://gcn.gsfc.nasa.gov/agile_grbs.html", "Ground", output_path, sernum
+    )
+    create_agile_jsons_one_webpage(
+        "https://gcn.gsfc.nasa.gov/agile_mcal.html", "MCAL", output_path, sernum
+    )

gcn_classic_text_to_json/notices/alexis/README.md

@@ -0,0 +1,18 @@
+# ALEXIS text conversion
+
+Parses through the table associated with all ALEXIS triggers and creates a JSON with GCN schema keywords. Creates a `alexis_jsons` directory inside an `output` directory and saves jsons as `ALEXIS_{serial_number}.json` where serial_number is a random iterating number with no association to the notices.
+
+### Uses the following fields from the core schema for text notice fields
+- `alert_datetime` → Date and Time UT
+- `ra` → RA
+- `dec` → Dec
+- `ra_dec_error` → Error
+- `containment_probability` → mentioned in mission description
+- `systematic_included` → mentioned in mission description
+
+### Defines the following new fields for the text notice fields
+- `map_duration` → Dur
+- `notice_type` → Type
+- `alpha` → Alpha
+- `telescope_id` → Tele
+- `energy_bandpass` →  mentioned in mission description

gcn_classic_text_to_json/notices/alexis/__main__.py

@@ -0,0 +1,4 @@
+from . import conversion
+
+if __name__ == "__main__":
+    conversion.create_all_alexis_jsons()