diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 40a1ca46..e6c00f9a 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -19,6 +19,7 @@ jobs: run: | touch env/.env.dev bash run/get_fsps_files.sh + bash run/get_sbi_files.sh bash run/blast.run.sh ci - uses: codecov/codecov-action@v3 with: diff --git a/.readthedocs.yml b/.readthedocs.yml index c4ded850..fec68bd6 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.8" + python: "3.11" apt_packages: - libcurl4-openssl-dev - libssl-dev diff --git a/app/Dockerfile b/app/Dockerfile index 9c26534d..a02c714c 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,16 +1,18 @@ -FROM python:3.8 +FROM python:3.11 ENV PYTHONUNBUFFERED 1 RUN apt-get update -RUN apt-get install ffmpeg libsm6 libxext6 gfortran libhealpix-cxx-dev -y +RUN apt-get install ffmpeg libsm6 libxext6 gfortran libhealpix-cxx-dev libhdf5-serial-dev netcdf-bin libnetcdf-dev -y COPY ./requirements.txt /requirements.txt -RUN pip install -r /requirements.txt +RUN pip install -r /requirements.txt --no-cache-dir COPY ./debug/debug_ipython.py /root/.ipython/profile_default/startup/ RUN mkdir /app -RUN mkdir /cutout_cdn +RUN mkdir /data +RUN mkdir /data/cutout_cdn +RUN mkdir /data/sed_output RUN mkdir /ghost_output RUN mkdir /fsps diff --git a/app/api/serializers.py b/app/api/serializers.py index 02a25ac1..e848fb71 100644 --- a/app/api/serializers.py +++ b/app/api/serializers.py @@ -12,66 +12,73 @@ def to_representation(self, value): class TransientSerializer(serializers.ModelSerializer): class Meta: model = models.Transient - fields = [ - "name", - "ra_deg", - "dec_deg", - "public_timestamp", - "redshift", - "milkyway_dust_reddening", - "spectroscopic_class", + depth = 1 + exclude = [ + "tns_id", + "tns_prefix", + "tasks_initialized", "photometric_class", "processing_status", ] -class HostSerializer(serializers.ModelSerializer): +class HostSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = models.Host + depth = 1 fields = ["name", "ra_deg", "dec_deg", "redshift", "milkyway_dust_reddening"] class ApertureSerializer(serializers.ModelSerializer): - cutout = CutoutField(read_only=True) - class Meta: model = models.Aperture - fields = [ - "ra_deg", - "dec_deg", - "orientation_deg", - "semi_major_axis_arcsec", - "semi_minor_axis_arcsec", - "cutout", - ] + depth = 1 + fields = "__all__" class AperturePhotometrySerializer(serializers.ModelSerializer): class Meta: model = models.AperturePhotometry - fields = ["flux", "flux_error", "magnitude", "magnitude_error", "is_validated"] + depth = 1 + fields = "__all__" class SEDFittingResultSerializer(serializers.ModelSerializer): class Meta: model = models.SEDFittingResult + depth = 1 + exclude = ["log_tau_16", "log_tau_50", "log_tau_84", "posterior"] + + +class CutoutSerializer(serializers.ModelSerializer): + class Meta: + model = models.Cutout + depth = 1 + exclude = ["fits"] + + +class FilterSerializer(serializers.ModelSerializer): + class Meta: + model = models.Filter + depth = 1 fields = [ - "log_mass_16", - "log_mass_50", - "log_mass_84", - "log_sfr_16", - "log_sfr_50", - "log_sfr_84", - "log_ssfr_16", - "log_ssfr_50", - "log_sfr_84", - "log_ssfr_16", - "log_ssfr_50", - "log_ssfr_84", - "log_age_16", - "log_age_50", - "log_age_84", - "log_tau_16", - "log_tau_50", - "log_tau_84", + "name", + "pixel_size_arcsec", + "image_fwhm_arcsec", + "wavelength_eff_angstrom", + "ab_offset", ] + + +class TaskRegisterSerializer(serializers.ModelSerializer): + class Meta: + model = models.TaskRegister + depth = 1 + fields = "__all__" + + +class TaskSerializer(serializers.ModelSerializer): + class Meta: + model = models.Task + depth = 1 + fields = ["name"] diff --git a/app/api/tests/test_api.py b/app/api/tests/test_api.py index 06c995eb..f9fd21dc 100644 --- a/app/api/tests/test_api.py +++ b/app/api/tests/test_api.py @@ -42,7 +42,7 @@ def test_transient_get(self): self.assertTrue(data["global_aperture_dec_deg"] == 10.03586) self.assertTrue(data["global_aperture_semi_major_axis_arcsec"] == 0.4) self.assertTrue(data["global_aperture_semi_minor_axis_arcsec"] == 0.5) - self.assertTrue(data["global_aperture_cutout"] == "2MASS_J") + self.assertTrue(data["global_aperture_cutout"]["name"] == "2022testone_2MASS_J") self.assertTrue(data["transient_name"] == "2022testone") self.assertTrue(data["host_name"] == "PSO J080624.103+010209.859") @@ -59,9 +59,6 @@ def test_transient_get(self): self.assertTrue(data["local_aperture_host_log_age_16"] == 1.0) self.assertTrue(data["local_aperture_host_log_age_50"] == 0.1) self.assertTrue(data["local_aperture_host_log_age_84"] == 5.0) - self.assertTrue(data["local_aperture_host_log_tau_16"] == 1.0) - self.assertTrue(data["local_aperture_host_log_tau_50"] == 5.0) - self.assertTrue(data["local_aperture_host_log_tau_84"] == 60) self.assertTrue(data["global_aperture_host_log_mass_16"] == 1.0) self.assertTrue(data["global_aperture_host_log_mass_50"] == 2.0) @@ -75,9 +72,6 @@ def test_transient_get(self): self.assertTrue(data["global_aperture_host_log_age_16"] == 1.0) self.assertTrue(data["global_aperture_host_log_age_50"] == 0.1) self.assertTrue(data["global_aperture_host_log_age_84"] == 5.0) - self.assertTrue(data["global_aperture_host_log_tau_16"] == 1.0) - self.assertTrue(data["global_aperture_host_log_tau_50"] == 5.0) - self.assertTrue(data["global_aperture_host_log_tau_84"] == 60) self.assertTrue(request.status_code == 200) diff --git a/app/api/tests/test_datamodel.py b/app/api/tests/test_datamodel.py index d49ecfef..dc16aea7 100644 --- a/app/api/tests/test_datamodel.py +++ b/app/api/tests/test_datamodel.py @@ -70,7 +70,7 @@ def test_aperture_build(self): self.assertTrue(data["global_aperture_dec_deg"] == 10.03586) self.assertTrue(data["global_aperture_semi_major_axis_arcsec"] == 0.4) self.assertTrue(data["global_aperture_semi_minor_axis_arcsec"] == 0.5) - self.assertTrue(data["global_aperture_cutout"] == "2MASS_J") + self.assertTrue(data["global_aperture_cutout"]["name"] == "2022testone_2MASS_J") def test_sed_fitting_result_build(self): sed_fit = sed_fit_component("2022testone") @@ -87,9 +87,6 @@ def test_sed_fitting_result_build(self): self.assertTrue(data["local_aperture_host_log_age_16"] == 1.0) self.assertTrue(data["local_aperture_host_log_age_50"] == 0.1) self.assertTrue(data["local_aperture_host_log_age_84"] == 5.0) - self.assertTrue(data["local_aperture_host_log_tau_16"] == 1.0) - self.assertTrue(data["local_aperture_host_log_tau_50"] == 5.0) - self.assertTrue(data["local_aperture_host_log_tau_84"] == 60) self.assertTrue(data["global_aperture_host_log_mass_16"] == 1.0) self.assertTrue(data["global_aperture_host_log_mass_50"] == 2.0) @@ -103,9 +100,6 @@ def test_sed_fitting_result_build(self): self.assertTrue(data["global_aperture_host_log_age_16"] == 1.0) self.assertTrue(data["global_aperture_host_log_age_50"] == 0.1) self.assertTrue(data["global_aperture_host_log_age_84"] == 5.0) - self.assertTrue(data["global_aperture_host_log_tau_16"] == 1.0) - self.assertTrue(data["global_aperture_host_log_tau_50"] == 5.0) - self.assertTrue(data["global_aperture_host_log_tau_84"] == 60) class DataModelComponentTests(TestCase): diff --git a/app/api/views.py b/app/api/views.py index c8182f0a..9e53a9aa 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -1,16 +1,167 @@ import itertools +import django_filters from astropy.coordinates import SkyCoord +from django_filters.rest_framework import DjangoFilterBackend +from host.models import * from host.models import Transient +from rest_framework import generics from rest_framework import status +from rest_framework import viewsets from rest_framework.decorators import api_view from rest_framework.response import Response from rest_framework.views import APIView from . import datamodel +from . import serializers from .components import data_model_components +### Filter Sets ### +class TransientFilter(django_filters.FilterSet): + + redshift_lte = django_filters.NumberFilter(field_name="redshift", lookup_expr="lte") + redshift_gte = django_filters.NumberFilter(field_name="redshift", lookup_expr="gte") + + class Meta: + model = Transient + fields = ("name",) + + +### Filter Sets ### +class HostFilter(django_filters.FilterSet): + + redshift_lte = django_filters.NumberFilter(field_name="redshift", lookup_expr="lte") + redshift_gte = django_filters.NumberFilter(field_name="redshift", lookup_expr="gte") + photometric_redshift_lte = django_filters.NumberFilter( + field_name="photometric_redshift", lookup_expr="lte" + ) + photometric_redshift_gte = django_filters.NumberFilter( + field_name="photometric_redshift", lookup_expr="gte" + ) + + class Meta: + model = Host + fields = ("name",) + + +class ApertureFilter(django_filters.FilterSet): + transient = django_filters.Filter(field_name="transient__name") + + class Meta: + model = Aperture + fields = () + + +class TaskRegisterFilter(django_filters.FilterSet): + + transient = django_filters.Filter(field_name="transient__name") + status = django_filters.Filter(field_name="status__message") + task = django_filters.Filter(field_name="task__name") + + class Meta: + model = TaskRegister + fields = () + + +class FilterFilter(django_filters.FilterSet): + class Meta: + model = Filter + fields = ("name",) + + +class CutoutFilter(django_filters.FilterSet): + + filter = django_filters.Filter(field_name="filter__name") + transient = django_filters.Filter(field_name="transient__name") + + class Meta: + model = Cutout + fields = ("name",) + + +class AperturePhotometryFilter(django_filters.FilterSet): + + filter = django_filters.Filter(field_name="filter__name") + transient = django_filters.Filter(field_name="transient__name") + + class Meta: + model = AperturePhotometry + fields = () + + +class SEDFittingResultFilter(django_filters.FilterSet): + + transient = django_filters.Filter(field_name="transient__name") + aperture_type = django_filters.Filter(field_name="aperture__type") + + class Meta: + model = SEDFittingResult + fields = () + + +### ViewSets ### +class TransientViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Transient.objects.all() + serializer_class = serializers.TransientSerializer + filter_backends = [DjangoFilterBackend] + filterset_class = TransientFilter + + +class ApertureViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Aperture.objects.all() + serializer_class = serializers.ApertureSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = ApertureFilter + + +class CutoutViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Cutout.objects.all() + serializer_class = serializers.CutoutSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = CutoutFilter + + +class FilterViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Filter.objects.all() + serializer_class = serializers.FilterSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = FilterFilter + + +class AperturePhotometryViewSet(viewsets.ReadOnlyModelViewSet): + queryset = AperturePhotometry.objects.all() + serializer_class = serializers.AperturePhotometrySerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = AperturePhotometryFilter + + +class SEDFittingResultViewSet(viewsets.ReadOnlyModelViewSet): + queryset = SEDFittingResult.objects.all() + serializer_class = serializers.SEDFittingResultSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = SEDFittingResultFilter + + +class TaskRegisterViewSet(viewsets.ReadOnlyModelViewSet): + queryset = TaskRegister.objects.all() + serializer_class = serializers.TaskRegisterSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = TaskRegisterFilter + + +class TaskViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Task.objects.all() + serializer_class = serializers.TaskSerializer + + +class HostViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Host.objects.all() + serializer_class = serializers.HostSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = HostFilter + + def transient_exists(transient_name: str) -> bool: """ Checks if a transient exists in the database. diff --git a/app/app/settings.py b/app/app/settings.py index e1d58a5d..25ac2e88 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -39,11 +39,15 @@ "django.contrib.staticfiles", "host", "crispy_forms", + "django_tables2", + "bootstrap3", "django_celery_beat", "revproxy", "rest_framework", "api", "users", + "django_cron", + "django_filters", ] MIDDLEWARE = [ @@ -134,11 +138,14 @@ MEDIA_URL = "/cutouts/" # os.path.join(os.path.dirname(BASE_DIR), '../cutout_cdn') -MEDIA_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../cutout_cdn") -SED_OUTPUT_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../sed_output") +MEDIA_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../data") +CUTOUT_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../data/cutout_cdn") +SED_OUTPUT_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../data/sed_output") GHOST_OUTPUT_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../ghost_output") TNS_STAGING_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../tns_staging") TRANSMISSION_CURVES_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../transmission") +SBIPP_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../sbipp") +SBIPP_PHOT_ROOT = os.path.join(os.path.dirname(BASE_DIR), "../sbipp_phot") CUTOUT_OVERWRITE = os.environ.get("CUTOUT_OVERWRITE", "False") @@ -163,7 +170,8 @@ REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": [ f'rest_framework.permissions.{os.environ.get("API_AUTHENTICATION")}', - ] + ], + "DEFAULT_FILTER_BACKENDS": ["django_filters.rest_framework.DjangoFilterBackend"], } diff --git a/app/entrypoints/docker-entrypoint.celery.sh b/app/entrypoints/docker-entrypoint.celery.sh index 71c8a591..e643c157 100644 --- a/app/entrypoints/docker-entrypoint.celery.sh +++ b/app/entrypoints/docker-entrypoint.celery.sh @@ -1,5 +1,4 @@ #!/bin/env bash - bash entrypoints/wait-for-it.sh ${DATABASE_HOST}:${DATABASE_PORT} --timeout=0 && bash entrypoints/wait-for-it.sh ${MESSAGE_BROKER_HOST}:${MESSAGE_BROKER_PORT} --timeout=0 && bash entrypoints/wait-for-it.sh ${WEB_APP_HOST}:${WEB_APP_PORT} --timeout=0 && diff --git a/app/host/SBI/.gitattributes b/app/host/SBI/.gitattributes new file mode 100644 index 00000000..a2dd5d11 --- /dev/null +++ b/app/host/SBI/.gitattributes @@ -0,0 +1 @@ +sbi_phot.h5 filter=lfs diff=lfs merge=lfs -text diff --git a/app/host/SBI/priors/behroozi_19_sfrd.txt b/app/host/SBI/priors/behroozi_19_sfrd.txt new file mode 100644 index 00000000..1c5b59b4 --- /dev/null +++ b/app/host/SBI/priors/behroozi_19_sfrd.txt @@ -0,0 +1,502 @@ +# redshift; lookback time [Gyr]; sfrd [Msun yr^-1 Mpc-3] from Behroozi+19, interpolated & extrapolated to cover [z=1e-8, z=35] +1.000000000000000021e-08 1.410548492051279652e+02 1.193494661286486139e-02 +5.999999999999999778e-02 8.112741864768534899e+08 1.263755040109705227e-02 +1.199999999999999956e-01 1.557102203868118286e+09 1.392728886840220726e-02 +1.799999999999999933e-01 2.243809848856364250e+09 1.564292478520986490e-02 +2.399999999999999911e-01 2.876964828322405815e+09 1.765207666828923216e-02 +2.999999999999999889e-01 3.461507961372695923e+09 1.987912226239903213e-02 +3.599999999999999867e-01 4.001851241638014793e+09 2.229168332826417590e-02 +4.199999999999999845e-01 4.501952704015921593e+09 2.487733860607543174e-02 +4.799999999999999822e-01 4.965374992987153053e+09 2.762808353790876387e-02 +5.400000000000000355e-01 5.395332399591747284e+09 3.053178182904907681e-02 +5.999999999999999778e-01 5.794729641715249062e+09 3.356716753257296681e-02 +6.599999999999999201e-01 6.166194621432714462e+09 3.670281723682584929e-02 +7.199999999999999734e-01 6.512106674164171219e+09 3.990000385608642036e-02 +7.800000000000000266e-01 6.834621334239222527e+09 4.311587370304586359e-02 +8.399999999999999689e-01 7.135692312382609367e+09 4.630412592952687367e-02 +8.999999999999999112e-01 7.417091163251782417e+09 4.941484668537550473e-02 +9.599999999999999645e-01 7.680424979852748871e+09 5.239639344368429014e-02 +1.020000000000000018e+00 7.927152361108431816e+09 5.519946401746095993e-02 +1.080000000000000071e+00 8.158597841398835182e+09 5.778084590702340512e-02 +1.139999999999999902e+00 8.375964934487644196e+09 6.010388804397823886e-02 +1.199999999999999956e+00 8.580347920869254112e+09 6.213634326265790597e-02 +1.260000000000000009e+00 8.772742492002471924e+09 6.385159305320570189e-02 +1.319999999999999840e+00 8.954055353857219696e+09 6.523609397681361111e-02 +1.379999999999999893e+00 9.125112883687217712e+09 6.629433652623728523e-02 +1.439999999999999947e+00 9.286668926787101746e+09 6.704175675501020115e-02 +1.500000000000000000e+00 9.439411813553792953e+09 6.749302730350920099e-02 +1.560000000000000053e+00 9.583970671124351501e+09 6.766126874407812908e-02 +1.619999999999999885e+00 9.720921098066802979e+09 6.756686493999167209e-02 +1.679999999999999938e+00 9.850790265016170502e+09 6.723965083406582832e-02 +1.739999999999999991e+00 9.974061498784755707e+09 6.670904086256257415e-02 +1.799999999999999822e+00 1.009117840236095619e+10 6.599451046660764764e-02 +1.859999999999999876e+00 1.020254855837451363e+10 6.510746239947770575e-02 +1.919999999999999929e+00 1.030854685907291222e+10 6.406163036397409527e-02 +1.979999999999999982e+00 1.040951850163852310e+10 6.288114810414920097e-02 +2.040000000000000036e+00 1.050578168378620148e+10 6.159972780356806621e-02 +2.100000000000000089e+00 1.059763003101389694e+10 6.025393343347881792e-02 +2.160000000000000142e+00 1.068533478362651634e+10 5.887691584440686576e-02 +2.219999999999999751e+00 1.076914676870197296e+10 5.749418448919954572e-02 +2.279999999999999805e+00 1.084929817950140953e+10 5.611989963751207178e-02 +2.339999999999999858e+00 1.092600418242366409e+10 5.475597497529145763e-02 +2.399999999999999911e+00 1.099946436944738197e+10 5.339832647410142769e-02 +2.459999999999999964e+00 1.106986407207201767e+10 5.204726627727234572e-02 +2.520000000000000018e+00 1.113737555104164696e+10 5.071152499756186038e-02 +2.580000000000000071e+00 1.120215907459350014e+10 4.940223026830799224e-02 +2.639999999999999680e+00 1.126436389659832001e+10 4.812638670601410151e-02 +2.699999999999999734e+00 1.132412914473521423e+10 4.688870857894452543e-02 +2.759999999999999787e+00 1.138158462775387573e+10 4.569695062685616427e-02 +2.819999999999999840e+00 1.143685156990727043e+10 4.456101357372867916e-02 +2.879999999999999893e+00 1.149004327977585220e+10 4.348623017760915033e-02 +2.939999999999999947e+00 1.154126575993733978e+10 4.246815005473969507e-02 +3.000000000000000000e+00 1.159061826325444794e+10 4.149302580404837237e-02 +3.060000000000000053e+00 1.163819380094635010e+10 4.054389880551932668e-02 +3.120000000000000107e+00 1.168407960707031631e+10 3.960929062375941057e-02 +3.179999999999999716e+00 1.172835756355995941e+10 3.868786576253097770e-02 +3.239999999999999769e+00 1.177110458953895950e+10 3.778456524574692194e-02 +3.299999999999999822e+00 1.181239299824878883e+10 3.690329174760384467e-02 +3.359999999999999876e+00 1.185229082458942413e+10 3.604459578679952009e-02 +3.419999999999999929e+00 1.189086212596986389e+10 3.520795224740198154e-02 +3.479999999999999982e+00 1.192816725889507484e+10 3.439197062231078711e-02 +3.540000000000000036e+00 1.196426313347522545e+10 3.359179550747840026e-02 +3.599999999999999645e+00 1.199920344782761955e+10 3.279958620496090105e-02 +3.659999999999999698e+00 1.203303890414915085e+10 3.201062493699869532e-02 +3.719999999999999751e+00 1.206581740806488419e+10 3.122988167389281061e-02 +3.779999999999999805e+00 1.209758425270393372e+10 3.047172747180219160e-02 +3.839999999999999858e+00 1.212838228881555748e+10 2.975111753498640293e-02 +3.899999999999999911e+00 1.215825208211421776e+10 2.907269947701198246e-02 +3.959999999999999964e+00 1.218723205893100166e+10 2.842701820625347098e-02 +4.019999999999999574e+00 1.221535864114865112e+10 2.779685619569291499e-02 +4.080000000000000071e+00 1.224266637130743599e+10 2.716793734476826566e-02 +4.139999999999999680e+00 1.226918802868805504e+10 2.653572073936246584e-02 +4.200000000000000178e+00 1.229495473710471153e+10 2.590481269570508310e-02 +4.259999999999999787e+00 1.231999606507581520e+10 2.528350898716998085e-02 +4.320000000000000284e+00 1.234434011898015022e+10 2.467821397428865068e-02 +4.379999999999999893e+00 1.236801362975295830e+10 2.409086792060666227e-02 +4.439999999999999503e+00 1.239104203362765503e+10 2.351962854697293298e-02 +4.500000000000000000e+00 1.241344954738523293e+10 2.296117944392685184e-02 +4.559999999999999609e+00 1.243525923853353691e+10 2.241281168835723703e-02 +4.620000000000000107e+00 1.245649309080271149e+10 2.187324750310838326e-02 +4.679999999999999716e+00 1.247717206531038475e+10 2.134228017113489612e-02 +4.740000000000000213e+00 1.249731615772062111e+10 2.082007855505046620e-02 +4.799999999999999822e+00 1.251694445169364929e+10 2.030696697715612259e-02 +4.859999999999999432e+00 1.253607516889906883e+10 1.980372822458624157e-02 +4.919999999999999929e+00 1.255472571584270477e+10 1.931178345848718128e-02 +4.979999999999999538e+00 1.257291272773734474e+10 1.883270715582588059e-02 +5.040000000000000036e+00 1.259065210962882233e+10 1.836730175908979704e-02 +5.099999999999999645e+00 1.260795907497223091e+10 1.791503905888963688e-02 +5.160000000000000142e+00 1.262484818183761978e+10 1.747443677857566519e-02 +5.219999999999999751e+00 1.264133336691048813e+10 1.704411794968790222e-02 +5.279999999999999361e+00 1.265742797743949127e+10 1.662371475919680042e-02 +5.339999999999999858e+00 1.267314480127189255e+10 1.621395647147561014e-02 +5.399999999999999467e+00 1.268849609510840797e+10 1.581597982981578984e-02 +5.459999999999999964e+00 1.270349361109316444e+10 1.543045026876788996e-02 +5.519999999999999574e+00 1.271814862185578728e+10 1.505707352102602509e-02 +5.580000000000000071e+00 1.273247194410402489e+10 1.469466155792920753e-02 +5.639999999999999680e+00 1.274647396086375427e+10 1.434153319772437166e-02 +5.700000000000000178e+00 1.276016464245449448e+10 1.399594731536203847e-02 +5.759999999999999787e+00 1.277355356628199387e+10 1.365641464319226200e-02 +5.819999999999999396e+00 1.278664993552399826e+10 1.332188578712643380e-02 +5.879999999999999893e+00 1.279946259677948380e+10 1.299184764600865710e-02 +5.939999999999999503e+00 1.281200005674687004e+10 1.266632782723026325e-02 +6.000000000000000000e+00 1.282427049799201202e+10 1.234580385616953699e-02 +6.059999999999999609e+00 1.283628179386259460e+10 1.203105103270635072e-02 +6.120000000000000107e+00 1.284804152260142326e+10 1.172297867221659858e-02 +6.179999999999999716e+00 1.285955698070780182e+10 1.142247037850033925e-02 +6.240000000000000213e+00 1.287083519559251404e+10 1.113020329483218468e-02 +6.299999999999999822e+00 1.288188293756898880e+10 1.084643685803231457e-02 +6.359999999999999432e+00 1.289270673122037888e+10 1.057084365333326066e-02 +6.419999999999999929e+00 1.290331286617954254e+10 1.030252613415308729e-02 +6.479999999999999538e+00 1.291370740735650444e+10 1.004030849446231437e-02 +6.540000000000000036e+00 1.292389620464570045e+10 9.783201672835108859e-03 +6.599999999999999645e+00 1.293388490214315796e+10 9.530771858008227349e-03 +6.660000000000000142e+00 1.294367894690186310e+10 9.283182881387741503e-03 +6.719999999999999751e+00 1.295328359725171089e+10 9.040917786015519042e-03 +6.779999999999999361e+00 1.296270393070869255e+10 8.804402905537066573e-03 +6.839999999999999858e+00 1.297194485149654579e+10 8.573775931032909248e-03 +6.899999999999999467e+00 1.298101109770246124e+10 8.348881014224153810e-03 +6.959999999999999964e+00 1.298990724808721352e+10 8.129406845482177804e-03 +7.019999999999999574e+00 1.299863772856877136e+10 7.915033474947059250e-03 +7.080000000000000071e+00 1.300720681839722443e+10 7.705510639807696023e-03 +7.139999999999999680e+00 1.301561865603782845e+10 7.500667565779459665e-03 +7.199999999999999289e+00 1.302387724477795410e+10 7.300389797266085626e-03 +7.259999999999999787e+00 1.303198645807269096e+10 7.104591987279772203e-03 +7.319999999999999396e+00 1.303995004464303017e+10 6.913196907976086603e-03 +7.379999999999999893e+00 1.304777163333976173e+10 6.726120766044162494e-03 +7.439999999999999503e+00 1.305545473778530312e+10 6.543263648879402150e-03 +7.500000000000000000e+00 1.306300276080511665e+10 6.364504430810808897e-03 +7.559999999999999609e+00 1.307041899865956688e+10 6.189698330068668999e-03 +7.620000000000000107e+00 1.307770664508652687e+10 6.018674080613312799e-03 +7.679999999999999716e+00 1.308486879516438103e+10 5.851227303561792534e-03 +7.739999999999999325e+00 1.309190844900453568e+10 5.687106137159511979e-03 +7.799999999999999822e+00 1.309882851528210640e+10 5.525986001108241195e-03 +7.859999999999999432e+00 1.310563181461277199e+10 5.367438507569866762e-03 +7.919999999999999929e+00 1.311232108278362465e+10 5.210916116173707271e-03 +7.979999999999999538e+00 1.311889897384501839e+10 5.055787493655348325e-03 +8.039999999999999147e+00 1.312536806307050133e+10 4.901451532539990649e-03 +8.099999999999999645e+00 1.313173084979104996e+10 4.747522655562693220e-03 +8.160000000000000142e+00 1.313798976010986328e+10 4.594026839940900005e-03 +8.219999999999998863e+00 1.314414714950339699e+10 4.441510527861839384e-03 +8.279999999999999361e+00 1.315020530531409454e+10 4.290982257080981979e-03 +8.339999999999999858e+00 1.315616644914003754e+10 4.143680339995093123e-03 +8.400000000000000355e+00 1.316203273912630844e+10 4.000743109537686394e-03 +8.459999999999999076e+00 1.316780627216274071e+10 3.862905261222826167e-03 +8.519999999999999574e+00 1.317348908599243546e+10 3.730335963777898188e-03 +8.580000000000000071e+00 1.317908316123516846e+10 3.602671636480374480e-03 +8.640000000000000568e+00 1.318459042332963181e+10 3.479208055152495545e-03 +8.699999999999999289e+00 1.319001274439824295e+10 3.359157470233696367e-03 +8.759999999999999787e+00 1.319535194503801727e+10 3.241873603098151865e-03 +8.820000000000000284e+00 1.320060979604096031e+10 3.126981014168387848e-03 +8.879999999999999005e+00 1.320578802004700279e+10 3.014390774122154872e-03 +8.939999999999999503e+00 1.321088829313263702e+10 2.904228545459932014e-03 +9.000000000000000000e+00 1.321591224633806801e+10 2.796726202284947720e-03 +9.060000000000000497e+00 1.322086146713560677e+10 2.692123089109841293e-03 +9.119999999999999218e+00 1.322573750084186172e+10 2.590601401183086853e-03 +9.179999999999999716e+00 1.323054185197630882e+10 2.492260968505626006e-03 +9.240000000000000213e+00 1.323527598556838036e+10 2.397125902159511976e-03 +9.299999999999998934e+00 1.323994132841551590e+10 2.305167706305882164e-03 +9.359999999999999432e+00 1.324453927029409409e+10 2.216329209064739000e-03 +9.419999999999999929e+00 1.324907116512546349e+10 2.130540821249023425e-03 +9.480000000000000426e+00 1.325353833209873772e+10 2.047728644099653782e-03 +9.539999999999999147e+00 1.325794205675245476e+10 1.967817654041672561e-03 +9.599999999999999645e+00 1.326228359201661491e+10 1.890733049076106370e-03 +9.660000000000000142e+00 1.326656415921687698e+10 1.816401591068207439e-03 +9.719999999999998863e+00 1.327078494904243279e+10 1.744753810288836825e-03 +9.779999999999999361e+00 1.327494712247913361e+10 1.675727056572033202e-03 +9.839999999999999858e+00 1.327905181170919228e+10 1.609268360752339963e-03 +9.900000000000000355e+00 1.328310012097893143e+10 1.545335438155824234e-03 +9.959999999999999076e+00 1.328709312743581581e+10 1.483894450446082269e-03 +1.001999999999999957e+01 1.329103188193607330e+10 1.424913671758905754e-03 +1.008000000000000007e+01 1.329491740982400513e+10 1.368351995003694744e-03 +1.013999999999999879e+01 1.329875071168421936e+10 1.314141368307733277e-03 +1.019999999999999929e+01 1.330253276406778336e+10 1.262165344708815300e-03 +1.025999999999999979e+01 1.330626452019338608e+10 1.212241290531868281e-03 +1.032000000000000028e+01 1.330994691062447929e+10 1.164115658651676240e-03 +1.037999999999999901e+01 1.331358084392332840e+10 1.117476447608117297e-03 +1.043999999999999950e+01 1.331716720728288460e+10 1.071979958732650158e-03 +1.050000000000000000e+01 1.332070686713735580e+10 1.027288629139238435e-03 +1.055999999999999872e+01 1.332420066975227547e+10 9.831212310277689589e-04 +1.061999999999999922e+01 1.332764944179459763e+10 9.393143553817894387e-04 +1.067999999999999972e+01 1.333105399088527870e+10 8.958802943967625505e-04 +1.074000000000000021e+01 1.333441510613070679e+10 8.530354551323817954e-04 +1.079999999999999893e+01 1.333773355863954544e+10 8.111831008380768638e-04 +1.085999999999999943e+01 1.334101010202125549e+10 7.708586541858741164e-04 +1.091999999999999993e+01 1.334424547286859512e+10 7.326587769262312127e-04 +1.098000000000000043e+01 1.334744039122458076e+10 6.971643593737194850e-04 +1.103999999999999915e+01 1.335059556103432846e+10 6.648507635087838987e-04 +1.109999999999999964e+01 1.335371167058242226e+10 6.359831304943525327e-04 +1.116000000000000014e+01 1.335678939291631317e+10 6.105222989603461235e-04 +1.121999999999999886e+01 1.335982938625628853e+10 5.880875494723673954e-04 +1.127999999999999936e+01 1.336283229439244652e+10 5.680065904031515213e-04 +1.133999999999999986e+01 1.336579874706921005e+10 5.494420729292912161e-04 +1.140000000000000036e+01 1.336872936035776329e+10 5.315542345711893497e-04 +1.145999999999999908e+01 1.337162473701693726e+10 5.136539689679807050e-04 +1.151999999999999957e+01 1.337448546684285355e+10 4.953060105966677939e-04 +1.158000000000000007e+01 1.337731212700778008e+10 4.763554890762823233e-04 +1.163999999999999879e+01 1.338010528238860321e+10 4.568801762925426332e-04 +1.169999999999999929e+01 1.338286548588523483e+10 4.371012924596463052e-04 +1.175999999999999979e+01 1.338559327872930908e+10 4.172928872104028977e-04 +1.182000000000000028e+01 1.338828919078357887e+10 3.977139488109161249e-04 +1.187999999999999901e+01 1.339095374083223343e+10 3.785699201020513395e-04 +1.193999999999999950e+01 1.339358743686249924e+10 3.600015192912063740e-04 +1.200000000000000000e+01 1.339619077633788490e+10 3.420936036755071715e-04 +1.205999999999999872e+01 1.339876424646323204e+10 3.248925732650100257e-04 +1.211999999999999922e+01 1.340130832444192696e+10 3.084213962670065336e-04 +1.217999999999999972e+01 1.340382347772559166e+10 2.926873735268593706e-04 +1.224000000000000021e+01 1.340631016425634575e+10 2.776842145848613746e-04 +1.229999999999999893e+01 1.340876883270209503e+10 2.633926193276340871e-04 +1.235999999999999943e+01 1.341119992268489456e+10 2.497822213046651195e-04 +1.241999999999999993e+01 1.341360386500274849e+10 2.368151117134942784e-04 +1.248000000000000043e+01 1.341598108184498787e+10 2.244497040833691275e-04 +1.253999999999999915e+01 1.341833198700149155e+10 2.126439177657717355e-04 +1.259999999999999964e+01 1.342065698606590080e+10 2.013574037863938239e-04 +1.266000000000000014e+01 1.342295647663306999e+10 1.905528247802618569e-04 +1.271999999999999886e+01 1.342523084849093437e+10 1.801961858001613966e-04 +1.277999999999999936e+01 1.342748048380692101e+10 1.702563914086189535e-04 +1.283999999999999986e+01 1.342970575730918312e+10 1.607045573611266104e-04 +1.290000000000000036e+01 1.343190703646271706e+10 1.515137449096867880e-04 +1.295999999999999908e+01 1.343408468164062309e+10 1.426595220012604988e-04 +1.301999999999999957e+01 1.343623904629054832e+10 1.341212006143664671e-04 +1.308000000000000007e+01 1.343837047709661293e+10 1.258830549672767976e-04 +1.313999999999999879e+01 1.344047931413682747e+10 1.179347081460310851e-04 +1.319999999999999929e+01 1.344256589103623199e+10 1.102703884227927765e-04 +1.325999999999999979e+01 1.344463053511582947e+10 1.028874993214172001e-04 +1.332000000000000028e+01 1.344667356753752327e+10 9.578526072016685605e-05 +1.337999999999999901e+01 1.344869530344507790e+10 8.896390979869307010e-05 +1.343999999999999950e+01 1.345069605210132790e+10 8.242449621068858234e-05 +1.350000000000000000e+01 1.345267611702168655e+10 7.616901844273992492e-05 +1.355999999999999872e+01 1.345463579610410309e+10 7.020058492350673713e-05 +1.361999999999999922e+01 1.345657538175553131e+10 6.452338338015800199e-05 +1.367999999999999972e+01 1.345849516101510811e+10 5.914241828267493260e-05 +1.374000000000000021e+01 1.346039541567400551e+10 5.406308598100606120e-05 +1.379999999999999893e+01 1.346227642239221573e+10 4.929059928824097779e-05 +1.385999999999999943e+01 1.346413845281218529e+10 4.482913488172414430e-05 +1.391999999999999993e+01 1.346598177366959572e+10 4.068059322519703688e-05 +1.397999999999999865e+01 1.346780664690115929e+10 3.684317017347702101e-05 +1.403999999999999915e+01 1.346961332974972916e+10 3.331032158261317048e-05 +1.409999999999999964e+01 1.347140207486660385e+10 3.007069415098803438e-05 +1.416000000000000014e+01 1.347317313041129494e+10 2.710906331037148947e-05 +1.421999999999999886e+01 1.347492674014867592e+10 2.440774515229630356e-05 +1.427999999999999936e+01 1.347666314354370499e+10 2.194787200740227559e-05 +1.433999999999999986e+01 1.347838257585372543e+10 1.971029340055926258e-05 +1.439999999999999858e+01 1.348008526821842766e+10 1.767622854038092685e-05 +1.445999999999999908e+01 1.348177144774757576e+10 1.582785602616415419e-05 +1.451999999999999957e+01 1.348344133760649681e+10 1.414884043924962464e-05 +1.458000000000000007e+01 1.348509515709946251e+10 1.262461690064598725e-05 +1.463999999999999879e+01 1.348673312175099945e+10 1.124228892424685417e-05 +1.469999999999999929e+01 1.348835544338516045e+10 9.990209387942756075e-06 +1.475999999999999979e+01 1.348996233020286369e+10 8.857482263697770533e-06 +1.482000000000000028e+01 1.349155398685730553e+10 7.833602959932892536e-06 +1.487999999999999901e+01 1.349313061452756119e+10 6.908333149360193418e-06 +1.493999999999999950e+01 1.349469241099035072e+10 6.071818305633974359e-06 +1.500000000000000000e+01 1.349623957069007492e+10 5.314909207762506444e-06 +1.505999999999999872e+01 1.349777228480717659e+10 4.629573189704992031e-06 +1.511999999999999922e+01 1.349929074132479668e+10 4.009195449666791525e-06 +1.517999999999999972e+01 1.350079512509390068e+10 3.448593014076372904e-06 +1.524000000000000021e+01 1.350228561789676666e+10 2.943731494183558369e-06 +1.529999999999999893e+01 1.350376239850903893e+10 2.491308092950498339e-06 +1.535999999999999943e+01 1.350522564276022339e+10 2.088389851616506912e-06 +1.541999999999999993e+01 1.350667552359281349e+10 1.732192121367845888e-06 +1.547999999999999865e+01 1.350811221111998367e+10 1.419977410286548879e-06 +1.553999999999999915e+01 1.350953587268195534e+10 1.149018973456085349e-06 +1.559999999999999964e+01 1.351094667290100098e+10 9.165878847566171873e-07 +1.566000000000000014e+01 1.351234477373519325e+10 7.199337791999102696e-07 +1.571999999999999886e+01 1.351373033453090858e+10 5.562350856927268060e-07 +1.577999999999999936e+01 1.351510351207408714e+10 4.225013967974856621e-07 +1.583999999999999986e+01 1.351646446064032555e+10 3.154688518493461078e-07 +1.589999999999999858e+01 1.351781333204379845e+10 2.316110935490744058e-07 +1.595999999999999908e+01 1.351915027568510246e+10 1.673442087921665002e-07 +1.601999999999999957e+01 1.352047543859794235e+10 1.192827613305539493e-07 +1.607999999999999829e+01 1.352178896549480247e+10 8.431323149653649684e-08 +1.614000000000000057e+01 1.352309099881155014e+10 5.951867300131792713e-08 +1.619999999999999929e+01 1.352438167875106430e+10 4.221360416862879103e-08 +1.625999999999999801e+01 1.352566114332584190e+10 3.014503287201166420e-08 +1.632000000000000028e+01 1.352692952839966011e+10 2.164399814836018758e-08 +1.637999999999999901e+01 1.352818696772833633e+10 1.558628892769525017e-08 +1.643999999999999773e+01 1.352943359299951935e+10 1.123717641785626512e-08 +1.650000000000000000e+01 1.353066953387164879e+10 8.104428334147946510e-09 +1.655999999999999872e+01 1.353189491801202011e+10 5.845596465623565170e-09 +1.662000000000000099e+01 1.353310987113402557e+10 4.216336628572736685e-09 +1.667999999999999972e+01 1.353431451703356361e+10 3.041177178409241403e-09 +1.673999999999999844e+01 1.353550897762464714e+10 2.193553182590162067e-09 +1.680000000000000071e+01 1.353669337297425652e+10 1.582175349404762668e-09 +1.685999999999999943e+01 1.353786782133640099e+10 1.141198151078423053e-09 +1.691999999999999815e+01 1.353903243918546104e+10 8.231282458766459810e-10 +1.698000000000000043e+01 1.354018734124880028e+10 5.937094347021979974e-10 +1.703999999999999915e+01 1.354133264053867722e+10 4.282332608802587492e-10 +1.709999999999999787e+01 1.354246844838347626e+10 3.088779039129202694e-10 +1.716000000000000014e+01 1.354359487445824814e+10 2.227887654721807676e-10 +1.721999999999999886e+01 1.354471202681463051e+10 1.606940263186049244e-10 +1.728000000000000114e+01 1.354582001191009521e+10 1.159060693197696355e-10 +1.733999999999999986e+01 1.354691893463660049e+10 8.360122160685352257e-11 +1.739999999999999858e+01 1.354800889834863091e+10 6.030024394042771874e-11 +1.746000000000000085e+01 1.354909000489063263e+10 4.349361587530924666e-11 +1.751999999999999957e+01 1.355016235462390518e+10 3.137125985390392340e-11 +1.757999999999999829e+01 1.355122604645288658e+10 2.262759545314917679e-11 +1.764000000000000057e+01 1.355228117785091972e+10 1.632092808436131781e-11 +1.769999999999999929e+01 1.355332784488546562e+10 1.177202827787968924e-11 +1.775999999999999801e+01 1.355436614224280167e+10 8.490978519045541766e-12 +1.782000000000000028e+01 1.355539616325223541e+10 6.124409023580640863e-12 +1.787999999999999901e+01 1.355641799990973282e+10 4.417439733710719598e-12 +1.793999999999999773e+01 1.355743174290118599e+10 3.186229679603842588e-12 +1.800000000000000000e+01 1.355843748162508202e+10 2.298177266282818355e-12 +1.805999999999999872e+01 1.355943530421480560e+10 1.657639052535550966e-12 +1.812000000000000099e+01 1.356042529756041908e+10 1.195628931155222403e-12 +1.817999999999999972e+01 1.356140754733002853e+10 8.623883099452504153e-13 +1.823999999999999844e+01 1.356238213799073982e+10 6.220271003409443889e-13 +1.830000000000000071e+01 1.356334915282914734e+10 4.486583469378513212e-13 +1.835999999999999943e+01 1.356430867397142029e+10 3.236101966725764075e-13 +1.841999999999999815e+01 1.356526078240303230e+10 2.334149361205799838e-13 +1.848000000000000043e+01 1.356620555798802757e+10 1.683585157834163994e-13 +1.853999999999999915e+01 1.356714307948796654e+10 1.214343448105322840e-13 +1.859999999999999787e+01 1.356807342458043671e+10 8.758867961590679961e-14 +1.866000000000000014e+01 1.356899666987723541e+10 6.317633457674469524e-14 +1.871999999999999886e+01 1.356991289094220161e+10 4.556809473615976280e-14 +1.877999999999999758e+01 1.357082216230866432e+10 3.286754877114976242e-14 +1.883999999999999986e+01 1.357172455749655342e+10 2.370684507392128157e-14 +1.889999999999999858e+01 1.357262014902919579e+10 1.709937383137700258e-14 +1.896000000000000085e+01 1.357350900844977760e+10 1.233350893016222502e-14 +1.901999999999999957e+01 1.357439120633747482e+10 8.895965666956916267e-15 +1.907999999999999829e+01 1.357526681232328033e+10 6.416519872470336471e-15 +1.914000000000000057e+01 1.357613589510552216e+10 4.628134686573105862e-15 +1.919999999999999929e+01 1.357699852246508217e+10 3.338200629434764176e-15 +1.925999999999999801e+01 1.357785476128032684e+10 2.407791517971120893e-15 +1.932000000000000028e+01 1.357870467754173088e+10 1.736702085217484433e-15 +1.937999999999999901e+01 1.357954833636626434e+10 1.252655850926929988e-15 +1.943999999999999773e+01 1.358038580201147270e+10 9.035209286715223100e-16 +1.950000000000000000e+01 1.358121713788927460e+10 6.516954101506599662e-16 +1.955999999999999872e+01 1.358204240657954788e+10 4.700576313554770361e-16 +1.962000000000000099e+01 1.358286166984342194e+10 3.390451633600441046e-16 +1.967999999999999972e+01 1.358367498863631248e+10 2.445479344019154677e-16 +1.973999999999999844e+01 1.358448242312075996e+10 1.763885720343867855e-16 +1.980000000000000071e+01 1.358528403267896271e+10 1.272262978643518515e-16 +1.985999999999999943e+01 1.358607987592512894e+10 9.176632409674046462e-17 +1.991999999999999815e+01 1.358687001071757698e+10 6.618960371861571772e-17 +1.998000000000000043e+01 1.358765449417061234e+10 4.774151829170853101e-17 +2.003999999999999915e+01 1.358843338266616821e+10 3.443520493772805257e-17 +2.009999999999999787e+01 1.358920673186526871e+10 2.483757076718840041e-17 +2.016000000000000014e+01 1.358997459671926689e+10 1.791494845843637213e-17 +2.021999999999999886e+01 1.359073703148082924e+10 1.292177005862489081e-17 +2.027999999999999758e+01 1.359149408971481514e+10 9.320269150389070946e-18 +2.033999999999999986e+01 1.359224582430885315e+10 6.722563289826755013e-18 +2.039999999999999858e+01 1.359299228748380852e+10 4.848878981551700323e-18 +2.046000000000000085e+01 1.359373353080400085e+10 3.497420011398619295e-18 +2.051999999999999957e+01 1.359446960518729210e+10 2.522633949552014552e-18 +2.057999999999999829e+01 1.359520056091493034e+10 1.819536121681753777e-18 +2.064000000000000057e+01 1.359592644764126015e+10 1.312402736311630732e-18 +2.069999999999999929e+01 1.359664731440326881e+10 9.466154157391951880e-19 +2.075999999999999801e+01 1.359736320962989998e+10 6.827787846842121063e-19 +2.082000000000000028e+01 1.359807418115127945e+10 4.924775796629219585e-19 +2.087999999999999901e+01 1.359878027620771027e+10 3.552163188298574354e-19 +2.093999999999999773e+01 1.359948154145854187e+10 2.562119340527101809e-19 +2.100000000000000000e+01 1.360017802299090195e+10 1.848016312067941439e-19 +2.105999999999999872e+01 1.360086976632820129e+10 1.332945048908845373e-19 +2.111999999999999744e+01 1.360155681643858147e+10 9.614322621548828850e-20 +2.117999999999999972e+01 1.360223921774284744e+10 6.934659425524958601e-20 +2.123999999999999844e+01 1.360291701412376785e+10 5.001860582485331897e-20 +2.130000000000000071e+01 1.360359024893227196e+10 3.607763229803683886e-20 +2.135999999999999943e+01 1.360425896499644852e+10 2.602222774441289163e-20 +2.141999999999999815e+01 1.360492320462895584e+10 1.876942287088334614e-20 +2.148000000000000043e+01 1.360558300963458443e+10 1.353808898939015730e-20 +2.153999999999999915e+01 1.360623842131765938e+10 9.764810284548794364e-21 +2.159999999999999787e+01 1.360688948048938751e+10 7.043203805792472447e-21 +2.166000000000000014e+01 1.360753622747499466e+10 5.080151933768135173e-21 +2.171999999999999886e+01 1.360817870212079048e+10 3.664233547940660042e-21 +2.177999999999999758e+01 1.360881694380109596e+10 2.642953925178135834e-21 +2.183999999999999986e+01 1.360945099142506790e+10 1.906321024362728208e-21 +2.189999999999999858e+01 1.361008088344339180e+10 1.374999319249360573e-21 +2.196000000000000085e+01 1.361070665785485649e+10 9.917653447525869245e-22 +2.201999999999999957e+01 1.361132835221284866e+10 7.153447171080665149e-22 +2.207999999999999829e+01 1.361194600363168526e+10 5.159668736177476920e-22 +2.214000000000000057e+01 1.361255964879289818e+10 3.721587764667239566e-22 +2.219999999999999929e+01 1.361316932395136261e+10 2.684322618041131796e-22 +2.225999999999999801e+01 1.361377506494137192e+10 1.936159610727727875e-22 +2.232000000000000028e+01 1.361437690718255424e+10 1.396521421463471392e-22 +2.237999999999999901e+01 1.361497488568574905e+10 1.007288897981568019e-22 +2.243999999999999773e+01 1.361556903505873489e+10 7.265416114660445399e-23 +2.250000000000000000e+01 1.361615938951189804e+10 5.240430171020666673e-23 +2.255999999999999872e+01 1.361674598286378098e+10 3.779839715158164325e-23 +2.261999999999999744e+01 1.361732884854658890e+10 2.726338832123827143e-23 +2.267999999999999972e+01 1.361790801961150742e+10 1.966465243946273195e-23 +2.273999999999999844e+01 1.361848352873404884e+10 1.418380397214339742e-23 +2.280000000000000071e+01 1.361905540821925163e+10 1.023055432784894741e-23 +2.285999999999999943e+01 1.361962369000676155e+10 7.379137646052253306e-24 +2.291999999999999815e+01 1.362018840567590904e+10 5.322455719839231421e-24 +2.298000000000000043e+01 1.362074958645065117e+10 3.839003451142392758e-24 +2.303999999999999915e+01 1.362130726320443344e+10 2.769012702716930008e-24 +2.309999999999999787e+01 1.362186146646500778e+10 1.997245234443871146e-24 +2.316000000000000014e+01 1.362241222641914368e+10 1.440581519396712785e-24 +2.321999999999999886e+01 1.362295957291724777e+10 1.039068752418476762e-24 +2.327999999999999758e+01 1.362350353547797585e+10 7.494639197541778783e-25 +2.333999999999999986e+01 1.362404414329268456e+10 5.405765169108633905e-25 +2.339999999999999858e+01 1.362458142522988319e+10 3.899093244292922370e-25 +2.346000000000000085e+01 1.362511540983956909e+10 2.812354523753302709e-25 +2.351999999999999957e+01 1.362564612535753441e+10 2.028507007072096556e-25 +2.357999999999999829e+01 1.362617359970954704e+10 1.463130143439037751e-25 +2.364000000000000057e+01 1.362669786051553345e+10 1.055332719668467941e-25 +2.369999999999999929e+01 1.362721893509363174e+10 7.611948630796898122e-26 +2.375999999999999801e+01 1.362773685046424484e+10 5.490378615010928338e-26 +2.382000000000000028e+01 1.362825163335396194e+10 3.960123589669248916e-26 +2.387999999999999901e+01 1.362876331019948196e+10 2.856374750290994642e-26 +2.393999999999999773e+01 1.362927190715142250e+10 2.060258102899604825e-26 +2.400000000000000000e+01 1.362977745007811546e+10 1.486031708595328187e-26 +2.405999999999999872e+01 1.363027996456931114e+10 1.071851257783092622e-26 +2.411999999999999744e+01 1.363077947593981743e+10 7.731094243588937780e-27 +2.417999999999999972e+01 1.363127600923313904e+10 5.576316468282713278e-27 +2.423999999999999844e+01 1.363176958922499657e+10 4.022109209214074613e-27 +2.430000000000000071e+01 1.363226024042682266e+10 2.901084001035301802e-27 +2.435999999999999943e+01 1.363274798708919525e+10 2.092506181031208238e-27 +2.441999999999999815e+01 1.363323285320523071e+10 1.509291739257202147e-27 +2.448000000000000043e+01 1.363371486251390076e+10 1.088628351418980264e-27 +2.453999999999999915e+01 1.363419403850334358e+10 7.852104776618332066e-28 +2.459999999999999787e+01 1.363467040441405487e+10 5.663599459138377223e-28 +2.466000000000000014e+01 1.363514398324210739e+10 4.085065055304427229e-28 +2.471999999999999886e+01 1.363561479774225616e+10 2.946493060900201161e-28 +2.477999999999999758e+01 1.363608287043105698e+10 2.125259020455441391e-28 +2.483999999999999986e+01 1.363654822358987045e+10 1.532915846286530868e-28 +2.489999999999999858e+01 1.363701087926787186e+10 1.105668047602396849e-28 +2.496000000000000085e+01 1.363747085928502846e+10 7.975009420447908139e-29 +2.501999999999999957e+01 1.363792818523496056e+10 5.752248642270975368e-29 +2.507999999999999829e+01 1.363838287848782539e+10 4.149006314358675169e-29 +2.514000000000000057e+01 1.363883496019313049e+10 2.992612883609984751e-29 +2.519999999999999929e+01 1.363928445128253555e+10 2.158524521921048785e-29 +2.525999999999999801e+01 1.363973137247253799e+10 1.556909728368899961e-29 +2.532000000000000028e+01 1.364017574426722145e+10 1.122974456705469373e-29 +2.537999999999999901e+01 1.364061758696088600e+10 8.099837822544203108e-30 +2.543999999999999773e+01 1.364105692064065742e+10 5.842285401931001961e-30 +2.550000000000000000e+01 1.364149376518908501e+10 4.213948410499771529e-30 +2.555999999999999872e+01 1.364192814028666878e+10 3.039454594341514967e-30 +2.561999999999999744e+01 1.364236006541437531e+10 2.192310709842810236e-30 +2.567999999999999972e+01 1.364278955985608101e+10 1.581279173388255402e-30 +2.573999999999999844e+01 1.364321664270103073e+10 1.140551753437689721e-30 +2.580000000000000071e+01 1.364364133284621620e+10 8.226620094428992257e-31 +2.585999999999999943e+01 1.364406364899874687e+10 5.933731457084663057e-31 +2.591999999999999815e+01 1.364448360967815971e+10 4.279907009275837110e-31 +2.598000000000000043e+01 1.364490123321872902e+10 3.087029492407832022e-31 +2.603999999999999915e+01 1.364531653777171707e+10 2.226625734237202176e-31 +2.609999999999999787e+01 1.364572954130761147e+10 1.606030059823079108e-31 +2.616000000000000014e+01 1.364614026161829948e+10 1.158404177852975116e-31 +2.621999999999999886e+01 1.364654871631926537e+10 8.355386818943173663e-32 +2.627999999999999758e+01 1.364695492285168076e+10 6.026608866653271042e-32 +2.633999999999999986e+01 1.364735889848455620e+10 4.346898021439268740e-32 +2.639999999999999858e+01 1.364776066031676865e+10 3.135349053983950788e-32 +2.645999999999999730e+01 1.364816022527912903e+10 2.261477872688437620e-32 +2.651999999999999957e+01 1.364855761013639069e+10 1.631168358164435522e-32 +2.657999999999999829e+01 1.364895283148924637e+10 1.176536036372451625e-32 +2.664000000000000057e+01 1.364934590577626228e+10 8.486169057623755432e-33 +2.669999999999999929e+01 1.364973684927583504e+10 6.120940034834029461e-33 +2.675999999999999801e+01 1.365012567810806274e+10 4.414937606784490562e-33 +2.682000000000000028e+01 1.365051240823666573e+10 3.184424934874976467e-33 +2.687999999999999901e+01 1.365089705547079086e+10 2.296875532345095268e-33 +2.693999999999999773e+01 1.365127963546686745e+10 1.656700132356141242e-33 +2.700000000000000000e+01 1.365166016373039818e+10 1.194951702823261528e-33 +2.705999999999999872e+01 1.365203865561772346e+10 8.618998358196870286e-34 +2.711999999999999744e+01 1.365241512633777809e+10 6.216747716504798577e-34 +2.717999999999999972e+01 1.365278959095383453e+10 4.484042178046415940e-34 +2.723999999999999844e+01 1.365316206438515663e+10 3.234268973328009148e-34 +2.730000000000000071e+01 1.365353256140873718e+10 2.332827251948281194e-34 +2.735999999999999943e+01 1.365390109666090775e+10 1.682631541257607697e-34 +2.741999999999999815e+01 1.365426768463899231e+10 1.213655619493648402e-34 +2.748000000000000043e+01 1.365463233970290947e+10 8.753906762187612987e-35 +2.753999999999999915e+01 1.365499507607676506e+10 6.314055022712733809e-35 +2.759999999999999787e+01 1.365535590785042191e+10 4.554228404859207703e-35 +2.766000000000000014e+01 1.365571484898102379e+10 3.284893192887540626e-35 +2.771999999999999886e+01 1.365607191329454994e+10 2.369341703891217321e-35 +2.777999999999999758e+01 1.365642711448730278e+10 1.708968840129445931e-35 +2.783999999999999986e+01 1.365678046612738228e+10 1.232652298204544498e-35 +2.789999999999999858e+01 1.365713198165616417e+10 8.890926812649521737e-36 +2.795999999999999730e+01 1.365748167438974380e+10 6.412885426249615844e-36 +2.801999999999999957e+01 1.365782955752035141e+10 4.625513217777724002e-36 +2.807999999999999829e+01 1.365817564411774826e+10 3.336309805296004732e-36 +2.814000000000000057e+01 1.365851994713064384e+10 2.406427696311290107e-36 +2.819999999999999929e+01 1.365886247938803673e+10 1.735718382142356856e-36 +2.825999999999999801e+01 1.365920325360056305e+10 1.251946321397874255e-36 +2.832000000000000028e+01 1.365954228236184883e+10 9.030091562014229840e-37 +2.837999999999999901e+01 1.365987957814980507e+10 6.513262767313648598e-37 +2.843999999999999773e+01 1.366021515332793427e+10 4.697913812361339239e-37 +2.850000000000000000e+01 1.366054902014660072e+10 3.388531213439472885e-37 +2.855999999999999872e+01 1.366088119074430847e+10 2.444094175214817946e-37 +2.861999999999999744e+01 1.366121167714892960e+10 1.762886619909752767e-37 +2.867999999999999972e+01 1.366154049127894020e+10 1.271542343242025365e-37 +2.873999999999999844e+01 1.366186764494464111e+10 9.171434580064998155e-38 +2.879999999999999716e+01 1.366219314984933853e+10 6.615211259260558419e-38 +2.885999999999999943e+01 1.366251701759055328e+10 4.771447653322027051e-38 +2.891999999999999815e+01 1.366283925966116142e+10 3.441570014339509836e-38 +2.898000000000000043e+01 1.366315988745054436e+10 2.482350226634973043e-38 +2.903999999999999915e+01 1.366347891224575424e+10 1.790480107044196438e-38 +2.909999999999999787e+01 1.366379634523260689e+10 1.291445090754478075e-38 +2.916000000000000014e+01 1.366411219749678993e+10 9.314989962034102870e-39 +2.921999999999999886e+01 1.366442648002498817e+10 6.718755494444185120e-39 +2.927999999999999758e+01 1.366473920370591927e+10 4.846132478737141304e-39 +2.933999999999999986e+01 1.366505037933142281e+10 3.495439002191849976e-39 +2.939999999999999858e+01 1.366536001759751320e+10 2.521205078823572049e-39 +2.945999999999999730e+01 1.366566812910540009e+10 1.818505499738340086e-39 +2.951999999999999957e+01 1.366597472436254120e+10 1.311659364942125880e-39 +2.957999999999999829e+01 1.366627981378361320e+10 9.460806814531577879e-40 +2.964000000000000057e+01 1.366658340769155884e+10 6.824059848999257191e-40 +2.969999999999999929e+01 1.366688551631853294e+10 4.922839479801067247e-40 +2.975999999999999801e+01 1.366718614980689049e+10 3.554185651420573788e-40 +2.982000000000000028e+01 1.366748531821015930e+10 2.576055330849668747e-40 +2.987999999999999901e+01 1.366778303149396515e+10 1.895218626372867002e-40 +2.993999999999999773e+01 1.366807929953698730e+10 1.458684854258055728e-40 +3.000000000000000000e+01 1.366837413213188171e+10 1.243514360975379915e-40 diff --git a/app/host/SBI/priors/gallazzi_05_massmet.txt b/app/host/SBI/priors/gallazzi_05_massmet.txt new file mode 100644 index 00000000..dbe9b141 --- /dev/null +++ b/app/host/SBI/priors/gallazzi_05_massmet.txt @@ -0,0 +1,18 @@ +# mass P50 P16 P84 +# corrected to Chabrier IMF +8.87 -0.60 -1.11 -0.00 +9.07 -0.61 -1.07 -0.00 +9.27 -0.65 -1.10 -0.05 +9.47 -0.61 -1.03 -0.01 +9.68 -0.52 -0.97 0.05 +9.87 -0.41 -0.90 0.09 +10.07 -0.23 -0.80 0.14 +10.27 -0.11 -0.65 0.17 +10.47 -0.01 -0.41 0.20 +10.68 0.04 -0.24 0.22 +10.87 0.07 -0.14 0.24 +11.07 0.10 -0.09 0.25 +11.27 0.12 -0.06 0.26 +11.47 0.13 -0.04 0.28 +11.68 0.14 -0.03 0.29 +11.87 0.15 -0.03 0.30 diff --git a/app/host/SBI/priors/mc_from_mocks.txt b/app/host/SBI/priors/mc_from_mocks.txt new file mode 100644 index 00000000..cf0d755f --- /dev/null +++ b/app/host/SBI/priors/mc_from_mocks.txt @@ -0,0 +1,19 @@ +0.75 6.994456230590262 +1.25 7.057507361043797 +1.75 7.218187794744033 +2.25 7.338137827796697 +2.75 7.380600596059972 +3.25 7.328217902820565 +3.75 7.426101298033281 +4.25 7.553686053384422 +4.75 7.574571335405563 +5.25 7.340189472315103 +5.75 7.367732376803446 +6.25 7.339592209403573 +6.75 7.311604282765306 +7.25 7.085854719192774 +7.75 7.1152642491584075 +8.25 7.194151423382382 +8.75 7.109542286651048 +9.25 7.396987624854887 +10.75 7.673548473737973 diff --git a/app/host/SBI/priors/pdf_of_z_l20.txt b/app/host/SBI/priors/pdf_of_z_l20.txt new file mode 100644 index 00000000..5890dd63 --- /dev/null +++ b/app/host/SBI/priors/pdf_of_z_l20.txt @@ -0,0 +1,502 @@ +# z, probability of observing a galaxy at z +0.000000000000000000e+00 0.000000000000000000e+00 +4.000000000000000083e-02 1.777612496205435060e-03 +8.000000000000000167e-02 6.819929832504362070e-03 +1.199999999999999956e-01 1.468812660822723347e-02 +1.600000000000000033e-01 2.494913290884148704e-02 +2.000000000000000111e-01 3.718618177444110717e-02 +2.399999999999999911e-01 4.964067238099263008e-02 +2.800000000000000266e-01 6.257175036727403994e-02 +3.200000000000000067e-01 7.562205410261779726e-02 +3.599999999999999867e-01 8.850415735347777479e-02 +4.000000000000000222e-01 1.009941269387400059e-01 +4.400000000000000022e-01 1.129231266927637062e-01 +4.799999999999999822e-01 1.241649395651596710e-01 +5.200000000000000178e-01 1.346313858186587720e-01 +5.600000000000000533e-01 1.442672786283898667e-01 +5.999999999999999778e-01 1.530448122584609494e-01 +6.400000000000000133e-01 1.609561860790680865e-01 +6.800000000000000488e-01 1.680013271415999043e-01 +7.199999999999999734e-01 1.741898641923088353e-01 +7.600000000000000089e-01 1.795405530653554071e-01 +8.000000000000000444e-01 1.840790608871810119e-01 +8.399999999999999689e-01 1.878329107701774692e-01 +8.800000000000000044e-01 1.908198860588189438e-01 +9.200000000000000400e-01 1.930560986804207013e-01 +9.599999999999999645e-01 1.945588207772757094e-01 +1.000000000000000000e+00 1.953463799924812505e-01 +1.040000000000000036e+00 1.954462311927627205e-01 +1.080000000000000071e+00 1.949190871878553610e-01 +1.120000000000000107e+00 1.938339238125476038e-01 +1.159999999999999920e+00 1.922587542131066196e-01 +1.199999999999999956e+00 1.902596608125621813e-01 +1.239999999999999991e+00 1.879004372965209502e-01 +1.280000000000000027e+00 1.852431709412659944e-01 +1.320000000000000062e+00 1.823465698855508965e-01 +1.360000000000000098e+00 1.792652840026994765e-01 +1.400000000000000133e+00 1.760497660778165552e-01 +1.439999999999999947e+00 1.727432528308013859e-01 +1.479999999999999982e+00 1.693733014851502572e-01 +1.520000000000000018e+00 1.659617110011370467e-01 +1.560000000000000053e+00 1.625278254687242407e-01 +1.600000000000000089e+00 1.590887307761662262e-01 +1.640000000000000124e+00 1.556591937665695313e-01 +1.679999999999999938e+00 1.522511339222427929e-01 +1.719999999999999973e+00 1.488746204556722352e-01 +1.760000000000000009e+00 1.455382917747538307e-01 +1.800000000000000044e+00 1.422495015279024999e-01 +1.840000000000000080e+00 1.390144300743142158e-01 +1.880000000000000115e+00 1.358381429949551966e-01 +1.919999999999999929e+00 1.327247790930241589e-01 +1.959999999999999964e+00 1.296776761664239497e-01 +2.000000000000000000e+00 1.266994633897017553e-01 +2.040000000000000036e+00 1.237921479646512113e-01 +2.080000000000000071e+00 1.209571992159637760e-01 +2.120000000000000107e+00 1.181956065188857347e-01 +2.160000000000000142e+00 1.155079373372622370e-01 +2.200000000000000178e+00 1.128943921808837625e-01 +2.240000000000000213e+00 1.103548531049912684e-01 +2.280000000000000249e+00 1.078889256611700598e-01 +2.319999999999999840e+00 1.054959805944311324e-01 +2.359999999999999876e+00 1.031751895846135803e-01 +2.399999999999999911e+00 1.009255563449312637e-01 +2.439999999999999947e+00 9.874594439581065031e-02 +2.479999999999999982e+00 9.663510200592194399e-02 +2.520000000000000018e+00 9.459168321594917106e-02 +2.560000000000000053e+00 9.261426670814359996e-02 +2.600000000000000089e+00 9.070137256703013351e-02 +2.640000000000000124e+00 8.885147694195544288e-02 +2.680000000000000160e+00 8.706302479054214838e-02 +2.720000000000000195e+00 8.533444123632964728e-02 +2.760000000000000231e+00 8.366414136043960614e-02 +2.800000000000000266e+00 8.205053863780784051e-02 +2.839999999999999858e+00 8.049205221594017690e-02 +2.879999999999999893e+00 7.898711317403298016e-02 +2.919999999999999929e+00 7.753416979605923243e-02 +2.959999999999999964e+00 7.613169204856379269e-02 +3.000000000000000000e+00 7.477817534444325398e-02 +3.040000000000000036e+00 7.436907342945554833e-02 +3.080000000000000071e+00 7.396920420707196742e-02 +3.120000000000000107e+00 7.357865150302082824e-02 +3.160000000000000142e+00 7.319747573124960049e-02 +3.200000000000000178e+00 7.282571596653805712e-02 +3.240000000000000213e+00 7.246339184845851378e-02 +3.280000000000000249e+00 7.211050533028207088e-02 +3.320000000000000284e+00 7.176704227946142933e-02 +3.359999999999999876e+00 7.143297394626194063e-02 +3.399999999999999911e+00 7.110825831014257192e-02 +3.439999999999999947e+00 7.079284131257880530e-02 +3.479999999999999982e+00 7.048665798499050006e-02 +3.520000000000000018e+00 7.018963348135465186e-02 +3.560000000000000053e+00 6.990168402162802441e-02 +3.600000000000000089e+00 6.962271775315621858e-02 +3.640000000000000124e+00 6.935263553673955139e-02 +3.680000000000000160e+00 6.909133166336048348e-02 +3.720000000000000195e+00 6.883869450671287193e-02 +3.760000000000000231e+00 6.859460711698610325e-02 +3.800000000000000266e+00 6.835894776055069511e-02 +3.839999999999999858e+00 6.813159040982227910e-02 +3.879999999999999893e+00 6.791240518728763886e-02 +3.919999999999999929e+00 6.770125876749073790e-02 +3.959999999999999964e+00 6.749801474027986026e-02 +4.000000000000000000e+00 6.730253393848330268e-02 +4.040000000000000036e+00 6.711467473293163122e-02 +4.080000000000000071e+00 6.693429329750788526e-02 +4.120000000000000107e+00 6.676124384668087608e-02 +4.160000000000000142e+00 6.659537884783257866e-02 +4.200000000000000178e+00 6.643654921048589013e-02 +4.240000000000000213e+00 6.628460445438237592e-02 +4.280000000000000249e+00 6.613939285820878700e-02 +4.320000000000000284e+00 6.600076159064682813e-02 +4.360000000000000320e+00 6.586855682527718858e-02 +4.400000000000000355e+00 6.574262384076423604e-02 +4.440000000000000391e+00 6.562280710763690461e-02 +4.480000000000000426e+00 6.550895036288870155e-02 +4.520000000000000462e+00 6.540089667351889346e-02 +4.560000000000000497e+00 6.529848849006633649e-02 +4.600000000000000533e+00 6.520156769109998474e-02 +4.639999999999999680e+00 6.510997561956781421e-02 +4.679999999999999716e+00 6.502355311183552478e-02 +4.719999999999999751e+00 6.494214052018845718e-02 +4.759999999999999787e+00 6.486557772951886958e-02 +4.799999999999999822e+00 6.479370416886179318e-02 +4.839999999999999858e+00 6.472635881840665961e-02 +4.879999999999999893e+00 6.466338021256080870e-02 +4.919999999999999929e+00 6.460460643961292437e-02 +4.959999999999999964e+00 6.454987513848024772e-02 +5.000000000000000000e+00 6.449902349303948679e-02 +5.040000000000000036e+00 6.445188822446093480e-02 +5.080000000000000071e+00 6.440830558196468381e-02 +5.120000000000000107e+00 6.436811133238652671e-02 +5.160000000000000142e+00 6.433114074891518874e-02 +5.200000000000000178e+00 6.429722859934382251e-02 +5.240000000000000213e+00 6.426620913415137526e-02 +5.280000000000000249e+00 6.423791607472306708e-02 +5.320000000000000284e+00 6.421218260198846561e-02 +5.360000000000000320e+00 6.418884134575006395e-02 +5.400000000000000355e+00 6.416772437495438242e-02 +5.440000000000000391e+00 6.414866318914738685e-02 +5.480000000000000426e+00 6.413148871134051465e-02 +5.520000000000000462e+00 6.411603128250473183e-02 +5.560000000000000497e+00 6.410212065789611113e-02 +5.600000000000000533e+00 6.408958600540849704e-02 +5.639999999999999680e+00 6.407825590613612510e-02 +5.679999999999999716e+00 6.406795835732431721e-02 +5.719999999999999751e+00 6.405852077787155263e-02 +5.759999999999999787e+00 6.404977001654491020e-02 +5.799999999999999822e+00 6.404153236305672181e-02 +5.839999999999999858e+00 6.403363356214475388e-02 +5.879999999999999893e+00 6.402589883079580491e-02 +5.919999999999999929e+00 6.401815287873399707e-02 +5.959999999999999964e+00 6.401021993230138207e-02 +6.000000000000000000e+00 6.400192376184016263e-02 +6.040000000000000036e+00 6.399308771268920337e-02 +6.080000000000000071e+00 6.398353473988969975e-02 +6.120000000000000107e+00 6.397308744669835678e-02 +6.160000000000000142e+00 6.396156812699348160e-02 +6.200000000000000178e+00 6.394879881165670132e-02 +6.240000000000000213e+00 6.393460131899821042e-02 +6.280000000000000249e+00 6.391879730930020986e-02 +6.320000000000000284e+00 6.390120834353046897e-02 +6.360000000000000320e+00 6.388165594628357480e-02 +6.400000000000000355e+00 6.385996167299280712e-02 +6.440000000000000391e+00 6.383594718144844360e-02 +6.480000000000000426e+00 6.380943430765634306e-02 +6.520000000000000462e+00 6.378024514605480644e-02 +6.560000000000000497e+00 6.374820213410670178e-02 +6.600000000000000533e+00 6.371312814126857405e-02 +6.640000000000000568e+00 6.367484656233661500e-02 +6.679999999999999716e+00 6.363318141515957038e-02 +6.719999999999999751e+00 6.358795744268533334e-02 +6.759999999999999787e+00 6.353900021932563924e-02 +6.799999999999999822e+00 6.348613626159151091e-02 +6.839999999999999858e+00 6.342919314294885591e-02 +6.879999999999999893e+00 6.336799961283279270e-02 +6.919999999999999929e+00 6.330238571977282735e-02 +6.959999999999999964e+00 6.323218293852850280e-02 +7.000000000000000000e+00 6.315722430114145691e-02 +7.040000000000000036e+00 6.307734453182596568e-02 +7.080000000000000071e+00 6.299238018558905861e-02 +7.120000000000000107e+00 6.290216979036615530e-02 +7.160000000000000142e+00 6.280655399263071481e-02 +7.200000000000000178e+00 6.270537570632345181e-02 +7.240000000000000213e+00 6.259848026487918604e-02 +7.280000000000000249e+00 6.248571557614676641e-02 +7.320000000000000284e+00 6.236693228037126085e-02 +7.360000000000000320e+00 6.224198391069019759e-02 +7.400000000000000355e+00 6.211072705598373600e-02 +7.440000000000000391e+00 6.197302152606943421e-02 +7.480000000000000426e+00 6.182873051918191115e-02 +7.520000000000000462e+00 6.167772079016472370e-02 +7.560000000000000497e+00 6.151986282046268545e-02 +7.600000000000000533e+00 6.135503098947635292e-02 +7.640000000000000568e+00 6.118310374623099623e-02 +7.679999999999999716e+00 6.100396378047785034e-02 +7.719999999999999751e+00 6.081749819781104988e-02 +7.759999999999999787e+00 6.062359869358240061e-02 +7.799999999999999822e+00 6.042216172591836215e-02 +7.839999999999999858e+00 6.021308869037101524e-02 +7.879999999999999893e+00 5.999628609804526919e-02 +7.919999999999999929e+00 5.977166573892572082e-02 +7.959999999999999964e+00 5.953914484813559865e-02 +8.000000000000000000e+00 5.929864627244545161e-02 +8.040000000000000924e+00 5.905009862620445166e-02 +8.080000000000000071e+00 5.879343642839623962e-02 +8.120000000000000995e+00 5.852860028603684012e-02 +8.160000000000000142e+00 5.825553705725219217e-02 +8.199999999999999289e+00 5.797420000217470837e-02 +8.240000000000000213e+00 5.768454895945052813e-02 +8.279999999999999361e+00 5.738655057665211640e-02 +8.320000000000000284e+00 5.708017835505622278e-02 +8.359999999999999432e+00 5.676541275046894514e-02 +8.400000000000000355e+00 5.644224129835831438e-02 +8.439999999999999503e+00 5.611065862405770210e-02 +8.480000000000000426e+00 5.577066623359194092e-02 +8.519999999999999574e+00 5.542227295736240622e-02 +8.560000000000000497e+00 5.506549515207048023e-02 +8.599999999999999645e+00 5.470035679778628007e-02 +8.640000000000000568e+00 5.432688996972630247e-02 +8.679999999999999716e+00 5.394513606875004502e-02 +8.720000000000000639e+00 5.355514466846747479e-02 +8.759999999999999787e+00 5.315697316927912169e-02 +8.800000000000000711e+00 5.275068684069403324e-02 +8.839999999999999858e+00 5.233635744132923706e-02 +8.880000000000000782e+00 5.191405909376288486e-02 +8.919999999999999929e+00 5.148387269594375398e-02 +8.960000000000000853e+00 5.104588736831001949e-02 +9.000000000000000000e+00 5.060020045531358340e-02 +9.040000000000000924e+00 5.014692256911601476e-02 +9.080000000000000071e+00 4.968619236377493781e-02 +9.120000000000000995e+00 4.921816068088195190e-02 +9.160000000000000142e+00 4.874298531787589484e-02 +9.200000000000001066e+00 4.826083092850638051e-02 +9.240000000000000213e+00 4.777183957546197318e-02 +9.279999999999999361e+00 4.727604498793501475e-02 +9.320000000000000284e+00 4.677346472865646226e-02 +9.359999999999999432e+00 4.626413020507481877e-02 +9.400000000000000355e+00 4.574808673879743326e-02 +9.439999999999999503e+00 4.522517967968767844e-02 +9.480000000000000426e+00 4.469443227086057624e-02 +9.519999999999999574e+00 4.415472278535303302e-02 +9.560000000000000497e+00 4.360500550414982829e-02 +9.599999999999999645e+00 4.304431315547573716e-02 +9.640000000000000568e+00 4.246958488506545615e-02 +9.679999999999999716e+00 4.186938221535609073e-02 +9.720000000000000639e+00 4.123086530416501111e-02 +9.759999999999999787e+00 4.054212262062352468e-02 +9.800000000000000711e+00 3.979230328029435132e-02 +9.839999999999999858e+00 3.897450856814422410e-02 +9.880000000000000782e+00 3.809368158794705639e-02 +9.919999999999999929e+00 3.715762660272940027e-02 +9.960000000000000853e+00 3.617408301630460621e-02 +1.000000000000000000e+01 3.515064608253378603e-02 +1.004000000000000092e+01 3.410004284298195215e-02 +1.008000000000000007e+01 3.305482031072375787e-02 +1.012000000000000099e+01 3.204904037385152699e-02 +1.016000000000000014e+01 3.111283301411926425e-02 +1.020000000000000107e+01 3.027291008841730829e-02 +1.024000000000000021e+01 2.954751588012200988e-02 +1.027999999999999936e+01 2.893087309353207376e-02 +1.032000000000000028e+01 2.841264397657392721e-02 +1.035999999999999943e+01 2.798378656454881055e-02 +1.040000000000000036e+01 2.763634778401454287e-02 +1.043999999999999950e+01 2.736187153754084112e-02 +1.048000000000000043e+01 2.714708534221194416e-02 +1.051999999999999957e+01 2.697809350842888740e-02 +1.056000000000000050e+01 2.684157775153940897e-02 +1.059999999999999964e+01 2.672461768314412026e-02 +1.064000000000000057e+01 2.661644670165546830e-02 +1.067999999999999972e+01 2.651395276829744685e-02 +1.072000000000000064e+01 2.641593495728193283e-02 +1.075999999999999979e+01 2.632121179894745394e-02 +1.080000000000000071e+01 2.622861964149897709e-02 +1.083999999999999986e+01 2.613718198877732282e-02 +1.088000000000000078e+01 2.604661552472379085e-02 +1.091999999999999993e+01 2.595681013672950649e-02 +1.096000000000000085e+01 2.586765720513308636e-02 +1.100000000000000000e+01 2.577904957511305967e-02 +1.104000000000000092e+01 2.569090086305537632e-02 +1.108000000000000007e+01 2.560320289921372283e-02 +1.112000000000000099e+01 2.551596680430360739e-02 +1.116000000000000014e+01 2.542920352876891152e-02 +1.120000000000000107e+01 2.534292385527306055e-02 +1.124000000000000021e+01 2.525713541106652110e-02 +1.127999999999999936e+01 2.517183378085237691e-02 +1.132000000000000028e+01 2.508701161963346540e-02 +1.135999999999999943e+01 2.500266166233249682e-02 +1.140000000000000036e+01 2.491877672289816853e-02 +1.143999999999999950e+01 2.483535048125476430e-02 +1.148000000000000043e+01 2.475237982487044422e-02 +1.151999999999999957e+01 2.466986244794808561e-02 +1.156000000000000050e+01 2.458779605829344503e-02 +1.159999999999999964e+01 2.450617837731830506e-02 +1.164000000000000057e+01 2.442500693242810605e-02 +1.167999999999999972e+01 2.434427843967625071e-02 +1.172000000000000064e+01 2.426398943609317863e-02 +1.175999999999999979e+01 2.418413648844756556e-02 +1.180000000000000071e+01 2.410471619301205598e-02 +1.183999999999999986e+01 2.402572523004955357e-02 +1.188000000000000078e+01 2.394716052630707984e-02 +1.191999999999999993e+01 2.386901908814445181e-02 +1.196000000000000085e+01 2.379129794627773964e-02 +1.200000000000000000e+01 2.371399415560346627e-02 +1.204000000000000092e+01 2.363710478059675296e-02 +1.208000000000000007e+01 2.356062685223371186e-02 +1.212000000000000099e+01 2.348455741173503808e-02 +1.216000000000000014e+01 2.340889352489385067e-02 +1.220000000000000107e+01 2.333363228188353386e-02 +1.224000000000000021e+01 2.325877080086934112e-02 +1.228000000000000114e+01 2.318430623912958594e-02 +1.232000000000000028e+01 2.311023578128598363e-02 +1.235999999999999943e+01 2.303655663528906064e-02 +1.240000000000000036e+01 2.296326603223185214e-02 +1.243999999999999950e+01 2.289036122516070748e-02 +1.248000000000000043e+01 2.281783948590668590e-02 +1.251999999999999957e+01 2.274569810795353322e-02 +1.256000000000000050e+01 2.267393440726136328e-02 +1.259999999999999964e+01 2.260254572208251697e-02 +1.264000000000000057e+01 2.253152941304135612e-02 +1.267999999999999972e+01 2.246088286373870008e-02 +1.272000000000000064e+01 2.239060347976339418e-02 +1.275999999999999979e+01 2.232068868824531663e-02 +1.280000000000000071e+01 2.225113593767422127e-02 +1.283999999999999986e+01 2.218194269765035379e-02 +1.288000000000000078e+01 2.211310645849770898e-02 +1.291999999999999993e+01 2.204462473129832623e-02 +1.296000000000000085e+01 2.197649504778543053e-02 +1.300000000000000000e+01 2.190871496016686543e-02 +1.304000000000000092e+01 2.184128204096839540e-02 +1.308000000000000007e+01 2.177419388291430488e-02 +1.312000000000000099e+01 2.170744809869753697e-02 +1.316000000000000014e+01 2.164104232078946366e-02 +1.320000000000000107e+01 2.157497420126844659e-02 +1.324000000000000021e+01 2.150924141164468548e-02 +1.328000000000000114e+01 2.144384164267689255e-02 +1.332000000000000028e+01 2.137877260421881809e-02 +1.335999999999999943e+01 2.131403202505752562e-02 +1.340000000000000036e+01 2.124961765274732337e-02 +1.343999999999999950e+01 2.118552725344642965e-02 +1.348000000000000043e+01 2.112175861175728461e-02 +1.351999999999999957e+01 2.105830953056031865e-02 +1.356000000000000050e+01 2.099517783085108960e-02 +1.359999999999999964e+01 2.093236135158068126e-02 +1.364000000000000057e+01 2.086985794949571676e-02 +1.367999999999999972e+01 2.080766549898024897e-02 +1.372000000000000064e+01 2.074578189189964922e-02 +1.375999999999999979e+01 2.068420503744594283e-02 +1.380000000000000071e+01 2.062293286198312037e-02 +1.383999999999999986e+01 2.056196330889452362e-02 +1.388000000000000078e+01 2.050129433843080398e-02 +1.391999999999999993e+01 2.044092392755935891e-02 +1.396000000000000085e+01 2.038085006981431649e-02 +1.400000000000000000e+01 2.032107077514849411e-02 +1.404000000000000092e+01 2.026158406978599905e-02 +1.408000000000000007e+01 2.020238799607640412e-02 +1.412000000000000099e+01 2.014348061234967624e-02 +1.416000000000000014e+01 2.008485999277292988e-02 +1.420000000000000107e+01 2.002652422720786404e-02 +1.424000000000000021e+01 1.996847142106957657e-02 +1.428000000000000114e+01 1.991069969518633950e-02 +1.432000000000000028e+01 1.985320718566097736e-02 +1.435999999999999943e+01 1.979599204373289068e-02 +1.440000000000000036e+01 1.973905243564167203e-02 +1.443999999999999950e+01 1.968238654249164846e-02 +1.448000000000000043e+01 1.962599256011757567e-02 +1.451999999999999957e+01 1.956986869895185194e-02 +1.456000000000000050e+01 1.951401318389221762e-02 +1.459999999999999964e+01 1.945842425417144278e-02 +1.464000000000000057e+01 1.940310016322720904e-02 +1.467999999999999972e+01 1.934803917857417718e-02 +1.472000000000000064e+01 1.929323958167619008e-02 +1.475999999999999979e+01 1.923869966782022420e-02 +1.480000000000000071e+01 1.918441774599143404e-02 +1.483999999999999986e+01 1.913039213874893549e-02 +1.488000000000000078e+01 1.907662118210311925e-02 +1.491999999999999993e+01 1.902310322539379345e-02 +1.496000000000000085e+01 1.896983663116960650e-02 +1.500000000000000000e+01 1.891681977506831647e-02 +1.504000000000000092e+01 1.886405104569854538e-02 +1.508000000000000007e+01 1.881152884452226212e-02 +1.512000000000000099e+01 1.875925158573825413e-02 +1.516000000000000014e+01 1.870721769616718461e-02 +1.520000000000000107e+01 1.865542561513721873e-02 +1.524000000000000021e+01 1.860387379437088498e-02 +1.528000000000000114e+01 1.855256069787295653e-02 +1.532000000000000028e+01 1.850148480181934910e-02 +1.535999999999999943e+01 1.845064459444708052e-02 +1.540000000000000036e+01 1.840003857594545156e-02 +1.543999999999999950e+01 1.834966525834770892e-02 +1.548000000000000043e+01 1.829952316542413429e-02 +1.551999999999999957e+01 1.824961083257631289e-02 +1.556000000000000050e+01 1.819992680673167967e-02 +1.559999999999999964e+01 1.815046964623998407e-02 +1.564000000000000057e+01 1.810123792076973742e-02 +1.567999999999999972e+01 1.805223021120664142e-02 +1.572000000000000064e+01 1.800344510955205479e-02 +1.575999999999999979e+01 1.795488121882308358e-02 +1.580000000000000071e+01 1.790653715295333070e-02 +1.583999999999999986e+01 1.785841153669452328e-02 +1.588000000000000078e+01 1.781050300551942703e-02 +1.591999999999999993e+01 1.776281020552498632e-02 +1.596000000000000085e+01 1.771533179333728211e-02 +1.600000000000000000e+01 1.766806643601684032e-02 +1.603999999999999915e+01 1.762101281096467922e-02 +1.608000000000000185e+01 1.757416960582974808e-02 +1.612000000000000099e+01 1.752753551841711860e-02 +1.616000000000000014e+01 1.748110925659667261e-02 +1.619999999999999929e+01 1.743488953821329887e-02 +1.624000000000000199e+01 1.738887509099704132e-02 +1.628000000000000114e+01 1.734306465247541229e-02 +1.632000000000000028e+01 1.729745696988515408e-02 +1.635999999999999943e+01 1.725205080008585315e-02 +1.639999999999999858e+01 1.720684490947396728e-02 +1.644000000000000128e+01 1.716183807389747712e-02 +1.648000000000000043e+01 1.711702907857220665e-02 +1.651999999999999957e+01 1.707241671799757457e-02 +1.655999999999999872e+01 1.702799979587463555e-02 +1.660000000000000142e+01 1.698377712502386824e-02 +1.664000000000000057e+01 1.693974752730410122e-02 +1.667999999999999972e+01 1.689590983353247283e-02 +1.671999999999999886e+01 1.685226288340456804e-02 +1.676000000000000156e+01 1.680880552541592909e-02 +1.680000000000000071e+01 1.676553661678405888e-02 +1.683999999999999986e+01 1.672245502337094472e-02 +1.687999999999999901e+01 1.667955961960695260e-02 +1.692000000000000171e+01 1.663684928824549908e-02 +1.696000000000000085e+01 1.659432292096801528e-02 +1.700000000000000000e+01 1.655197941744911486e-02 +1.703999999999999915e+01 1.650981768531171665e-02 +1.708000000000000185e+01 1.646783664087156721e-02 +1.712000000000000099e+01 1.642603520840269990e-02 +1.716000000000000014e+01 1.638441232023210570e-02 +1.719999999999999929e+01 1.634296691666924442e-02 +1.724000000000000199e+01 1.630169794593545537e-02 +1.728000000000000114e+01 1.626060436409523802e-02 +1.732000000000000028e+01 1.621968513498695322e-02 +1.735999999999999943e+01 1.617893923015499905e-02 +1.740000000000000213e+01 1.613836562878245146e-02 +1.744000000000000128e+01 1.609796331762400334e-02 +1.748000000000000043e+01 1.605773129094039200e-02 +1.751999999999999957e+01 1.601766855043217086e-02 +1.755999999999999872e+01 1.597777410517559754e-02 +1.760000000000000142e+01 1.593804697155796687e-02 +1.764000000000000057e+01 1.589848617321414428e-02 +1.767999999999999972e+01 1.585909074096369595e-02 +1.771999999999999886e+01 1.581985971274824793e-02 +1.776000000000000156e+01 1.578079213356992430e-02 +1.780000000000000071e+01 1.574188705543020858e-02 +1.783999999999999986e+01 1.570314353726906878e-02 +1.787999999999999901e+01 1.566456064490535580e-02 +1.792000000000000171e+01 1.562613745097701096e-02 +1.796000000000000085e+01 1.558787303488244624e-02 +1.800000000000000000e+01 1.554976648272223677e-02 +1.803999999999999915e+01 1.551181688724125735e-02 +1.808000000000000185e+01 1.547402334777157017e-02 +1.812000000000000099e+01 1.543638497017603244e-02 +1.816000000000000014e+01 1.539890086679188141e-02 +1.819999999999999929e+01 1.536157015637550603e-02 +1.824000000000000199e+01 1.532439196404704676e-02 +1.828000000000000114e+01 1.528736542123644397e-02 +1.832000000000000028e+01 1.525048966562909426e-02 +1.835999999999999943e+01 1.521376384111254761e-02 +1.840000000000000213e+01 1.517718709772367461e-02 +1.844000000000000128e+01 1.514075859159600551e-02 +1.848000000000000043e+01 1.510447748490828176e-02 +1.851999999999999957e+01 1.506834294583240912e-02 +1.855999999999999872e+01 1.503235414848310900e-02 +1.860000000000000142e+01 1.499651027286721602e-02 +1.864000000000000057e+01 1.496081050483371276e-02 +1.867999999999999972e+01 1.492525403602445143e-02 +1.871999999999999886e+01 1.488984006382489296e-02 +1.876000000000000156e+01 1.485456779131570822e-02 +1.880000000000000071e+01 1.481943642722485799e-02 +1.883999999999999986e+01 1.478444518587964177e-02 +1.887999999999999901e+01 1.474959328715976133e-02 +1.892000000000000171e+01 1.471487995645075382e-02 +1.896000000000000085e+01 1.468030442459731903e-02 +1.900000000000000000e+01 1.464586592785796157e-02 +1.903999999999999915e+01 1.461156370785910742e-02 +1.908000000000000185e+01 1.457739701155059790e-02 +1.912000000000000099e+01 1.454336509116095980e-02 +1.916000000000000014e+01 1.450946720415321502e-02 +1.919999999999999929e+01 1.447570261318141538e-02 +1.924000000000000199e+01 1.444207058604701777e-02 +1.928000000000000114e+01 1.440857039565655515e-02 +1.932000000000000028e+01 1.437520131997846451e-02 +1.935999999999999943e+01 1.434196264200174840e-02 +1.940000000000000213e+01 1.430885364969371361e-02 +1.944000000000000128e+01 1.427587363595916523e-02 +1.948000000000000043e+01 1.424302189859921221e-02 +1.951999999999999957e+01 1.421029774027083616e-02 +1.955999999999999872e+01 1.417770046844697360e-02 +1.960000000000000142e+01 1.414522939537643173e-02 +1.964000000000000057e+01 1.411288383804485715e-02 +1.967999999999999972e+01 1.408066311813546516e-02 +1.971999999999999886e+01 1.404856656199073885e-02 +1.976000000000000156e+01 1.401659350057368920e-02 +1.980000000000000071e+01 1.398474326943018910e-02 +1.983999999999999986e+01 1.395301520865208436e-02 +1.987999999999999901e+01 1.392140866283865172e-02 +1.992000000000000171e+01 1.388992298106065705e-02 +1.996000000000000085e+01 1.385855751682368159e-02 +2.000000000000000000e+01 1.382731162803183325e-02 diff --git a/app/host/SBI/priors/pdf_of_z_l20t18.txt b/app/host/SBI/priors/pdf_of_z_l20t18.txt new file mode 100644 index 00000000..62a774c0 --- /dev/null +++ b/app/host/SBI/priors/pdf_of_z_l20t18.txt @@ -0,0 +1,502 @@ +# z, probability of observing a galaxy at z +0.000000000000000000e+00 8.937185684028123300e-02 +4.000000000000000083e-02 9.033280629264767159e-02 +8.000000000000000167e-02 9.223808344236272827e-02 +1.199999999999999956e-01 9.505488956625882113e-02 +1.600000000000000033e-01 9.873512080188147355e-02 +2.000000000000000111e-01 1.032166325087160996e-01 +2.399999999999999911e-01 1.084248468284314537e-01 +2.800000000000000266e-01 1.142746419619777576e-01 +3.200000000000000067e-01 1.206724524571751028e-01 +3.599999999999999867e-01 1.275185048910527086e-01 +4.000000000000000222e-01 1.347091117376287284e-01 +4.400000000000000022e-01 1.421389482589593689e-01 +4.799999999999999822e-01 1.497032426456338106e-01 +5.200000000000000178e-01 1.572998178313900208e-01 +5.600000000000000533e-01 1.648309336282346438e-01 +5.999999999999999778e-01 1.722048894945117092e-01 +6.400000000000000133e-01 1.793373604593410109e-01 +6.800000000000000488e-01 1.861524509053810450e-01 +7.199999999999999734e-01 1.925834632810347058e-01 +7.600000000000000089e-01 1.985733881613932494e-01 +8.000000000000000444e-01 2.040751310611085279e-01 +8.399999999999999689e-01 2.090514985627157929e-01 +8.800000000000000044e-01 2.134749714393212039e-01 +9.200000000000000400e-01 2.173272953957329501e-01 +9.599999999999999645e-01 2.205989220088954705e-01 +1.000000000000000000e+00 2.232883324297872951e-01 +1.040000000000000036e+00 2.254012752259471342e-01 +1.080000000000000071e+00 2.269499476540648153e-01 +1.120000000000000107e+00 2.279521466862017010e-01 +1.159999999999999920e+00 2.284304132351712424e-01 +1.199999999999999956e+00 2.284111895943410631e-01 +1.239999999999999991e+00 2.279240067108028178e-01 +1.280000000000000027e+00 2.270007146558309929e-01 +1.320000000000000062e+00 2.256747665157490257e-01 +1.360000000000000098e+00 2.239805633268442953e-01 +1.400000000000000133e+00 2.219528651434819200e-01 +1.439999999999999947e+00 2.196263014846700501e-01 +1.479999999999999982e+00 2.170348734989805128e-01 +1.520000000000000018e+00 2.142116059915762649e-01 +1.560000000000000053e+00 2.111882371237076284e-01 +1.600000000000000089e+00 2.079949740568373673e-01 +1.640000000000000124e+00 2.046603335308468385e-01 +1.679999999999999938e+00 2.012109917472147680e-01 +1.719999999999999973e+00 1.976716955915410745e-01 +1.760000000000000009e+00 1.940652679240972012e-01 +1.800000000000000044e+00 1.904126226753416196e-01 +1.840000000000000080e+00 1.867328184626033394e-01 +1.880000000000000115e+00 1.830431617689284585e-01 +1.919999999999999929e+00 1.793592932620243774e-01 +1.959999999999999964e+00 1.756953391937929043e-01 +2.000000000000000000e+00 1.720640704035981405e-01 +2.040000000000000036e+00 1.684770772498734859e-01 +2.080000000000000071e+00 1.649449567140706396e-01 +2.120000000000000107e+00 1.614775080505392590e-01 +2.160000000000000142e+00 1.580839331760540978e-01 +2.200000000000000178e+00 1.547730379652068611e-01 +2.240000000000000213e+00 1.515534303118794002e-01 +2.280000000000000249e+00 1.484337104153212739e-01 +2.319999999999999840e+00 1.454226482971641421e-01 +2.359999999999999876e+00 1.425293428141969077e-01 +2.399999999999999911e+00 1.397633562544708941e-01 +2.439999999999999947e+00 1.371348203045237302e-01 +2.479999999999999982e+00 1.346545038927688398e-01 +2.520000000000000018e+00 1.323338362283822955e-01 +2.560000000000000053e+00 1.301848824048806585e-01 +2.600000000000000089e+00 1.282202657182393291e-01 +2.640000000000000124e+00 1.264530335574478026e-01 +2.680000000000000160e+00 1.248964654027139715e-01 +2.720000000000000195e+00 1.235638236082316488e-01 +2.760000000000000231e+00 1.224680509709904219e-01 +2.800000000000000266e+00 1.216214207658932456e-01 +2.839999999999999858e+00 1.210351478383247387e-01 +2.879999999999999893e+00 1.207189720682675937e-01 +2.919999999999999929e+00 1.206807278095393537e-01 +2.959999999999999964e+00 1.209259147829859632e-01 +3.000000000000000000e+00 1.214572870846656050e-01 +3.040000000000000036e+00 1.222744774946118579e-01 +3.080000000000000071e+00 1.233736740348700539e-01 +3.120000000000000107e+00 1.247473646440589307e-01 +3.160000000000000142e+00 1.263841636391181966e-01 +3.200000000000000178e+00 1.282687313998994716e-01 +3.240000000000000213e+00 1.303817954109189603e-01 +3.280000000000000249e+00 1.327002770878720761e-01 +3.320000000000000284e+00 1.351975249056452388e-01 +3.359999999999999876e+00 1.378436503471633379e-01 +3.399999999999999911e+00 1.406059593903364602e-01 +3.439999999999999947e+00 1.434494625260750444e-01 +3.479999999999999982e+00 1.463374589969446693e-01 +3.520000000000000018e+00 1.492321807262977640e-01 +3.560000000000000053e+00 1.520954680313322260e-01 +3.600000000000000089e+00 1.548894621016670969e-01 +3.640000000000000124e+00 1.575772948181395350e-01 +3.680000000000000160e+00 1.601237569452594878e-01 +3.720000000000000195e+00 1.624959268463175288e-01 +3.760000000000000231e+00 1.646637435527792126e-01 +3.800000000000000266e+00 1.666005101601221727e-01 +3.839999999999999858e+00 1.682833160031413089e-01 +3.879999999999999893e+00 1.696933687629541276e-01 +3.919999999999999929e+00 1.708162304583453195e-01 +3.959999999999999964e+00 1.716419540690656553e-01 +4.000000000000000000e+00 1.721651202371857481e-01 +4.040000000000000036e+00 1.723847760215783120e-01 +4.080000000000000071e+00 1.723042799876121356e-01 +4.120000000000000107e+00 1.719310599666537820e-01 +4.160000000000000142e+00 1.712762916037186278e-01 +4.200000000000000178e+00 1.703545073272706989e-01 +4.240000000000000213e+00 1.691831466341141288e-01 +4.280000000000000249e+00 1.677820596012498944e-01 +4.320000000000000284e+00 1.661729763322186970e-01 +4.360000000000000320e+00 1.643789556295792154e-01 +4.400000000000000355e+00 1.624238265605460196e-01 +4.440000000000000391e+00 1.603316318110841465e-01 +4.480000000000000426e+00 1.581260941115244234e-01 +4.520000000000000462e+00 1.558301205174343340e-01 +4.560000000000000497e+00 1.534653493810527736e-01 +4.600000000000000533e+00 1.510517548327275639e-01 +4.639999999999999680e+00 1.486073188815831558e-01 +4.679999999999999716e+00 1.461477646468774816e-01 +4.719999999999999751e+00 1.436864132505372427e-01 +4.759999999999999787e+00 1.412340883646746870e-01 +4.799999999999999822e+00 1.387991173588264127e-01 +4.839999999999999858e+00 1.363874161431843923e-01 +4.879999999999999893e+00 1.340026520612319638e-01 +4.919999999999999929e+00 1.316464761290150887e-01 +4.959999999999999964e+00 1.293188131035259980e-01 +5.000000000000000000e+00 1.270181954575942684e-01 +5.040000000000000036e+00 1.247421255010330576e-01 +5.080000000000000071e+00 1.224874487517156052e-01 +5.120000000000000107e+00 1.202507213242982875e-01 +5.160000000000000142e+00 1.180285546216803400e-01 +5.200000000000000178e+00 1.158179219865278359e-01 +5.240000000000000213e+00 1.136164141446182885e-01 +5.280000000000000249e+00 1.114224331432893678e-01 +5.320000000000000284e+00 1.092353179051859524e-01 +5.360000000000000320e+00 1.070553982909147467e-01 +5.400000000000000355e+00 1.048839784803191899e-01 +5.440000000000000391e+00 1.027232511544312055e-01 +5.480000000000000426e+00 1.005761555008248126e-01 +5.520000000000000462e+00 9.844619112287240903e-02 +5.560000000000000497e+00 9.633719633133218385e-02 +5.600000000000000533e+00 9.425310789032669001e-02 +5.639999999999999680e+00 9.219771780339043599e-02 +5.679999999999999716e+00 9.017444106877063603e-02 +5.719999999999999751e+00 8.818611173799711933e-02 +5.759999999999999787e+00 8.623481876944419366e-02 +5.799999999999999822e+00 8.432178798908775141e-02 +5.839999999999999858e+00 8.244731850110401106e-02 +5.879999999999999893e+00 8.061077638584060456e-02 +5.919999999999999929e+00 7.881064497722556350e-02 +5.959999999999999964e+00 7.704462759015755224e-02 +6.000000000000000000e+00 7.530979547604872704e-02 +6.040000000000000036e+00 7.360277120282129881e-02 +6.080000000000000071e+00 7.191993573075791679e-02 +6.120000000000000107e+00 7.025764629055560040e-02 +6.160000000000000142e+00 6.861245181826119977e-02 +6.200000000000000178e+00 6.698129316518758314e-02 +6.240000000000000213e+00 6.536167653030491853e-02 +6.280000000000000249e+00 6.375181046234087989e-02 +6.320000000000000284e+00 6.215069921406052994e-02 +6.360000000000000320e+00 6.055818803755574908e-02 +6.400000000000000355e+00 5.897495900429335774e-02 +6.440000000000000391e+00 5.740247748496680319e-02 +6.480000000000000426e+00 5.584289591668432917e-02 +6.520000000000000462e+00 5.429892209180569446e-02 +6.560000000000000497e+00 5.277365848582982882e-02 +6.600000000000000533e+00 5.127042376201555729e-02 +6.640000000000000568e+00 4.979256734177955890e-02 +6.679999999999999716e+00 4.834329211496913442e-02 +6.719999999999999751e+00 4.692548585322243310e-02 +6.759999999999999787e+00 4.554157235429597689e-02 +6.799999999999999822e+00 4.419339934347796262e-02 +6.839999999999999858e+00 4.288216232443563952e-02 +6.879999999999999893e+00 4.160836743283599648e-02 +6.919999999999999929e+00 4.037183390687987955e-02 +6.959999999999999964e+00 3.917173441417490243e-02 +7.000000000000000000e+00 3.800666930451965936e-02 +7.040000000000000036e+00 3.687476901613277736e-02 +7.080000000000000071e+00 3.577381744701975397e-02 +7.120000000000000107e+00 3.470138818193790192e-02 +7.160000000000000142e+00 3.365498507395242589e-02 +7.200000000000000178e+00 3.263217881862678221e-02 +7.240000000000000213e+00 3.163073179613069136e-02 +7.280000000000000249e+00 3.064870452992306260e-02 +7.320000000000000284e+00 2.968453853365551551e-02 +7.360000000000000320e+00 2.873711198608451187e-02 +7.400000000000000355e+00 2.780576647251539399e-02 +7.440000000000000391e+00 2.689030431392394963e-02 +7.480000000000000426e+00 2.599095908594752438e-02 +7.520000000000000462e+00 2.510834273016583282e-02 +7.560000000000000497e+00 2.424337292393459464e-02 +7.600000000000000533e+00 2.339718646529105953e-02 +7.640000000000000568e+00 2.257104458466559216e-02 +7.679999999999999716e+00 2.176623951433669102e-02 +7.719999999999999751e+00 2.098400038162877984e-02 +7.759999999999999787e+00 2.022540513938783993e-02 +7.799999999999999822e+00 1.949131055165833834e-02 +7.839999999999999858e+00 1.878229868502678118e-02 +7.879999999999999893e+00 1.809864206008263490e-02 +7.919999999999999929e+00 1.744028835262829980e-02 +7.959999999999999964e+00 1.680686429493029321e-02 +8.000000000000000000e+00 1.619769729473594794e-02 +8.040000000000000924e+00 1.561185232014992291e-02 +8.080000000000000071e+00 1.504818084338381372e-02 +8.120000000000000995e+00 1.450537814987603626e-02 +8.160000000000000142e+00 1.398204505406648590e-02 +8.199999999999999289e+00 1.347675008085033078e-02 +8.240000000000000213e+00 1.298808843234431319e-02 +8.279999999999999361e+00 1.251473452636137421e-02 +8.320000000000000284e+00 1.205548550657579987e-02 +8.359999999999999432e+00 1.160929388693468858e-02 +8.400000000000000355e+00 1.117528829472488883e-02 +8.439999999999999503e+00 1.075278195489223765e-02 +8.480000000000000426e+00 1.034126969417138719e-02 +8.519999999999999574e+00 9.940414658862981381e-03 +8.560000000000000497e+00 9.550026291073869522e-03 +8.599999999999999645e+00 9.170031813146550456e-03 +8.640000000000000568e+00 8.800443594162224437e-03 +8.679999999999999716e+00 8.441326940912052129e-03 +8.720000000000000639e+00 8.092765703694715373e-03 +8.759999999999999787e+00 7.754828767160932938e-03 +8.800000000000000711e+00 7.427544086605984870e-03 +8.839999999999999858e+00 7.110878477314025274e-03 +8.880000000000000782e+00 6.804724033159918230e-03 +8.919999999999999929e+00 6.508891547972557243e-03 +8.960000000000000853e+00 6.223110821822047493e-03 +9.000000000000000000e+00 5.947037296207914603e-03 +9.040000000000000924e+00 5.680264096232807286e-03 +9.080000000000000071e+00 5.422338280477844580e-03 +9.120000000000000995e+00 5.172779917130580427e-03 +9.160000000000000142e+00 4.931102533054195977e-03 +9.200000000000001066e+00 4.696833505157391696e-03 +9.240000000000000213e+00 4.469533075599854861e-03 +9.279999999999999361e+00 4.248810859633563551e-03 +9.320000000000000284e+00 4.034338958236862414e-03 +9.359999999999999432e+00 3.825861067402441980e-03 +9.400000000000000355e+00 3.623197270232796953e-03 +9.439999999999999503e+00 3.426244461957445551e-03 +9.480000000000000426e+00 3.234972687336894453e-03 +9.519999999999999574e+00 3.049417865350284498e-03 +9.560000000000000497e+00 2.869671507694035104e-03 +9.599999999999999645e+00 2.695868216182251489e-03 +9.640000000000000568e+00 2.528171791613524795e-03 +9.679999999999999716e+00 2.366761757370881500e-03 +9.720000000000000639e+00 2.211818809685319170e-03 +9.759999999999999787e+00 2.063510374760202208e-03 +9.800000000000000711e+00 1.921979098848147641e-03 +9.839999999999999858e+00 1.787333285396232579e-03 +9.880000000000000782e+00 1.659639590130649722e-03 +9.919999999999999929e+00 1.538918126893931357e-03 +9.960000000000000853e+00 1.425139988986634521e-03 +1.000000000000000000e+01 1.318227060191130015e-03 +1.004000000000000092e+01 1.218053882111420124e-03 +1.008000000000000007e+01 1.124451263405701167e-03 +1.012000000000000099e+01 1.037211263323046650e-03 +1.016000000000000014e+01 9.560931562005189725e-04 +1.020000000000000107e+01 8.808299831509390975e-04 +1.024000000000000021e+01 8.111353187070302140e-04 +1.027999999999999936e+01 7.467099194512601601e-04 +1.032000000000000028e+01 6.872479739267523847e-04 +1.035999999999999943e+01 6.324427335521741770e-04 +1.040000000000000036e+01 5.819913681823180940e-04 +1.043999999999999950e+01 5.355989491697011468e-04 +1.048000000000000043e+01 4.929815321542927202e-04 +1.051999999999999957e+01 4.538683587575037515e-04 +1.056000000000000050e+01 4.180032320199636962e-04 +1.059999999999999964e+01 3.851451627670497702e-04 +1.064000000000000057e+01 3.550684042172817670e-04 +1.067999999999999972e+01 3.275623581615874570e-04 +1.072000000000000064e+01 3.024306366149237708e-04 +1.075999999999999979e+01 2.794896009876343397e-04 +1.080000000000000071e+01 2.585673133913396460e-04 +1.083999999999999986e+01 2.395024670043992613e-04 +1.088000000000000078e+01 2.221433828864501970e-04 +1.091999999999999993e+01 2.063471373192859960e-04 +1.096000000000000085e+01 1.919788601349097597e-04 +1.100000000000000000e+01 1.789112231163864997e-04 +1.104000000000000092e+01 1.670241178677487020e-04 +1.108000000000000007e+01 1.562045063091096662e-04 +1.112000000000000099e+01 1.463464142103378476e-04 +1.116000000000000014e+01 1.373510271568861895e-04 +1.120000000000000107e+01 1.291268466641654501e-04 +1.124000000000000021e+01 1.215898622625991216e-04 +1.127999999999999936e+01 1.146636992607756491e-04 +1.132000000000000028e+01 1.082797071039949565e-04 +1.135999999999999943e+01 1.023769452713527530e-04 +1.140000000000000036e+01 9.690205571753320924e-05 +1.143999999999999950e+01 9.180901334231003272e-05 +1.148000000000000043e+01 8.705875067170791896e-05 +1.151999999999999957e+01 8.261866200376480904e-05 +1.156000000000000050e+01 7.846201320535944922e-05 +1.159999999999999964e+01 7.456726978657725335e-05 +1.164000000000000057e+01 7.091736665408517092e-05 +1.167999999999999972e+01 6.749903447112548343e-05 +1.172000000000000064e+01 6.430197817826232751e-05 +1.175999999999999979e+01 6.131803260932313098e-05 +1.180000000000000071e+01 5.854050946542824419e-05 +1.183999999999999986e+01 5.596360568461816178e-05 +1.188000000000000078e+01 5.358189299407844374e-05 +1.191999999999999993e+01 5.138990282272328414e-05 +1.196000000000000085e+01 4.938179674603575861e-05 +1.200000000000000000e+01 4.755113366673049502e-05 +1.204000000000000092e+01 4.589073223640926570e-05 +1.208000000000000007e+01 4.439261985084259427e-05 +1.212000000000000099e+01 4.304805681029727575e-05 +1.216000000000000014e+01 4.184761908289189199e-05 +1.220000000000000107e+01 4.078132697983325263e-05 +1.224000000000000021e+01 3.983880560552875222e-05 +1.228000000000000114e+01 3.900946265314294078e-05 +1.232000000000000028e+01 3.828267051257863024e-05 +1.235999999999999943e+01 3.764794443916759896e-05 +1.240000000000000036e+01 3.709510622623112935e-05 +1.243999999999999950e+01 3.661442640751263051e-05 +1.248000000000000043e+01 3.619674081615174686e-05 +1.251999999999999957e+01 3.583353936263523014e-05 +1.256000000000000050e+01 3.551702688114008081e-05 +1.259999999999999964e+01 3.524015715007331742e-05 +1.264000000000000057e+01 3.499664245120324090e-05 +1.267999999999999972e+01 3.478095858353203216e-05 +1.272000000000000064e+01 3.458830951160961467e-05 +1.275999999999999979e+01 3.441456361771163771e-05 +1.280000000000000071e+01 3.425620413501475806e-05 +1.283999999999999986e+01 3.411027252463413634e-05 +1.288000000000000078e+01 3.397430825344480696e-05 +1.291999999999999993e+01 3.384628785974165267e-05 +1.296000000000000085e+01 3.372456559961176028e-05 +1.300000000000000000e+01 3.360781739432717633e-05 +1.304000000000000092e+01 3.349498926641662939e-05 +1.308000000000000007e+01 3.338525098358351178e-05 +1.312000000000000099e+01 3.327795523701107311e-05 +1.316000000000000014e+01 3.317260236794217470e-05 +1.320000000000000107e+01 3.306881042003807347e-05 +1.324000000000000021e+01 3.296629013089419932e-05 +1.328000000000000114e+01 3.286482437401084340e-05 +1.332000000000000028e+01 3.276425151173006938e-05 +1.335999999999999943e+01 3.266445210907026499e-05 +1.340000000000000036e+01 3.256533847774956261e-05 +1.343999999999999950e+01 3.246684655903256140e-05 +1.348000000000000043e+01 3.236892970550698975e-05 +1.351999999999999957e+01 3.227155397903955849e-05 +1.356000000000000050e+01 3.217469464008779461e-05 +1.359999999999999964e+01 3.207833355875703142e-05 +1.364000000000000057e+01 3.198245732837338588e-05 +1.367999999999999972e+01 3.188705860696468074e-05 +1.372000000000000064e+01 3.179213415439568476e-05 +1.375999999999999979e+01 3.169768075715421183e-05 +1.380000000000000071e+01 3.160369522811445317e-05 +1.383999999999999986e+01 3.151017440630253221e-05 +1.388000000000000078e+01 3.141711515666354337e-05 +1.391999999999999993e+01 3.132451436983030535e-05 +1.396000000000000085e+01 3.123236896189439789e-05 +1.400000000000000000e+01 3.114067587417917733e-05 +1.404000000000000092e+01 3.104943207301455389e-05 +1.408000000000000007e+01 3.095863454951357154e-05 +1.412000000000000099e+01 3.086828031935078345e-05 +1.416000000000000014e+01 3.077836642254228776e-05 +1.420000000000000107e+01 3.068888992322744378e-05 +1.424000000000000021e+01 3.059984790945244489e-05 +1.428000000000000114e+01 3.051123749295563299e-05 +1.432000000000000028e+01 3.042305580895454762e-05 +1.435999999999999943e+01 3.033530001593482162e-05 +1.440000000000000036e+01 3.024796729544086240e-05 +1.443999999999999950e+01 3.016105485186826104e-05 +1.448000000000000043e+01 3.007455991225797687e-05 +1.451999999999999957e+01 2.998847972609226353e-05 +1.456000000000000050e+01 2.990281156509239408e-05 +1.459999999999999964e+01 2.981755272301801586e-05 +1.464000000000000057e+01 2.973270051546833152e-05 +1.467999999999999972e+01 2.964825227968497751e-05 +1.472000000000000064e+01 2.956420537435661697e-05 +1.475999999999999979e+01 2.948055717942521653e-05 +1.480000000000000071e+01 2.939730509589414249e-05 +1.483999999999999986e+01 2.931444654563777837e-05 +1.488000000000000078e+01 2.923197897121302611e-05 +1.491999999999999993e+01 2.914989983567231857e-05 +1.496000000000000085e+01 2.906820662237840380e-05 +1.500000000000000000e+01 2.898689683482075924e-05 +1.504000000000000092e+01 2.890596799643365910e-05 +1.508000000000000007e+01 2.882541765041590211e-05 +1.512000000000000099e+01 2.874524335955213158e-05 +1.516000000000000014e+01 2.866544270603581566e-05 +1.520000000000000107e+01 2.858601329129385396e-05 +1.524000000000000021e+01 2.850695273581273928e-05 +1.528000000000000114e+01 2.842825867896634222e-05 +1.532000000000000028e+01 2.834992877884527814e-05 +1.535999999999999943e+01 2.827196071208785966e-05 +1.540000000000000036e+01 2.819435217371256037e-05 +1.543999999999999950e+01 2.811710087695209459e-05 +1.548000000000000043e+01 2.804020455308895408e-05 +1.551999999999999957e+01 2.796366095129260493e-05 +1.556000000000000050e+01 2.788746783845802611e-05 +1.559999999999999964e+01 2.781162299904592521e-05 +1.564000000000000057e+01 2.773612423492433307e-05 +1.567999999999999972e+01 2.766096936521171979e-05 +1.572000000000000064e+01 2.758615622612159800e-05 +1.575999999999999979e+01 2.751168267080856620e-05 +1.580000000000000071e+01 2.743754656921581567e-05 +1.583999999999999986e+01 2.736374580792406391e-05 +1.588000000000000078e+01 2.729027829000191775e-05 +1.591999999999999993e+01 2.721714193485765953e-05 +1.596000000000000085e+01 2.714433467809241902e-05 +1.600000000000000000e+01 2.707185447135478193e-05 +1.603999999999999915e+01 2.699969928219671638e-05 +1.608000000000000185e+01 2.692786709393089523e-05 +1.612000000000000099e+01 2.685635590548941784e-05 +1.616000000000000014e+01 2.678516373128379208e-05 +1.619999999999999929e+01 2.671428860106637670e-05 +1.624000000000000199e+01 2.664372855979297563e-05 +1.628000000000000114e+01 2.657348166748690617e-05 +1.632000000000000028e+01 2.650354599910429698e-05 +1.635999999999999943e+01 2.643391964440064317e-05 +1.639999999999999858e+01 2.636460070779870980e-05 +1.644000000000000128e+01 2.629558730825766528e-05 +1.648000000000000043e+01 2.622687757914345824e-05 +1.651999999999999957e+01 2.615846966810047507e-05 +1.655999999999999872e+01 2.609036173692437320e-05 +1.660000000000000142e+01 2.602255196143621541e-05 +1.664000000000000057e+01 2.595503853135774585e-05 +1.667999999999999972e+01 2.588781965018786222e-05 +1.671999999999999886e+01 2.582089353508039902e-05 +1.676000000000000156e+01 2.575425841672287315e-05 +1.680000000000000071e+01 2.568791253921662190e-05 +1.683999999999999986e+01 2.562185415995791687e-05 +1.687999999999999901e+01 2.555608154952035169e-05 +1.692000000000000171e+01 2.549059299153828015e-05 +1.696000000000000085e+01 2.542538678259143338e-05 +1.700000000000000000e+01 2.536046123209062119e-05 +1.703999999999999915e+01 2.529581466216452445e-05 +1.708000000000000185e+01 2.523144540754765637e-05 +1.712000000000000099e+01 2.516735181546926898e-05 +1.716000000000000014e+01 2.510353224554348964e-05 +1.719999999999999929e+01 2.503998506966037898e-05 +1.724000000000000199e+01 2.497670867187817145e-05 +1.728000000000000114e+01 2.491370144831641357e-05 +1.732000000000000028e+01 2.485096180705028475e-05 +1.735999999999999943e+01 2.478848816800579899e-05 +1.740000000000000213e+01 2.472627896285614494e-05 +1.744000000000000128e+01 2.466433263491889005e-05 +1.748000000000000043e+01 2.460264763905433658e-05 +1.751999999999999957e+01 2.454122244156472127e-05 +1.755999999999999872e+01 2.448005552009445524e-05 +1.760000000000000142e+01 2.441914536353130229e-05 +1.764000000000000057e+01 2.435849047190856700e-05 +1.767999999999999972e+01 2.429808935630810936e-05 +1.771999999999999886e+01 2.423794053876442346e-05 +1.776000000000000156e+01 2.417804255216957662e-05 +1.780000000000000071e+01 2.411839394017904309e-05 +1.783999999999999986e+01 2.405899325711852697e-05 +1.787999999999999901e+01 2.399983906789157132e-05 +1.792000000000000171e+01 2.394092994788817342e-05 +1.796000000000000085e+01 2.388226448289419631e-05 +1.800000000000000000e+01 2.382384126900169508e-05 +1.803999999999999915e+01 2.376565891252008011e-05 +1.808000000000000185e+01 2.370771602988816450e-05 +1.812000000000000099e+01 2.365001124758702811e-05 +1.816000000000000014e+01 2.359254320205376592e-05 +1.819999999999999929e+01 2.353531053959598510e-05 +1.824000000000000199e+01 2.347831191630718982e-05 +1.828000000000000114e+01 2.342154599798298937e-05 +1.832000000000000028e+01 2.336501146003804827e-05 +1.835999999999999943e+01 2.330870698742388002e-05 +1.840000000000000213e+01 2.325263127454741673e-05 +1.844000000000000128e+01 2.319678366313299191e-05 +1.848000000000000043e+01 2.314116381322212630e-05 +1.851999999999999957e+01 2.308577120086397597e-05 +1.855999999999999872e+01 2.303060563547403055e-05 +1.860000000000000142e+01 2.297566738286748513e-05 +1.864000000000000057e+01 2.292095732784350793e-05 +1.867999999999999972e+01 2.286647718611998177e-05 +1.871999999999999886e+01 2.281222977676120766e-05 +1.876000000000000156e+01 2.275821936736810666e-05 +1.880000000000000071e+01 2.270445210502710956e-05 +1.883999999999999986e+01 2.265093654611711210e-05 +1.887999999999999901e+01 2.259768429729954703e-05 +1.892000000000000171e+01 2.254471077809404547e-05 +1.896000000000000085e+01 2.249203611210619897e-05 +1.900000000000000000e+01 2.243968614899352878e-05 +1.903999999999999915e+01 2.238769361247206171e-05 +1.908000000000000185e+01 2.233609936102943187e-05 +1.912000000000000099e+01 2.228495373762616602e-05 +1.916000000000000014e+01 2.223431797282778016e-05 +1.919999999999999929e+01 2.218426559302721893e-05 +1.924000000000000199e+01 2.213488377242851189e-05 +1.928000000000000114e+01 2.208627455522260016e-05 +1.932000000000000028e+01 2.203855586402821232e-05 +1.935999999999999943e+01 2.199186220343796568e-05 +1.940000000000000213e+01 2.194634496465976953e-05 +1.944000000000000128e+01 2.190217223992789824e-05 +1.948000000000000043e+01 2.185952806448824656e-05 +1.951999999999999957e+01 2.181861102007160938e-05 +1.955999999999999872e+01 2.177963215689089654e-05 +1.960000000000000142e+01 2.174281222077486603e-05 +1.964000000000000057e+01 2.170837820688573420e-05 +1.967999999999999972e+01 2.167655929974348906e-05 +1.971999999999999886e+01 2.164758229864043569e-05 +1.976000000000000156e+01 2.162166666522953630e-05 +1.980000000000000071e+01 2.159901936318822877e-05 +1.983999999999999986e+01 2.157982968554743956e-05 +1.987999999999999901e+01 2.156426428103106564e-05 +1.992000000000000171e+01 2.155246259468062946e-05 +1.996000000000000085e+01 2.154453292907457411e-05 +2.000000000000000000e+01 2.154054931050532990e-05 diff --git a/app/host/SBI/priors/wmap9_z_age.txt b/app/host/SBI/priors/wmap9_z_age.txt new file mode 100644 index 00000000..ed51fefc --- /dev/null +++ b/app/host/SBI/priors/wmap9_z_age.txt @@ -0,0 +1,1002 @@ +# redshift; age of the universe [Gyr] assuming WMAP9 cosmology +9.999999999999999547e-07 1.376888501145438681e+01 +1.021036794270710995e-06 1.376888471472062747e+01 +1.042516135254610219e-06 1.376888441174455657e+01 +1.064447332715860249e-06 1.376888410239484450e+01 +1.086839892266210797e-06 1.376888378653741363e+01 +1.109703519485016675e-06 1.376888346403536900e+01 +1.133048124125906864e-06 1.376888313474892733e+01 +1.156883824411960901e-06 1.376888279853536190e+01 +1.181220951421228738e-06 1.376888245524895815e+01 +1.206070053564530604e-06 1.376888210474091778e+01 +1.231441901157432966e-06 1.376888174685932498e+01 +1.257347491088417558e-06 1.376888138144906293e+01 +1.283798051585239116e-06 1.376888100835175699e+01 +1.310805047081577534e-06 1.376888062740569119e+01 +1.338380183186042290e-06 1.376888023844575848e+01 +1.366535411755723414e-06 1.376887984130337550e+01 +1.395282936076472616e-06 1.376887943580639906e+01 +1.424635216152146957e-06 1.376887902177908884e+01 +1.454604974105149670e-06 1.376887859904199019e+01 +1.485205199690552396e-06 1.376887816741187009e+01 +1.516449155926235936e-06 1.376887772670165866e+01 +1.548350384841449393e-06 1.376887727672033890e+01 +1.580922713346335141e-06 1.376887681727287571e+01 +1.614180259224896147e-06 1.376887634816013239e+01 +1.648137437254056422e-06 1.376887586917878181e+01 +1.682808965451426867e-06 1.376887538012122825e+01 +1.718209871454536554e-06 1.376887488077549015e+01 +1.754355499034230215e-06 1.376887437092515043e+01 +1.791261514745103808e-06 1.376887385034921429e+01 +1.828943914715842395e-06 1.376887331882206666e+01 +1.867419031582388310e-06 1.376887277611331228e+01 +1.906703541566997424e-06 1.376887222198774197e+01 +1.946814471706178375e-06 1.376887165620517806e+01 +1.987769207230708111e-06 1.376887107852039271e+01 +2.029585499100874460e-06 1.376887048868300489e+01 +2.072281471700277991e-06 1.376886988643737020e+01 +2.115875630691442995e-06 1.376886927152245477e+01 +2.160386871036709577e-06 1.376886864367174468e+01 +2.205834485187858107e-06 1.376886800261310206e+01 +2.252238171447994759e-06 1.376886734806868873e+01 +2.299618042509388660e-06 1.376886667975480094e+01 +2.347994634170873675e-06 1.376886599738177708e+01 +2.397388914238664589e-06 1.376886530065386793e+01 +2.447822291614386200e-06 1.376886458926908041e+01 +2.499316625574338315e-06 1.376886386291909403e+01 +2.551894235243913328e-06 1.376886312128908862e+01 +2.605577909271352829e-06 1.376886236405761643e+01 +2.660390915705008973e-06 1.376886159089648842e+01 +2.716357012078363537e-06 1.376886080147058600e+01 +2.773500455707259232e-06 1.376885999543776329e+01 +2.831846014203695737e-06 1.376885917244866064e+01 +2.891418976210837701e-06 1.376885833214657495e+01 +2.952245162363814636e-06 1.376885747416730155e+01 +3.014350936481164050e-06 1.376885659813897256e+01 +3.077763216991643460e-06 1.376885570368189704e+01 +3.142509488601464215e-06 1.376885479040839222e+01 +3.208617814206930684e-06 1.376885385792263072e+01 +3.276116847057740205e-06 1.376885290582044874e+01 +3.345035843176103923e-06 1.376885193368918614e+01 +3.415404674037153577e-06 1.376885094110749108e+01 +3.487253839516105220e-06 1.376884992764516547e+01 +3.560614481107752505e-06 1.376884889286294822e+01 +3.635518395424130242e-06 1.376884783631234299e+01 +3.711998047976053008e-06 1.376884675753541565e+01 +3.790086587244613460e-06 1.376884565606460242e+01 +3.869817859048659795e-06 1.376884453142250031e+01 +3.951226421214589430e-06 1.376884338312166811e+01 +4.034347558554678255e-06 1.376884221066440439e+01 +4.119217298160546785e-06 1.376884101354254319e+01 +4.205872425018303604e-06 1.376883979123722312e+01 +4.294350497952270246e-06 1.376883854321867595e+01 +4.384689865904016774e-06 1.376883726894598325e+01 +4.476929684553911233e-06 1.376883596786684549e+01 +4.571109933292320053e-06 1.376883463941735108e+01 +4.667271432547793892e-06 1.376883328302172238e+01 +4.765455861479868118e-06 1.376883189809207053e+01 +4.865705776043974187e-06 1.376883048402813969e+01 +4.968064627436430796e-06 1.376882904021704412e+01 +5.072576780927407464e-06 1.376882756603301239e+01 +5.179287535090162319e-06 1.376882606083710137e+01 +5.288243141434711857e-06 1.376882452397693157e+01 +5.399490824454583415e-06 1.376882295478639229e+01 +5.513078802095225740e-06 1.376882135258537865e+01 +5.629056306653120558e-06 1.376881971667945592e+01 +5.747473606114430550e-06 1.376881804635959838e+01 +5.868382025942601051e-06 1.376881634090185713e+01 +5.991833971324305674e-06 1.376881459956705811e+01 +6.117882949883312418e-06 1.376881282160047348e+01 +6.246583594872297434e-06 1.376881100623149834e+01 +6.377991688852424405e-06 1.376880915267332739e+01 +6.512164187871130918e-06 1.376880726012259082e+01 +6.649159246148466646e-06 1.376880532775903632e+01 +6.789036241282888033e-06 1.376880335474513650e+01 +6.931855799987156690e-06 1.376880134022575142e+01 +7.077679824365735793e-06 1.376879928332776259e+01 +7.226571518744879713e-06 1.376879718315967338e+01 +7.378595417067294823e-06 1.376879503881124478e+01 +7.533817410862950418e-06 1.376879284935307979e+01 +7.692304777808373401e-06 1.376879061383623615e+01 +7.854126210886752760e-06 1.376878833129180357e+01 +8.019351848161375513e-06 1.376878600073050407e+01 +8.188053303175590902e-06 1.376878362114222831e+01 +8.360303695992112521e-06 1.376878119149564306e+01 +8.536177684885362511e-06 1.376877871073769199e+01 +8.715751498700547575e-06 1.376877617779319607e+01 +8.899102969893351385e-06 1.376877359156433300e+01 +9.086311568264871620e-06 1.376877095093020920e+01 +9.277458435406058914e-06 1.376876825474633748e+01 +9.472626419866768776e-06 1.376876550184416459e+01 +9.671900113064806892e-06 1.376876269103054007e+01 +9.875365885950217814e-06 1.376875982108723839e+01 +1.008311192644094961e-05 1.376875689077039233e+01 +1.029522827764606093e-05 1.376875389880996892e+01 +1.051180687689290639e-05 1.376875084390922765e+01 +1.073294159557554744e-05 1.376874772474414144e+01 +1.095872827984122610e-05 1.376874453996284231e+01 +1.118926479213286891e-05 1.376874128818501219e+01 +1.142465105360550009e-05 1.376873796800131089e+01 +1.166498908743485997e-05 1.376873457797273836e+01 +1.191038306303731562e-05 1.376873111663003613e+01 +1.216093934121981711e-05 1.376872758247302642e+01 +1.241676652027965381e-05 1.376872397396999403e+01 +1.267797548307422772e-05 1.376872028955698823e+01 +1.294467944508077793e-05 1.376871652763716192e+01 +1.321699400346724391e-05 1.376871268658008418e+01 +1.349503718719543154e-05 1.376870876472101912e+01 +1.377892950817805586e-05 1.376870476036022417e+01 +1.406879401351222716e-05 1.376870067176219337e+01 +1.436475633881149303e-05 1.376869649715490951e+01 +1.466694476265996199e-05 1.376869223472909454e+01 +1.497549026221195100e-05 1.376868788263741372e+01 +1.529052656996113813e-05 1.376868343899364788e+01 +1.561219023170425202e-05 1.376867890187192955e+01 +1.594062066572385001e-05 1.376867426930587079e+01 +1.627596022321609412e-05 1.376866953928771409e+01 +1.661835424999019798e-05 1.376866470976748325e+01 +1.696795114946503860e-05 1.376865977865206325e+01 +1.732490244699180836e-05 1.376865474380432808e+01 +1.768936285552934819e-05 1.376864960304219387e+01 +1.806149034270107277e-05 1.376864435413765975e+01 +1.844144619926291027e-05 1.376863899481588582e+01 +1.882939510901118806e-05 1.376863352275417007e+01 +1.922550522016138759e-05 1.376862793558094467e+01 +1.962994821822844148e-05 1.376862223087477410e+01 +2.004289940044002126e-05 1.376861640616327342e+01 +2.046453775171563669e-05 1.376861045892206903e+01 +2.089504602224367627e-05 1.376860438657367247e+01 +2.133461080669065316e-05 1.376859818648638623e+01 +2.178342262507673431e-05 1.376859185597316326e+01 +2.224167600535242341e-05 1.376858539229042577e+01 +2.270956956771283375e-05 1.376857879263689632e+01 +2.318730611068525160e-05 1.376857205415236685e+01 +2.367509269902773737e-05 1.376856517391647827e+01 +2.417314075347719611e-05 1.376855814894742913e+01 +2.468166614238503380e-05 1.376855097620071611e+01 +2.520088927528075896e-05 1.376854365256778578e+01 +2.573103519840386170e-05 1.376853617487470771e+01 +2.627233369224510576e-05 1.376852853988078529e+01 +2.682501937114033377e-05 1.376852074427717021e+01 +2.738933178495884952e-05 1.376851278468539341e+01 +2.796551552293127327e-05 1.376850465765595466e+01 +2.855382031966161046e-05 1.376849635966677532e+01 +2.915450116336917868e-05 1.376848788712170801e+01 +2.976781840640817914e-05 1.376847923634896098e+01 +3.039403787811173323e-05 1.376847040359951535e+01 +3.103343100000976275e-05 1.376846138504549799e+01 +3.168627490347127165e-05 1.376845217677852418e+01 +3.235285254982079060e-05 1.376844277480799938e+01 +3.303345285298201552e-05 1.376843317505940512e+01 +3.372837080470149834e-05 1.376842337337250477e+01 +3.443790760240625782e-05 1.376841336549959216e+01 +3.516237077975183133e-05 1.376840314710358726e+01 +3.590207433991592626e-05 1.376839271375621010e+01 +3.665733889169650952e-05 1.376838206093603389e+01 +3.742849178847293607e-05 1.376837118402654525e+01 +3.821586727009003914e-05 1.376836007831412623e+01 +3.901980660772772018e-05 1.376834873898601863e+01 +3.984065825181749929e-05 1.376833716112824924e+01 +4.067877798307068524e-05 1.376832533972348394e+01 +4.153452906668447076e-05 1.376831326964887658e+01 +4.240828240979117473e-05 1.376830094567383433e+01 +4.330041672222016246e-05 1.376828836245775634e+01 +4.421131868064165002e-05 1.376827551454770315e+01 +4.514138309616315040e-05 1.376826239637608218e+01 +4.609101308545248375e-05 1.376824900225818205e+01 +4.706062024545979675e-05 1.376823532638974079e+01 +4.805062483181558294e-05 1.376822136284443765e+01 +4.906145594098170962e-05 1.376820710557129601e+01 +5.009355169623368984e-05 1.376819254839209705e+01 +5.114735943755657879e-05 1.376817768499867789e+01 +5.222333591553467109e-05 1.376816250895021732e+01 +5.332194748932000733e-05 1.376814701367043980e+01 +5.444367032876648024e-05 1.376813119244476624e+01 +5.558899062081515180e-05 1.376811503841740247e+01 +5.675840478022172548e-05 1.376809854458838522e+01 +5.795241966471710435e-05 1.376808170381053564e+01 +5.917155279469365976e-05 1.376806450878637378e+01 +6.041633257751414507e-05 1.376804695206495488e+01 +6.168729853653816275e-05 1.376802902603863821e+01 +6.298500154496724612e-05 1.376801072293980965e+01 +6.431000406460925765e-05 1.376799203483749956e+01 +6.566288038966502862e-05 1.376797295363395790e+01 +6.704421689564471834e-05 1.376795347106114242e+01 +6.845461229351946727e-05 1.376793357867716061e+01 +6.989467788921937294e-05 1.376791326786257841e+01 +7.136503784859263289e-05 1.376789252981672007e+01 +7.286632946793497958e-05 1.376787135555384367e+01 +7.439920345021376526e-05 1.376784973589925443e+01 +7.596432418710083608e-05 1.376782766148533454e+01 +7.756237004693847226e-05 1.376780512274747537e+01 +7.919403366876467672e-05 1.376778210991996687e+01 +8.086002226252222427e-05 1.376775861303173443e+01 +8.256105791558401527e-05 1.376773462190205599e+01 +8.429787790572658608e-05 1.376771012613611944e+01 +8.607123502068686808e-05 1.376768511512056392e+01 +8.788189788444306007e-05 1.376765957801885243e+01 +8.973065129035772802e-05 1.376763350376660355e+01 +9.161829654132987871e-05 1.376760688106679531e+01 +9.354565179710302536e-05 1.376757969838486773e+01 +9.551355242887824145e-05 1.376755194394374726e+01 +9.752285138138931740e-05 1.376752360571873268e+01 +9.957441954259293310e-05 1.376749467143231698e+01 +1.016691461211358127e-04 1.376746512854884053e+01 +1.038079390317651104e-04 1.376743496426910696e+01 +1.059917252888428551e-04 1.376740416552479829e+01 +1.082214514081420597e-04 1.376737271897287229e+01 +1.104980838170928857e-04 1.376734061098974493e+01 +1.128226092736609674e-04 1.376730782766542305e+01 +1.151960352940357735e-04 1.376727435479748429e+01 +1.176193905893180909e-04 1.376724017788490961e+01 +1.200937255113919611e-04 1.376720528212184469e+01 +1.226201125081784887e-04 1.376716965239117130e+01 +1.251996465884644568e-04 1.376713327325798453e+01 +1.278334457965117057e-04 1.376709612896290302e+01 +1.305226516966491413e-04 1.376705820341525310e+01 +1.332684298680591973e-04 1.376701948018612853e+01 +1.360719704099743875e-04 1.376697994250125312e+01 +1.389344884574992829e-04 1.376693957323376694e+01 +1.418572247082863010e-04 1.376689835489678693e+01 +1.448414459602885454e-04 1.376685626963585918e+01 +1.478884456608275683e-04 1.376681329922124064e+01 +1.509995444672096216e-04 1.376676942504000500e+01 +1.541760908191375386e-04 1.376672462808802244e+01 +1.574194615231621947e-04 1.376667888896170311e+01 +1.607310623494312039e-04 1.376663218784965181e+01 +1.641123286409890190e-04 1.376658450452407223e+01 +1.675647259358968002e-04 1.376653581833203788e+01 +1.710897506024384952e-04 1.376648610818657303e+01 +1.746889304876892508e-04 1.376643535255751161e+01 +1.783638255797294846e-04 1.376638352946222454e+01 +1.821160286837872138e-04 1.376633061645609679e+01 +1.859471661126071496e-04 1.376627659062284614e+01 +1.898588983913397795e-04 1.376622142856460052e+01 +1.938529209772624164e-04 1.376616510639182245e+01 +1.979309649946374720e-04 1.376610759971294407e+01 +2.020947979850331612e-04 1.376604888362385815e+01 +2.063462246734251747e-04 1.376598893269715873e+01 +2.106870877504179182e-04 1.376592772097115791e+01 +2.151192686709189254e-04 1.376586522193866102e+01 +2.196446884696148273e-04 1.376580140853552692e+01 +2.242653085936047663e-04 1.376573625312898308e+01 +2.289831317525459252e-04 1.376566972750569029e+01 +2.338002027866875794e-04 1.376560180285955681e+01 +2.387186095531616218e-04 1.376553244977931278e+01 +2.437404838309219083e-04 1.376546163823579683e+01 +2.488680022447165821e-04 1.376538933756900818e+01 +2.541033872085017424e-04 1.376531551647486928e+01 +2.594489078886978283e-04 1.376524014299170595e+01 +2.649068811877129952e-04 1.376516318448646103e+01 +2.704796727481548831e-04 1.376508460764062569e+01 +2.761696979781670083e-04 1.376500437843583313e+01 +2.819794230983383929e-04 1.376492246213920900e+01 +2.879113662106318758e-04 1.376483882328837183e+01 +2.939680983898045790e-04 1.376475342567614213e+01 +3.001522447977829919e-04 1.376466623233494069e+01 +3.064664858214863688e-04 1.376457720552080310e+01 +3.129135582345807438e-04 1.376448630669715101e+01 +3.194962563836780734e-04 1.376439349651815469e+01 +3.262174333994838468e-04 1.376429873481176891e+01 +3.330800024334284342e-04 1.376420198056242228e+01 +3.400869379203083913e-04 1.376410319189333009e+01 +3.472412768674939696e-04 1.376400232604845897e+01 +3.545461201712548000e-04 1.376389933937408117e+01 +3.620046339607762621e-04 1.376379418729999315e+01 +3.696200509704534727e-04 1.376368682432029189e+01 +3.773956719410486338e-04 1.376357720397379225e+01 +3.853348670503295719e-04 1.376346527882398441e+01 +3.934410773737991160e-04 1.376335100043863946e+01 +4.017178163761590056e-04 1.376323431936893371e+01 +4.101686714341435075e-04 1.376311518512816967e+01 +4.187973053913948392e-04 1.376299354617004234e+01 +4.276074581460417618e-04 1.376286934986644184e+01 +4.366029482716816826e-04 1.376274254248482620e+01 +4.457876746724593448e-04 1.376261306916507543e+01 +4.551656182729625004e-04 1.376248087389590147e+01 +4.647408437436722595e-04 1.376234589949077680e+01 +4.745175012627045048e-04 1.376220808756330527e+01 +4.844998283146203415e-04 1.376206737850212924e+01 +4.946921515270697811e-04 1.376192371144531101e+01 +5.050988885460806202e-04 1.376177702425414928e+01 +5.157245499507892919e-04 1.376162725348649651e+01 +5.265737412084596416e-04 1.376147433436947232e+01 +5.376511646706205565e-04 1.376131820077166701e+01 +5.489616216112045040e-04 1.376115878517470037e+01 +5.605100143075559659e-04 1.376099601864424038e+01 +5.723013481652172458e-04 1.376082983080042332e+01 +5.843407338874200517e-04 1.376066014978758467e+01 +5.966333896902059446e-04 1.376048690224348903e+01 +6.091846435641563124e-04 1.376031001326779446e+01 +6.219999355836919319e-04 1.376012940638993243e+01 +6.350848202649621342e-04 1.375994500353631622e+01 +6.484449689733275471e-04 1.375975672499685487e+01 +6.620861723814977376e-04 1.375956448939076715e+01 +6.760143429793697952e-04 1.375936821363170992e+01 +6.902355176366773174e-04 1.375916781289216928e+01 +7.047558602195377973e-04 1.375896320056710032e+01 +7.195816642620540609e-04 1.375875428823686164e+01 +7.347193556941115315e-04 1.375854098562931505e+01 +7.501754956265578676e-04 1.375832320058121283e+01 +7.659567831949830796e-04 1.375810083899870939e+01 +7.820700584633115014e-04 1.375787380481714095e+01 +7.985223053884879041e-04 1.375764199995989756e+01 +8.153206548475194380e-04 1.375740532429650465e+01 +8.324723877282088139e-04 1.375716367559977549e+01 +8.499849380848947290e-04 1.375691694950216437e+01 +8.678658963605904606e-04 1.375666503945113028e+01 +8.861230126768944188e-04 1.375640783666364975e+01 +9.047642001931208286e-04 1.375614523007974199e+01 +9.237975385360888431e-04 1.375587710631507044e+01 +9.432312773020617340e-04 1.375560334961253695e+01 +9.630738396323660754e-04 1.375532384179292222e+01 +9.833338258642158533e-04 1.375503846220446924e+01 +1.004020017258353383e-03 1.375474708767143994e+01 +1.025141379805093221e-03 1.375444959244162568e+01 +1.046707068110446772e-03 1.375414584813278474e+01 +1.068726429363985216e-03 1.375383572367794649e+01 +1.091209007390187957e-03 1.375351908526962674e+01 +1.114164546785001994e-03 1.375319579630288125e+01 +1.137602997139438004e-03 1.375286571731718155e+01 +1.161534517352005944e-03 1.375252870593715215e+01 +1.185969480031869548e-03 1.375218461681200210e+01 +1.210918475994643287e-03 1.375183330155381967e+01 +1.236392318852745425e-03 1.375147460867450810e+01 +1.262402049702339175e-03 1.375110838352151532e+01 +1.288958941908851033e-03 1.375073446821217260e+01 +1.316074505993182226e-03 1.375035270156677747e+01 +1.343760494620688424e-03 1.374996291904023060e+01 +1.372028907695134079e-03 1.374956495265231560e+01 +1.400891997559784805e-03 1.374915863091657009e+01 +1.430362274307936735e-03 1.374874377876767184e+01 +1.460452511205139088e-03 1.374832021748738775e+01 +1.491175750225504732e-03 1.374788776462898809e+01 +1.522545307704473378e-03 1.374744623394011356e+01 +1.554574780110488767e-03 1.374699543528411816e+01 +1.587278049938110414e-03 1.374653517455977791e+01 +1.620669291725073714e-03 1.374606525361939369e+01 +1.654762978195954519e-03 1.374558547018521537e+01 +1.689573886535051953e-03 1.374509561776419808e+01 +1.725117104791257022e-03 1.374459548556099620e+01 +1.761408038417635290e-03 1.374408485838924499e+01 +1.798462416948605437e-03 1.374356351658101794e+01 +1.836296300817558862e-03 1.374303123589446329e+01 +1.874926088317925305e-03 1.374248778741959498e+01 +1.914368522710660405e-03 1.374193293748215261e+01 +1.954640699481249508e-03 1.374136644754557146e+01 +1.995760073749396849e-03 1.374078807411094871e+01 +2.037744467834562076e-03 1.374019756861500241e+01 +2.080612078980679115e-03 1.373959467732599649e+01 +2.124381487243351841e-03 1.373897914123759634e+01 +2.169071663542999472e-03 1.373835069596053415e+01 +2.214701977887382524e-03 1.373770907161219768e+01 +2.261292207767138379e-03 1.373705399270395411e+01 +2.308862546727897397e-03 1.373638517802621628e+01 +2.357433613122761687e-03 1.373570234053129191e+01 +2.407026459048886621e-03 1.373500518721381169e+01 +2.457662579472056057e-03 1.373429341898883749e+01 +2.509363921543237251e-03 1.373356673056754218e+01 +2.562152894111086650e-03 1.373282481033039204e+01 +2.616052377434610696e-03 1.373206734019789010e+01 +2.671085733100107221e-03 1.373129399549866925e+01 +2.727276814146768064e-03 1.373050444483505750e+01 +2.784649975405253681e-03 1.372969834994595928e+01 +2.843230084053797319e-03 1.372887536556702770e+01 +2.903042530396332966e-03 1.372803513928813324e+01 +2.964113238867408049e-03 1.372717731140797426e+01 +3.026468679268552470e-03 1.372630151478588445e+01 +3.090135878241075280e-03 1.372540737469072525e+01 +3.155142430980178731e-03 1.372449450864680287e+01 +3.221516513195499341e-03 1.372356252627684903e+01 +3.289286893323294884e-03 1.372261102914185038e+01 +3.358482944995482781e-03 1.372163961057784753e+01 +3.429134659771047606e-03 1.372064785552953126e+01 +3.501272660135215740e-03 1.371963534038059151e+01 +3.574928212772148645e-03 1.371860163278087441e+01 +3.650133242116796947e-03 1.371754629147008941e+01 +3.726920344191894923e-03 1.371646886609823923e+01 +3.805322800735986936e-03 1.371536889704248807e+01 +3.885374593628715661e-03 1.371424591522063174e+01 +3.967110419619534686e-03 1.371309944190090668e+01 +4.050565705366264424e-03 1.371192898850822139e+01 +4.135776622790055500e-03 1.371073405642668952e+01 +4.222780104753305619e-03 1.370951413679840769e+01 +4.311613861067456650e-03 1.370826871031840000e+01 +4.402316394837478553e-03 1.370699724702574862e+01 +4.494927019150257025e-03 1.370569920609069570e+01 +4.589485874113981637e-03 1.370437403559783185e+01 +4.686033944256056197e-03 1.370302117232516004e+01 +4.784613076286938632e-03 1.370164004151902049e+01 +4.885265997237740916e-03 1.370023005666489624e+01 +4.988036332979335456e-03 1.369879061925383645e+01 +5.092968627131053486e-03 1.369732111854465728e+01 +5.200108360367199783e-03 1.369582093132168588e+01 +5.309501970129648814e-03 1.369428942164805818e+01 +5.421196870755206308e-03 1.369272594061446036e+01 +5.535241474026305827e-03 1.369112982608333340e+01 +5.651685210154110302e-03 1.368950040242832422e+01 +5.770578549202941970e-03 1.368783698026907203e+01 +5.891973022965507970e-03 1.368613885620114523e+01 +6.015921247298211885e-03 1.368440531252112002e+01 +6.142476944926423578e-03 1.368263561694670472e+01 +6.271694968729432142e-03 1.368082902233187781e+01 +6.403631325515246608e-03 1.367898476637689420e+01 +6.538343200295597228e-03 1.367710207133319500e+01 +6.675888981071517909e-03 1.367518014370305934e+01 +6.816328284140432507e-03 1.367321817393395911e+01 +6.959721979935522794e-03 1.367121533610759698e+01 +7.106132219408778719e-03 1.366917078762346094e+01 +7.255622460968952012e-03 1.366708366887694126e+01 +7.408257497986313334e-03 1.366495310293183074e+01 +7.564103486875903282e-03 1.366277819518722403e+01 +7.723227975771686585e-03 1.366055803303869531e+01 +7.885699933803795927e-03 1.365829168553374551e+01 +8.051589780991784753e-03 1.365597820302139098e+01 +8.220969418766675696e-03 1.365361661679585659e+01 +8.393912261135077005e-03 1.365120593873433208e+01 +8.570493266498982812e-03 1.364874516092869783e+01 +8.750788970144834922e-03 1.364623325531114162e+01 +8.934877517416187076e-03 1.364366917327370032e+01 +9.122838697584073273e-03 1.364105184528152037e+01 +9.314753978430038647e-03 1.363838018047993117e+01 +9.510706541556557259e-03 1.363565306629514673e+01 +9.710781318440398976e-03 1.363286936802867899e+01 +9.915065027244292575e-03 1.363002792844523015e+01 +1.012364621040315153e-02 1.362712756735423092e+01 +1.033661527300086536e-02 1.362416708118478681e+01 +1.055406452195449540e-02 1.362114524255409620e+01 +1.077608820602266038e-02 1.361806079982928708e+01 +1.100278255665579526e-02 1.361491247668259064e+01 +1.123424582970553032e-02 1.361169897163987663e+01 +1.147057834801166153e-02 1.360841895762242615e+01 +1.171188254488485410e-02 1.360507108148208033e+01 +1.195826300850432768e-02 1.360165396352950928e+01 +1.220982652724928591e-02 1.359816619705580720e+01 +1.246668213598412303e-02 1.359460634784724142e+01 +1.272894116331716938e-02 1.359097295369321223e+01 +1.299671727985385675e-02 1.358726452388741812e+01 +1.327012654746473688e-02 1.358347953872219094e+01 +1.354928746959005101e-02 1.357961644897603648e+01 +1.383432104260256743e-02 1.357567367539436987e+01 +1.412535080825076499e-02 1.357164960816351851e+01 +1.442250290720555744e-02 1.356754260637784704e+01 +1.472590613373317252e-02 1.356335099750031503e+01 +1.503569199151834866e-02 1.355907307681622598e+01 +1.535199475066169501e-02 1.355470710688034863e+01 +1.567495150587639927e-02 1.355025131695750318e+01 +1.600470223590889224e-02 1.354570390245651801e+01 +1.634138986420969469e-02 1.354106302435781828e+01 +1.668516032088059042e-02 1.353632680863452364e+01 +1.703616260592478657e-02 1.353149334566735362e+01 +1.739454885382800409e-02 1.352656068965324287e+01 +1.776047439949781509e-02 1.352152685800790977e+01 +1.813409784559031832e-02 1.351638983076243328e+01 +1.851558113125294225e-02 1.351114754995401412e+01 +1.890508960231376823e-02 1.350579791901100712e+01 +1.930279208294699914e-02 1.350033880213244153e+01 +1.970886094884630527e-02 1.349476802366215367e+01 +2.012347220193723418e-02 1.348908336745774683e+01 +2.054680554666175885e-02 1.348328257625457205e+01 +2.097904446786718474e-02 1.347736335102493044e+01 +2.142037631033380518e-02 1.347132335033273520e+01 +2.187099235997555166e-02 1.346516018968390149e+01 +2.233108792674864951e-02 1.345887144087270038e+01 +2.280086242930481552e-02 1.345245463132437358e+01 +2.328051948142488559e-02 1.344590724343430210e+01 +2.377026698027094598e-02 1.343922671390410173e+01 +2.427031719649478184e-02 1.343241043307488880e+01 +2.478088686624234024e-02 1.342545574425819410e+01 +2.530219728509324265e-02 1.341835994306484459e+01 +2.583447440397674405e-02 1.341112027673221263e+01 +2.637794892710515135e-02 1.340373394345035507e+01 +2.693285641196798211e-02 1.339619809168737952e+01 +2.749943737142915290e-02 1.338850981951465435e+01 +2.807793737797220635e-02 1.338066617393226387e+01 +2.866860717013857210e-02 1.337266415019533206e+01 +2.927170276120460643e-02 1.336450069114174966e+01 +2.988748555014547267e-02 1.335617268652192280e+01 +3.051622243493272782e-02 1.334767697233116834e+01 +3.115818592821572447e-02 1.333901033014541859e+01 +3.181365427543616370e-02 1.333016948646099920e+01 +3.248291157542803481e-02 1.332115111203909663e+01 +3.316624790355401226e-02 1.331195182125580345e+01 +3.386395943743247816e-02 1.330256817145853887e+01 +3.457634858530951594e-02 1.329299666232961030e+01 +3.530372411713105646e-02 1.328323373525791418e+01 +3.604640129837308193e-02 1.327327577271966064e+01 +3.680470202668644636e-02 1.326311909766908848e+01 +3.757895497141674340e-02 1.325275997294022012e+01 +3.836949571605874354e-02 1.324219460066074205e+01 +3.917666690370839866e-02 1.323141912167906931e+01 +4.000081838557388580e-02 1.322042961500579317e+01 +4.084230737261135974e-02 1.320922209727075725e+01 +4.170149859035012646e-02 1.319779252219701249e+01 +4.257876443697566210e-02 1.318613678009289636e+01 +4.347448514473738201e-02 1.317425069736378695e+01 +4.438904894475230456e-02 1.316213003604477372e+01 +4.532285223527567075e-02 1.314977049335589498e+01 +4.627629975351099656e-02 1.313716770128140610e+01 +4.724980475103535832e-02 1.312431722617471763e+01 +4.824378917291415048e-02 1.311121456839070909e+01 +4.925868384058439825e-02 1.309785516194710908e+01 +5.029492863858477103e-02 1.308423437421682856e+01 +5.135297270521476526e-02 1.307034750565304293e+01 +5.243327462720380472e-02 1.305618978954903220e+01 +5.353630263847609233e-02 1.304175639183478808e+01 +5.466253482309622880e-02 1.302704241091245940e+01 +5.581245932248528330e-02 1.301204287753282784e+01 +5.698657454699482955e-02 1.299675275471505032e+01 +5.818538939193249049e-02 1.298116693771197383e+01 +5.940942345813190334e-02 1.296528025402347772e+01 +6.065920727716217487e-02 1.294908746346022888e+01 +6.193528254127624727e-02 1.293258325826050736e+01 +6.323820233819543724e-02 1.291576226326269428e+01 +6.456853139083378046e-02 1.289861903613614658e+01 +6.592684630206468999e-02 1.288114806767331366e+01 +6.731373580463800854e-02 1.286334378214593954e+01 +6.872980101635316197e-02 1.284520053772840065e+01 +7.017565570060108060e-02 1.282671262699114223e+01 +7.165192653238702802e-02 1.280787427746753160e+01 +7.315925336994894002e-02 1.278867965229724213e+01 +7.469828953209137201e-02 1.276912285094953603e+01 +7.626970208135198936e-02 1.274919791002993996e+01 +7.787417211312595844e-02 1.272889880417372410e+01 +7.951239505087173698e-02 1.270821944702988660e+01 +8.118508094752840965e-02 1.268715369233923518e+01 +8.289295479327259131e-02 1.266569533511045265e+01 +8.463675682975017878e-02 1.264383811289787651e+01 +8.641724287091782919e-02 1.262157570718511224e+01 +8.823518463063538542e-02 1.259890174487832226e+01 +9.009137005714826230e-02 1.257580979991346304e+01 +9.198660367460698151e-02 1.255229339498157337e+01 +9.392170693177130658e-02 1.252834600337637738e+01 +9.589751855804898306e-02 1.250396105096866961e+01 +9.791489492702634689e-02 1.247913191831165314e+01 +9.997471042764448368e-02 1.245385194288200381e+01 +1.020778578431849465e-01 1.242811442146095402e+01 +1.042252487382269027e-01 1.240191261266014955e+01 +1.064178138537466567e-01 1.237523973959684298e+01 +1.086565035105267474e-01 1.234808899272318783e+01 +1.109422880210527013e-01 1.232045353281429279e+01 +1.132761581100735493e-01 1.229232649411990508e+01 +1.156591253440116956e-01 1.226370098768441963e+01 +1.180922225694040484e-01 1.223457010484008833e+01 +1.205765043605676046e-01 1.220492692087820252e+01 +1.231130474766825994e-01 1.217476449890312651e+01 +1.257029513284898536e-01 1.214407589387393749e+01 +1.283473384548084739e-01 1.211285415683842537e+01 +1.310473550090756056e-01 1.208109233936426286e+01 +1.338041712561226226e-01 1.204878349817199279e+01 +1.366189820794006526e-01 1.201592069997448142e+01 +1.394930074988789404e-01 1.198249702652729454e+01 +1.424274931998356086e-01 1.194850557989460782e+01 +1.454237110727736315e-01 1.191393948793485436e+01 +1.484829597646951660e-01 1.187879191001030499e+01 +1.516065652419713305e-01 1.184305604292474534e+01 +1.547958813650557874e-01 1.180672512709300470e+01 +1.580522904752858504e-01 1.176979245294604404e+01 +1.613772039940294112e-01 1.173225136757515585e+01 +1.647720630344343684e-01 1.169409528161832768e+01 +1.682383390260503708e-01 1.165531767639185645e+01 +1.717775343525875242e-01 1.161591211126978074e+01 +1.753911830030932539e-01 1.157587223131347720e+01 +1.790808512368259331e-01 1.153519177515342697e+01 +1.828481382621188478e-01 1.149386458312461023e+01 +1.866946769295215469e-01 1.145188460565679556e+01 +1.906221344395247486e-01 1.140924591192039728e+01 +1.946322130651732263e-01 1.136594269872804830e+01 +1.987266508898784534e-01 1.132196929969161481e+01 +2.029072225607562452e-01 1.127732019463371316e+01 +2.071757400578082275e-01 1.123199001925230078e+01 +2.115340534792870542e-01 1.118597357503604606e+01 +2.159840518435803824e-01 1.113926583942777704e+01 +2.205276639079683521e-01 1.109186197623227521e+01 +2.251668590046007890e-01 1.104375734626414385e+01 +2.299036478940632289e-01 1.099494751823048233e+01 +2.347400836368966071e-01 1.094542827984228417e+01 +2.396782624834554731e-01 1.089519564914755456e+01 +2.447203247824814021e-01 1.084424588607813789e+01 +2.498684559087920309e-01 1.079257550420129697e+01 +2.551248872104860399e-01 1.074018128266584071e+01 +2.604918969760713354e-01 1.068706027833169614e+01 +2.659718114219442042e-01 1.063320983807046005e+01 +2.715670057006359661e-01 1.057862761122333595e+01 +2.772799049302738239e-01 1.052331156220160047e+01 +2.831129852456942819e-01 1.046725998321336526e+01 +2.890687748716747607e-01 1.041047150709901459e+01 +2.951498552187366564e-01 1.035294512025646796e+01 +3.013588620020033471e-01 1.029468017563572602e+01 +3.076984863835957040e-01 1.023567640578085758e+01 +3.141714761390565491e-01 1.017593393589597284e+01 +3.207806368483194226e-01 1.011545329691015382e+01 +3.275288331117252039e-01 1.005423543851489576e+01 +3.344189897916232668e-01 9.992281742145758372e+00 +3.414540932800886330e-01 9.929594033878583303e+00 +3.486371927933140102e-01 9.866174597208718922e+00 +3.559714016932251646e-01 9.802026185680421833e+00 +3.634598988369028838e-01 9.737152035331645195e+00 +3.711059299543882362e-01 9.671555876918057848e+00 +3.789128090554795270e-01 9.605241947878369047e+00 +3.868839198661168655e-01 9.538215004001802555e+00 +3.950227172949865584e-01 9.470480330756641862e+00 +4.033327289309792674e-01 9.402043754237714879e+00 +4.118175565721446940e-01 9.332911651689158816e+00 +4.204808777868197511e-01 9.263090961557463032e+00 +4.293264475075890818e-01 9.192589193028748440e+00 +4.383580996587823075e-01 9.121414435003190491e+00 +4.475797488182038930e-01 9.049575364458377180e+00 +4.569953919138289300e-01 8.977081254152750489e+00 +4.666091099561830990e-01 8.903941979619766300e+00 +4.764250698071718637e-01 8.830168025402615584e+00 +4.864475259861144041e-01 8.755770490479589441e+00 +4.966808225137806510e-01 8.680761092829579084e+00 +5.071293947952105752e-01 8.605152173087974532e+00 +5.177977715421475668e-01 8.528956697243236462e+00 +5.286905767359133534e-01 8.452188258325342929e+00 +5.398125316315702804e-01 8.374861077038284307e+00 +5.511684568042553023e-01 8.296990001289945127e+00 +5.627632742385516584e-01 8.218590504574560640e+00 +5.746020094618209395e-01 8.139678683164532913e+00 +5.866897937224063719e-01 8.060271252070968728e+00 +5.990318662136705496e-01 7.980385539734522915e+00 +6.116335763448075280e-01 7.900039481411504916e+00 +6.245003860594324774e-01 7.819251611223043241e+00 +6.376378722029455437e-01 7.738041052839083278e+00 +6.510517289396928753e-01 7.656427508772805979e+00 +6.647477702209878547e-01 7.574431248265357297e+00 +6.787319323050405862e-01 7.492073093745618806e+00 +6.930102763299053059e-01 7.409374405854565637e+00 +7.075889909405460587e-01 7.326357067029132253e+00 +7.224743949711822655e-01 7.243043463646047009e+00 +7.376729401840473876e-01 7.159456466732065927e+00 +7.531912140657712751e-01 7.075619411252983326e+00 +7.690359426825799094e-01 6.991556074000195409e+00 +7.852139935955755323e-01 6.907290650099959350e+00 +8.017323788373290627e-01 6.822847728177279691e+00 +8.185982579510976409e-01 6.738252264212825970e+00 +8.358189410939789177e-01 6.653529554138177815e+00 +8.534018922053364653e-01 6.568705205221422538e+00 +8.713547322418956353e-01 6.483805106301645083e+00 +8.896852424808787685e-01 6.398855396937742412e+00 +9.084013678926384783e-01 6.313882435543097138e+00 +9.275112205842283020e-01 6.228912766584074667e+00 +9.470230833154347216e-01 6.143973086926067140e+00 +9.669454130887559229e-01 6.059090211416608973e+00 +9.872868448149136889e-01 5.974291037800160353e+00 +1.008056195055464510e+00 5.889602511064091495e+00 +1.029262465844161811e+00 5.805051587319562501e+00 +1.050914848588690109e+00 5.720665197325032203e+00 +1.073022728054485730e+00 5.636470209763168704e+00 +1.095595686432367266e+00 5.552493394384648617e+00 +1.118643507491723366e+00 5.468761385134418340e+00 +1.142176180821093290e+00 5.385300643377075147e+00 +1.166203906157933012e+00 5.302137421339071288e+00 +1.190737097809479339e+00 5.219297725885037487e+00 +1.215786389166600756e+00 5.136807282745041725e+00 +1.241362637312629191e+00 5.054691501307909896e+00 +1.267476927729122016e+00 4.972975440093907018e+00 +1.294140579100632404e+00 4.891683773016917591e+00 +1.321365148220553953e+00 4.810840756542981822e+00 +1.349162435000157201e+00 4.730470197847883362e+00 +1.377544487583026944e+00 4.650595424071553019e+00 +1.406523607567063072e+00 4.571239252762122618e+00 +1.436112355336352397e+00 4.492423963596243297e+00 +1.466323555505189447e+00 4.414171271456255319e+00 +1.497170302476649617e+00 4.336502300937906895e+00 +1.528665966118068820e+00 4.259437562355404161e+00 +1.560824197555935600e+00 4.182996929303036460e+00 +1.593658935092667317e+00 4.107199617825145310e+00 +1.627184410247892066e+00 4.032064167238204710e+00 +1.661415153926784960e+00 3.957608422641152313e+00 +1.696366002718184385e+00 3.883849519141865603e+00 +1.732052105325198710e+00 3.810803867819884339e+00 +1.768488929131076581e+00 3.738487143437573490e+00 +1.805692266903237142e+00 3.666914273904008592e+00 +1.843678243638294134e+00 3.596099431488601894e+00 +1.882463323551102574e+00 3.526056025773743663e+00 +1.922064317210805839e+00 3.456796698329010820e+00 +1.962498388827044193e+00 3.388333319082430961e+00 +2.003783063689400290e+00 3.320676984358208017e+00 +2.045936235763373467e+00 3.253838016544010170e+00 +2.088976175446120465e+00 3.187825965345593371e+00 +2.132921537485396701e+00 3.122649610581101687e+00 +2.177791369065045668e+00 3.058316966463028752e+00 +2.223605118060596908e+00 2.994835287311307770e+00 +2.270382641468542406e+00 2.932211074637404380e+00 +2.318144214012909377e+00 2.870450085536038198e+00 +2.366910536932937958e+00 2.809557342318218964e+00 +2.416702746955574188e+00 2.749537143317229759e+00 +2.467542425456745736e+00 2.690393074797078565e+00 +2.519451607815330618e+00 2.632128023891760815e+00 +2.572452792963953350e+00 2.574744192502520068e+00 +2.626568953140651974e+00 2.518243112079977752e+00 +2.681823543845708535e+00 2.462625659217673402e+00 +2.738240514007945325e+00 2.407892071983866789e+00 +2.795844316364856308e+00 2.354041966919083517e+00 +2.854659918061160262e+00 2.301074356627646811e+00 +2.914712811470257403e+00 2.248987667892861531e+00 +2.976029025243368764e+00 2.197779760246779457e+00 +3.038635135591078118e+00 2.147447944927384000e+00 +3.102558277802261788e+00 2.097989004157795545e+00 +3.167826158005278891e+00 2.049399210684434092e+00 +3.234467065176619638e+00 2.001674347513227037e+00 +3.302509883402130253e+00 1.954809727785465467e+00 +3.371984104396250626e+00 1.908800214737421896e+00 +3.442919840284541788e+00 1.863640241690607935e+00 +3.515347836655156932e+00 1.819323832022130993e+00 +3.589299485884867469e+00 1.775844619067515229e+00 +3.664806840745395977e+00 1.733195865911097089e+00 +3.741902628296051248e+00 1.691370485021907344e+00 +3.820620264068547822e+00 1.650361057695893363e+00 +3.900993866550275246e+00 1.610159853267982166e+00 +3.983058271972198749e+00 1.570758848060396451e+00 +4.066849049407931460e+00 1.532149744036233940e+00 +4.152402516190362824e+00 1.494323987130143028e+00 +4.239755753652650228e+00 1.457272785230399048e+00 +4.328946623200303989e+00 1.420987125789319849e+00 +4.420013782721458284e+00 1.385457793041324992e+00 +4.512996703342276206e+00 1.350675384810434032e+00 +4.607935686534884390e+00 1.316630328891144730e+00 +4.704871881585195759e+00 1.283312898988911321e+00 +4.803847303428156579e+00 1.250713230208498006e+00 +4.904904850858283538e+00 1.218821334080417707e+00 +5.008088325123202011e+00 1.187627113117660560e+00 +5.113442448908378068e+00 1.157120374896579262e+00 +5.221012885721184382e+00 1.127290845657565210e+00 +5.330846259682831878e+00 1.098128183422638182e+00 +5.442990175736568403e+00 1.069621990628658814e+00 +5.557493240281039526e+00 1.041761826276146330e+00 +5.674405082237710118e+00 1.014537217595019136e+00 +5.793776374561422138e+00 9.879376712297573970e-01 +5.915658856203576121e+00 9.619526839475401880e-01 +6.040105354537240068e+00 9.365717528739803122e-01 +6.167169808254072016e+00 9.117843852619336165e-01 +6.296907290742852936e+00 8.875801077997397259e-01 +6.429374033959950552e+00 8.639484754659508825e-01 +6.564627452801817142e+00 8.408790799383704107e-01 +6.702726169990284255e+00 8.183615575657363639e-01 +6.843730041481279613e+00 7.963855969109949751e-01 +6.987700182408206118e+00 7.749409458755115443e-01 +7.134698993570936842e+00 7.540174184140631120e-01 +7.284790188482136664e+00 7.336049008507202762e-01 +7.438038820982544763e+00 7.136933578060753458e-01 +7.594511313437116051e+00 6.942728377465137157e-01 +7.754275485524479095e+00 6.753334781663793418e-01 +7.917400583631875044e+00 6.568655104141041257e-01 +8.083957310868562729e+00 6.388592641734185396e-01 +8.254017857710513439e+00 6.213051716108696221e-01 +8.427655933289944556e+00 6.041937712008634831e-01 +8.604946797342901021e+00 5.875157112395151815e-01 +8.785967292829035102e+00 5.712617530584842562e-01 +8.970795879237474679e+00 5.554227739499765226e-01 +9.159512666593535357e+00 5.399897698139519697e-01 +9.352199450180634699e+00 5.249538575385309658e-01 +9.548939745992740313e+00 5.103062771244121221e-01 +9.749818826932624916e+00 4.960383935639869901e-01 +9.954923759771510561e+00 4.821416984856828902e-01 +1.016434344288643743e+01 4.686078115738468997e-01 +1.037816864479128931e+01 4.554284817743470826e-01 +1.059649204347852880e+01 4.425955882958021448e-01 +1.081940826658841104e+01 4.301011414161844959e-01 +1.104701393242346086e+01 4.179372831042684755e-01 +1.127940769182553105e+01 4.060962874652201404e-01 +1.151669027093393893e+01 3.945705610193344093e-01 +1.175896451484310035e+01 3.833526428227110872e-01 +1.200633543217844412e+01 3.724352044384152127e-01 +1.225891024061033008e+01 3.618110497663900582e-01 +1.251679841332516041e+01 3.514731147401841116e-01 +1.278011172647427074e+01 3.414144668982542763e-01 +1.304896430762081039e+01 3.316283048373933617e-01 +1.332347268520607919e+01 3.221079575555364505e-01 +1.360375583905619656e+01 3.128468836909991113e-01 +1.388993525195143341e+01 3.038386706649025548e-01 +1.418213496228023018e+01 2.950770337333294413e-01 +1.448048161780117660e+01 2.865558149554760514e-01 +1.478510453053567275e+01 2.782689820838640249e-01 +1.509613573281550813e+01 2.702106273823998706e-01 +1.541371003450950816e+01 2.623749663778575103e-01 +1.573796508145387740e+01 2.547563365501267496e-01 +1.606904141511205708e+01 2.473491959663308981e-01 +1.640708253348930157e+01 2.401481218637218085e-01 +1.675223495332892654e+01 2.331478091860172963e-01 +1.710464827361671780e+01 2.263430690776578325e-01 +1.746447524042166677e+01 2.197288273402303871e-01 +1.783187181310034219e+01 2.133001228551371919e-01 +1.820699723189422414e+01 2.070521059763620331e-01 +1.859001408694902224e+01 2.009800368970179330e-01 +1.898108838878578908e+01 1.950792839931695666e-01 +1.938038964025485811e+01 1.893453221482389037e-01 +1.978809091000311682e+01 1.837737310611473607e-01 +2.020436890748701941e+01 1.783601935411601447e-01 +2.062940405956337031e+01 1.731004937922558662e-01 +2.106338058869177488e+01 1.679905156896678020e-01 +2.150648659278177277e+01 1.630262410511181814e-01 +2.195891412671997145e+01 1.582037479050975581e-01 +2.242085928561198571e+01 1.535192087584186238e-01 +2.289252228977596459e+01 1.489688888651254373e-01 +2.337410757152364837e+01 1.445491444987258012e-01 +2.386582386376725751e+01 1.402564212295748236e-01 +2.436788429049040516e+01 1.360872522091263748e-01 +2.488050645912193914e+01 1.320382564626565447e-01 +2.540391255485358357e+01 1.281061371919426650e-01 +2.593832943694117077e+01 1.242876800892934452e-01 +2.648398873703208167e+01 1.205797516642080208e-01 +2.704112695956085233e+01 1.169792975838590543e-01 +2.760998558425730920e+01 1.134833410284912247e-01 +2.819081117081062615e+01 1.100889810627529303e-01 +2.878385546573548837e+01 1.067933910238814399e-01 +2.938937551148604399e+01 1.035938169275934795e-01 +3.000763375786584675e+01 1.004875758924468065e-01 +3.063889817578091268e+01 9.747205458337872275e-02 +3.128344237338607670e+01 9.454470767504784268e-02 +3.194154571467470660e+01 9.170305633554598490e-02 +3.261349344056282717e+01 8.894468673098619715e-02 +3.329957679252112968e+01 8.626724855140863180e-02 +3.400009313880713790e+01 8.366845355839978848e-02 +3.471534610335330484e+01 8.114607415475948160e-02 +3.544564569736607496e+01 7.869794197650700762e-02 +3.619130845369407723e+01 7.632194650746476627e-02 +3.695265756402228163e+01 7.401603371662252018e-02 +3.773002301895265020e+01 7.177820471843512906e-02 +3.852374175103162202e+01 6.960651445617276056e-02 +3.933415778078607161e+01 6.749907040840608108e-02 +4.016162236583215162e+01 6.545403131866973923e-02 +4.100649415312015122e+01 6.346960594832315450e-02 +4.186913933438253821e+01 6.154405185259008426e-02 +4.274993180485166988e+01 5.967567417973534960e-02 +4.364925332531726099e+01 5.786282449330629185e-02 +4.456749368759210483e+01 5.610389961734859460e-02 +4.550505088345928328e+01 5.439734050447644215e-02 +4.646233127717285072e+01 5.274163112666125502e-02 +4.743974978158835398e+01 5.113529738857922752e-02 +4.843773003799763188e+01 4.957690606334529126e-02 +4.945670459974722633e+01 4.806506375043947260e-02 +5.049711511971953826e+01 4.659841585561916993e-02 +5.155941254175748867e+01 4.517564559259694412e-02 +5.264405729611715401e+01 4.379547300624826028e-02 +5.375151949903109028e+01 4.245665401710645159e-02 +5.488227915647042465e+01 4.115797948688688923e-02 +5.603682637219282014e+01 3.989827430477699277e-02 +5.721566156016819349e+01 3.867639649421608650e-02 +5.841929566147207709e+01 3.749123633988689364e-02 +5.964825036574242745e+01 3.634171553462835907e-02 +6.090305833729441076e+01 3.522678634597806169e-02 +6.218426344599317446e+01 3.414543080204346381e-02 +6.349242100298222624e+01 3.309665989640138417e-02 +6.482809800137133038e+01 3.207951281171741303e-02 +6.619187336198780258e+01 3.109305616177626685e-02 +6.758433818429689666e+01 3.013638325161161965e-02 +6.900609600260210641e+01 2.920861335542032991e-02 +7.045776304763377595e+01 2.830889101194789328e-02 +7.193996851364148881e+01 2.743638533702798601e-02 +7.345335483110439156e+01 2.659028935296113214e-02 +7.499857794517987486e+01 2.576981933441526221e-02 +7.657630760000850501e+01 2.497421417053449083e-02 +7.818722762900056011e+01 2.420273474294054941e-02 +7.983203625122925473e+01 2.345466331931413129e-02 +8.151144637405830906e+01 2.272930296224503002e-02 +8.322618590213745904e+01 2.202597695304035069e-02 +8.497699805289666131e+01 2.134402823018486176e-02 +8.676464167867823107e+01 2.068281884214731234e-02 +8.858989159564454496e+01 2.004172941423120607e-02 +9.045353891960670012e+01 1.942015862916968150e-02 +9.235639140891622390e+01 1.881752272116915883e-02 +9.429927381457103763e+01 1.823325498310796738e-02 +9.628302823768561325e+01 1.766680528660070221e-02 +9.830851449448286417e+01 1.711763961464141809e-02 +1.003766104889627115e+02 1.658523960654426782e-02 +1.024882125934100969e+02 1.606910211490249207e-02 +1.046442360369107547e+02 1.556873877429017748e-02 +1.068456153020347585e+02 1.508367558143823561e-02 +1.090933045298714035e+02 1.461345248661503719e-02 +1.113882779335785642e+02 1.415762299595223032e-02 +1.137315302206357899e+02 1.371575378445501550e-02 +1.161240770239806892e+02 1.328742431944384544e-02 +1.185669553422106048e+02 1.287222649417853713e-02 +1.210612239890490258e+02 1.246976427141775756e-02 +1.236079640522673486e+02 1.207965333667366213e-02 +1.262082793622560786e+02 1.170152076092563498e-02 +1.288632969704605387e+02 1.133500467255886719e-02 +1.315741676378739271e+02 1.097975393830195358e-02 +1.343420663338116299e+02 1.063542785293809280e-02 +1.371681927451785157e+02 1.030169583757118909e-02 +1.400537717964437832e+02 9.978237146232677920e-03 +1.430000541805629553e+02 9.664740580616596258e-03 +1.460083169010602830e+02 9.360904212738545283e-03 +1.490798638255203628e+02 9.066435115314769311e-03 +1.522160262507237292e+02 8.781049099664130220e-03 +1.554181634796656795e+02 8.504470460939670198e-03 +1.586876634107187840e+02 8.236431730499260695e-03 +1.620259431391902467e+02 7.976673435230261650e-03 +1.654344495715269545e+02 7.724943863647405068e-03 +1.689146600524518362e+02 7.480998838585106725e-03 +1.724680830052826934e+02 7.244601496312370360e-03 +1.760962585857283500e+02 7.015522176256188194e-03 +1.798007593494386072e+02 6.793537795859833431e-03 +1.835831909335900036e+02 6.578432271628248049e-03 +1.874451927528209580e+02 6.369995800376245139e-03 +1.913884387097961906e+02 6.168025198314851243e-03 +1.954146379207263635e+02 5.972323076423545311e-03 +1.995255354561505499e+02 5.782698085010593361e-03 +2.037229130972954465e+02 5.598964632373067650e-03 +2.080085901083527631e+02 5.420942714530118008e-03 +2.123844240250032556e+02 5.248457749849708143e-03 +2.168523114595202514e+02 5.081340418435627455e-03 +2.214141889228227740e+02 4.919426506143693111e-03 +2.260720336638089805e+02 4.762556753101163283e-03 +2.308278645263553130e+02 4.610576706604645365e-03 +2.356837428243442503e+02 4.463336578276114533e-03 +2.406417732350906533e+02 4.320691105359555050e-03 +2.457041047115768322e+02 4.182499416042762853e-03 +2.508729314138640234e+02 4.048624898693321816e-03 +2.561504936601071449e+02 3.918935074899033283e-03 +2.615390788975763599e+02 3.793301476206934354e-03 +2.670410226940953748e+02 3.671599524457622513e-03 +2.726587097503518748e+02 3.553708415613535381e-03 +2.783945749334881157e+02 3.439511006983698906e-03 +2.842511043324453226e+02 3.328893707748777856e-03 +2.902308363355099914e+02 3.221746372693525071e-03 +2.963363627305171235e+02 3.117962199056223491e-03 +3.025703298282091396e+02 3.017437626406398090e-03 +3.089354396092269894e+02 2.920072239465136641e-03 +3.154344508952173101e+02 2.825768673784555696e-03 +3.220701805445953028e+02 2.734432524204508542e-03 +3.288455046734433722e+02 2.645972256007910091e-03 +3.357633599021060036e+02 2.560299118697101448e-03 +3.428267446280100330e+02 2.477327062316445061e-03 +3.500387203252462882e+02 2.396972656248327633e-03 +3.574024128715121833e+02 2.319155010411133155e-03 +3.649210139029466404e+02 2.243795698790651029e-03 +3.725977821974814219e+02 2.170818685237276783e-03 +3.804360450872937918e+02 2.100150251463877345e-03 +3.884391999009588972e+02 2.031718927180853427e-03 +3.966107154359540914e+02 1.965455422306367460e-03 +4.049541334621405895e+02 1.901292561191752963e-03 +4.134730702568568290e+02 1.839165218803814721e-03 +4.221712181723303843e+02 1.779010258806867787e-03 +4.310523472360380310e+02 1.720766473489673126e-03 +4.401203067847487205e+02 1.664374525483260786e-03 +4.493790271329426105e+02 1.609776891217568756e-03 +4.588325212763095919e+02 1.556917806066275469e-03 +4.684848866311119195e+02 1.505743211130235182e-03 +4.783403068101089275e+02 1.456200701611920631e-03 +4.884030534358609543e+02 1.408239476734021766e-03 +4.986774879921792376e+02 1.361810291157063927e-03 +5.091680637145066726e+02 1.316865407852131371e-03 +5.198793275200839616e+02 1.273358552385793832e-03 +5.308159219787205529e+02 1.231244868575811580e-03 +5.419825873250035784e+02 1.190480875477343235e-03 +5.533841635128684402e+02 1.151024425660267941e-03 +5.650255923133593114e+02 1.112834664739781928e-03 +5.769119194565408861e+02 1.075871992123088924e-03 +5.890482968184702486e+02 1.040098022936334840e-03 +6.014399846541518855e+02 1.005475551096957163e-03 +6.140923538775020916e+02 9.719685134973791952e-04 +6.270108883892410176e+02 9.395419555721918042e-04 +6.402011874537798803e+02 9.081619963978283892e-04 +6.536689681261112810e+02 8.777957978156107838e-04 +6.674200677297295670e+02 8.484124389809582405e-04 +6.814604463867024151e+02 8.199793114724965347e-04 +6.957961896009678640e+02 7.924674770505269437e-04 +7.104335108959464833e+02 7.658480721657819935e-04 +7.253787545076849028e+02 7.400935827299400839e-04 +7.406383981346091332e+02 7.151721140338816512e-04 +7.562190557451542645e+02 6.910609058500863074e-04 +7.721274804444580013e+02 6.677391882724353868e-04 +7.883705674013287990e+02 6.451765429361022361e-04 +8.049553568368359038e+02 6.233498019069456161e-04 +8.218890370757208075e+02 6.022358488320405821e-04 +8.391789476620338064e+02 5.818121802495077217e-04 +8.568325825403134104e+02 5.620569603828427102e-04 +8.748575933036578363e+02 5.429490183776058965e-04 +8.932617925101543506e+02 5.244678337169872981e-04 +9.120531570690787930e+02 5.065935186063269969e-04 +9.312398316982915958e+02 4.893067996554970999e-04 +9.508301324544219142e+02 4.725889996108232293e-04 +9.708325503372604999e+02 4.564220194114960668e-04 +9.912557549700130721e+02 4.407883206773271492e-04 +1.012108598356977495e+03 4.256709086686605065e-04 +1.033400118720228875e+03 4.110533157307862790e-04 +1.055139544417076650e+03 3.969195852220832221e-04 +1.077336297939872566e+03 3.832542559190025427e-04 +1.099999999999999773e+03 3.700423468878129133e-04 diff --git a/app/host/SBI/run_sbi.py b/app/host/SBI/run_sbi.py new file mode 100644 index 00000000..0b59f42d --- /dev/null +++ b/app/host/SBI/run_sbi.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# D. Jones - 5/26/23 +"""SED-fitting implementation with SBI++""" +import os +import signal +import sys +import time +import warnings + +from host.models import SEDFittingResult +from host.prospector import build_model +from host.prospector import build_obs + +# recommend running the full script without the line below first +# if an error is threw, then uncomment this line +os.environ["KMP_DUPLICATE_LIB_OK"] = "True" +import numpy as np +from numpy.random import normal, uniform +from scipy.interpolate import interp1d + +# torch +import torch +import torch.nn as nn +import torch.nn.functional as F +from sbi import utils as Ut +from sbi import inference as Inference +from host.prospector import build_obs +from host.models import Transient, Filter +from django.db.models import Q +from host.SBI.train_sbi import maggies_to_asinh + +# plot +import corner +import matplotlib.pyplot as plt +import matplotlib_inline.backend_inline + +matplotlib_inline.backend_inline.set_matplotlib_formats("retina") + +# all the functions implementing SBI++ are contained in `sbi_pp.py` +from host.SBI import sbi_pp +import h5py + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + +run_params = { + "nmc": 50, # number of MC samples + "nposterior": 50, # number of posterior samples per MC drawn + "np_baseline": 500, # number of posterior samples used in baseline SBI + "ini_chi2": 5, # chi^2 cut usedi in the nearest neighbor search + "max_chi2": 500, # the maximum chi^2 to reach in case we Incremently increase the chi^2 + # in the case of insufficient neighbors + "noisy_sig": 3, # deviation from the noise model, above which the measuremnt is taked as OOD + "tmax_per_obj": 1200, # max time spent on one object / mc sample in secs + "tmax_all": 40, # max time spent on all mc samples in mins + "outdir": "output", # output directory + "verbose": True, + "tmax_per_iter": 20, +} + +sbi_params = { + "anpe_fname": "../data/sbipp/SBI_model.pt", # trained sbi model + "train_fname": "../data/sbipp/sbi_phot.h5", # training set + # "test_fname": "host/SBI/test.i20.npz", # testing set + # "noise_fname": "host/SBI/toy_noise_xy.i20.txt", # toy model used in training + # this files contains the median and the 1 sigma values + # of uncertainties in magnitude bins + "nhidden": 500, # architecture of the trained density estimator + "nblocks": 15, # architecture of the trained density estimator +} + +all_filters = Filter.objects.filter(~Q(name="DES_i") & ~Q(name="DES_Y")) + + +def main(): + + # training set + data = h5py.File(sbi_params["train_fname"], "r") + x_train = np.array(data["theta"]) # physical parameters + y_train = np.array(data["phot"]) # fluxes & uncertainties + + # we will only need the lower & upper limits to be passed to sbi as "priors" + # here we simply read in the bounds from the training set + prior_low = sbi_pp.prior_from_train("ll", x_train=x_train) + prior_high = sbi_pp.prior_from_train("ul", x_train=x_train) + lower_bounds = torch.tensor(prior_low).to(device) + upper_bounds = torch.tensor(prior_high).to(device) + prior = Ut.BoxUniform(low=lower_bounds, high=upper_bounds, device=device) + + # density estimater + anpe = Inference.SNPE( + prior=prior, + density_estimator=Ut.posterior_nn( + "maf", + hidden_features=sbi_params["nhidden"], + num_transforms=sbi_params["nblocks"], + ), + device=device, + ) + x_tensor = torch.as_tensor(x_train.astype(np.float32)).to(device) + y_tensor = torch.as_tensor(y_train.astype(np.float32)).to(device) + anpe.append_simulations(x_tensor, y_tensor) + p_x_y_estimator = anpe._build_neural_net(x_tensor, y_tensor) + p_x_y_estimator.load_state_dict( + torch.load(sbi_params["anpe_fname"], map_location=torch.device(device)) + ) + anpe._x_shape = Ut.x_shape_from_simulation(y_tensor) + hatp_x_y = anpe.build_posterior(p_x_y_estimator, sample_with="rejection") + + # toy noise model + meds_sigs, stds_sigs = [], [] + for f in all_filters: + toy_noise_x, toy_noise_y = np.loadtxt( + f"host/SBI/snrfiles/{f}_magvsnr.txt", dtype=float, unpack=True + ) + meds_sigs += [ + interp1d( + toy_noise_x, + 1.0857 * 1 / toy_noise_y, + kind="slinear", + fill_value="extrapolate", + ) + ] + stds_sigs += [ + interp1d( + toy_noise_x, + 1.0857 * 1 / toy_noise_y, + kind="slinear", + fill_value="extrapolate", + ) + ] + + # prepare to pass the reconstructed model to sbi_pp + sbi_params["y_train"] = y_train + sbi_params["hatp_x_y"] = hatp_x_y + sbi_params["toynoise_meds_sigs"] = meds_sigs + sbi_params["toynoise_stds_sigs"] = stds_sigs + + for transient_name in np.loadtxt( + "host/slurm/hostgalmasses_list.csv", + unpack=True, + usecols=[0], + delimiter=",", + dtype=str, + )[36:]: + print(transient_name) + + np.random.seed(200) # make results reproducible + pobs = build_obs(Transient.objects.get(name=transient_name), "global") + + # a testing object of which the noises are OOD + mags, mags_unc, filternames = np.array([]), np.array([]), np.array([]) + for f in all_filters: + if f.name in pobs["filternames"]: + iflt = np.array(pobs["filternames"]) == f.name + mags = np.append(mags, maggies_to_asinh(pobs["maggies"][iflt])) + mags_unc = np.append( + mags_unc, + 2.5 + / np.log(10) + * pobs["maggies_unc"][iflt] + / pobs["maggies"][iflt], + ) + else: + mags = np.append(mags, np.nan) + mags_unc = np.append(mags_unc, np.nan) + filternames = np.append(filternames, f.name) + + obs = {} + obs["bands"] = filternames + obs["mags"] = mags ##np.array([maggies_to_asinh(p) for p in pobs['maggies']]) + obs["mags_unc"] = mags_unc ##2.5/np.log(10)*pobs['maggies_unc']/pobs['maggies'] + obs["redshift"] = pobs["redshift"] + ###obs["mags"][np.where(filternames == 'WISE_W3')[0][0]] = np.nan + # Run SBI++ + # signal.alarm(0) + # signal.alarm(600) + # try: + t = time.time() + chain, obs, flags = sbi_pp.sbi_pp( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + print(f"SBI took {time.time()-t:.0f} seconds") + + transient = Transient.objects.get(name=transient_name) + observations = build_obs(transient, "global") + model_components = build_model(observations) + theta_max = np.mean(chain[:, 1:], axis=0) + _, _, mfrac = model_components["model"].predict( + theta_max, obs=observations, sps=model_components["sps"] + ) + + signal.alarm(0) + with open("masses.txt", "a") as fout: + try: + print( + transient_name, + np.mean(chain[:, 1]), + mfrac, + SEDFittingResult.objects.get( + transient__name=transient_name, aperture__type="global" + ).log_mass_50, + file=fout, + ) + except Exception as e: + print(transient_name, np.mean(chain[:, 1]), mfrac, None, file=fout) diff --git a/app/host/SBI/run_sbi_blast.py b/app/host/SBI/run_sbi_blast.py new file mode 100644 index 00000000..3fc1bc60 --- /dev/null +++ b/app/host/SBI/run_sbi_blast.py @@ -0,0 +1,229 @@ +import os +import signal +import sys +import time +import warnings + +from django.conf import settings + +os.environ["KMP_DUPLICATE_LIB_OK"] = "True" +import numpy as np +import math +from numpy.random import normal, uniform +from scipy.interpolate import interp1d + +# torch +import torch +import torch.nn as nn +import torch.nn.functional as F +from sbi import utils as Ut +from sbi import inference as Inference +from host.models import Transient, Filter +from django.db.models import Q + +# all the functions implementing SBI++ are contained in `sbi_pp.py` +from host.SBI import sbi_pp +import h5py + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + +run_params = { + "nmc": 50, # number of MC samples + "nposterior": 50, # number of posterior samples per MC drawn + "np_baseline": 500, # number of posterior samples used in baseline SBI + "ini_chi2": 5, # chi^2 cut usedi in the nearest neighbor search + "max_chi2": 5000, # the maximum chi^2 to reach in case we Incremently increase the chi^2 + # in the case of insufficient neighbors + "noisy_sig": 3, # deviation from the noise model, above which the measuremnt is taked as OOD + "tmax_per_obj": 120000, # max time spent on one object / mc sample in secs + "tmax_all": 600000, # max time spent on all mc samples in mins + "outdir": "output", # output directory + "verbose": True, + "tmax_per_iter": 60, +} + +sbi_params = { + "anpe_fname_global": f"{settings.SBIPP_ROOT}/SBI_model_global.pt", # trained sbi model + "train_fname_global": f"{settings.SBIPP_PHOT_ROOT}/sbi_phot_global.h5", # training set + "anpe_fname_local": f"{settings.SBIPP_ROOT}/SBI_model_local.pt", # trained sbi model + "train_fname_local": f"{settings.SBIPP_PHOT_ROOT}/sbi_phot_local.h5", # training set + "nhidden": 500, # architecture of the trained density estimator + "nblocks": 15, # architecture of the trained density estimator +} + +all_filters = Filter.objects.filter(~Q(name="DES_i") & ~Q(name="DES_Y")) +uv_filters = ["GALEX_NUV", "GALEX_FUV", "SDSS_u", "DES_u"] +opt_filters = [ + "SDSS_g", + "SDSS_r", + "SDSS_i", + "SDSS_z", + "PanSTARRS_g", + "PanSTARRS_r", + "PanSTARRS_i", + "PanSTARRS_z", + "PanSTARRS_y", + "DES_g", + "DES_r", +] +ir_filters = [ + "WISE_W1", + "WISE_W2", + "WISE_W3", + "WISE_W4", + "2MASS_J", + "2MASS_H", + "2MASS_K", +] + +# training set +### --- GLOBAL --- ### +for _fit_type in ["global", "local"]: + data = h5py.File(sbi_params[f"train_fname_{_fit_type}"], "r") + x_train = np.array(data["theta"]) # physical parameters + y_train = np.array(data["phot"]) # fluxes & uncertainties + + # we will only need the lower & upper limits to be passed to sbi as "priors" + # here we simply read in the bounds from the training set + prior_low = sbi_pp.prior_from_train("ll", x_train=x_train) + prior_high = sbi_pp.prior_from_train("ul", x_train=x_train) + lower_bounds = torch.tensor(prior_low).to(device) + upper_bounds = torch.tensor(prior_high).to(device) + prior = Ut.BoxUniform(low=lower_bounds, high=upper_bounds, device=device) + + # density estimater + anpe = Inference.SNPE( + prior=prior, + density_estimator=Ut.posterior_nn( + "maf", + hidden_features=sbi_params["nhidden"], + num_transforms=sbi_params["nblocks"], + ), + device=device, + ) + x_tensor = torch.as_tensor(x_train.astype(np.float32)).to(device) + y_tensor = torch.as_tensor(y_train.astype(np.float32)).to(device) + anpe.append_simulations(x_tensor, y_tensor) + p_x_y_estimator = anpe._build_neural_net(x_tensor, y_tensor) + p_x_y_estimator.load_state_dict( + torch.load( + sbi_params[f"anpe_fname_{_fit_type}"], map_location=torch.device(device) + ) + ) + anpe._x_shape = Ut.x_shape_from_simulation(y_tensor) + if _fit_type == "global": + hatp_x_y_global = anpe.build_posterior(p_x_y_estimator, sample_with="rejection") + y_train_global = y_train[:] + x_train_global = x_train[:] + elif _fit_type == "local": + hatp_x_y_local = anpe.build_posterior(p_x_y_estimator, sample_with="rejection") + y_train_local = y_train[:] + x_train_local = x_train[:] + + +def maggies_to_asinh(x): + """asinh magnitudes""" + a = 2.50 * np.log10(np.e) + mu = 35.0 + return -a * math.asinh((x / 2.0) * np.exp(mu / a)) + mu + + +def fit_sbi_pp(observations, n_filt_cuts=True, fit_type="global"): + np.random.seed(100) # make results reproducible + + # toy noise model + meds_sigs, stds_sigs = [], [] + + for f in all_filters: + toy_noise_x, toy_noise_y = np.loadtxt( + f"host/SBI/snrfiles/{f.name}_magvsnr.txt", dtype=float, unpack=True + ) + meds_sigs += [ + interp1d( + toy_noise_x, + 1.0857 * 1 / toy_noise_y, + kind="slinear", + fill_value="extrapolate", # (0.01,1.0), + bounds_error=False, + ) + ] + stds_sigs += [ + interp1d( + toy_noise_x, + 1.0857 * 1 / toy_noise_y, + kind="slinear", + fill_value="extrapolate", # (0.01,1.0), + bounds_error=False, + ) + ] + sbi_params["toynoise_meds_sigs"] = meds_sigs + sbi_params["toynoise_stds_sigs"] = stds_sigs + + # a testing object of which the noises are OOD + mags, mags_unc, filternames, wavelengths = ( + np.array([]), + np.array([]), + np.array([]), + np.array([]), + ) + + has_uv, has_opt, has_ir = False, False, False + for f in all_filters: + if f.name in observations["filternames"]: + iflt = np.array(observations["filternames"]) == f.name + mags = np.append(mags, maggies_to_asinh(observations["maggies"][iflt])) + mags_unc = np.append( + mags_unc, + 2.5 + / np.log(10) + * observations["maggies_unc"][iflt] + / observations["maggies"][iflt], + ) + if f.name in uv_filters: + has_uv = True + elif f.name in opt_filters: + has_opt = True + elif f.name in ir_filters: + has_ir = True + else: + mags = np.append(mags, np.nan) + mags_unc = np.append(mags_unc, np.nan) + filternames = np.append(filternames, f.name) + wavelengths = np.append(wavelengths, f.transmission_curve().wave_effective) + + obs = {} + obs[ + "mags" + ] = mags ##np.array([maggies_to_asinh(p) for p in observations['maggies']]) + obs[ + "mags_unc" + ] = mags_unc ##2.5/np.log(10)*observations['maggies_unc']/observations['maggies'] + obs["redshift"] = observations["redshift"] + obs["wavelengths"] = wavelengths + obs["filternames"] = filternames + + if n_filt_cuts and not has_opt and (not has_ir or not has_uv): + print("not enough filters for reliable/fast inference") + return {}, 1 + + # prepare to pass the reconstructed model to sbi_pp + if fit_type == "global": + sbi_params["y_train"] = y_train_global + sbi_params["theta_train"] = x_train_global + sbi_params["hatp_x_y"] = hatp_x_y_global + elif fit_type == "local": + sbi_params["y_train"] = y_train_local + sbi_params["hatp_x_y"] = hatp_x_y_local + sbi_params["theta_train"] = x_train_local + + # Run SBI++ + chain, obs, flags = sbi_pp.sbi_pp( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + + # pathological format as we're missing some stuff that prospector usually spits out + output = {"sampling": [{"samples": chain[:, :], "eff": 100}, 0]} + return output, 0 diff --git a/app/host/SBI/sbi.py b/app/host/SBI/sbi.py new file mode 100644 index 00000000..4e570b45 --- /dev/null +++ b/app/host/SBI/sbi.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# D. Jones - 5/26/23 +"""SED-fitting implementation with SBI++""" diff --git a/app/host/SBI/sbi_pp.py b/app/host/SBI/sbi_pp.py new file mode 100644 index 00000000..c5a25f4c --- /dev/null +++ b/app/host/SBI/sbi_pp.py @@ -0,0 +1,990 @@ +# $ conda activate sbi_env +import copy +import os +import signal +import sys +import time +import warnings + +import pylab as plt +from astropy.stats import sigma_clipped_stats +from prospect.fitting import fit_model as fit_model_prospect +from prospect.fitting import lnprobfn +from prospect.io import write_results as writer +from prospect.io.write_results import write_h5_header +from prospect.io.write_results import write_obs_to_h5 +from prospect.models import priors +from prospect.models import SpecModel +from prospect.models.sedmodel import PolySpecModel +from prospect.models.templates import TemplateLibrary +from prospect.models.transforms import logsfr_ratios_to_sfrs +from prospect.models.transforms import zred_to_agebins +from prospect.sources import CSPSpecBasis +from prospect.sources import FastStepBasis + +os.environ["KMP_DUPLICATE_LIB_OK"] = "True" +from operator import itemgetter +import numpy as np +from numpy.random import normal, uniform +from scipy import stats +from scipy.interpolate import interp1d + +# torch +import torch + +torch.set_num_threads(1) +import torch.nn as nn +import torch.nn.functional as F +from sbi import utils as Ut +from sbi import inference as Inference + +from host.models import AperturePhotometry, Transient, Filter +from django.db.models import Q +from host.host_utils import get_dust_maps +import extinction +from prospect.utils.obsutils import fix_obs +from host.photometric_calibration import mJy_to_maggies +from astropy.cosmology import WMAP9 as cosmo + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + +# prior +def prior_from_train(ll_or_ul, x_train): + """We will only need the lower & upper limits to be passed to sbi as 'priors' + Note that I have not checked how this affects the sbi functions that are not used in the script. + Here we simply read in the bounds from the training set + """ + + assert ll_or_ul in ["ll", "ul"] + + if ll_or_ul == "ll": + res = [] + for i in range(x_train.shape[1]): + res.append(np.min(x_train.T[i])) + else: + res = [] + for i in range(x_train.shape[1]): + res.append(np.max(x_train.T[i])) + + return res + + +def toy_noise(flux, meds_sigs, stds_sigs, verbose=False, **extra): + """toy noise; must be the same as the noise model used when generating the training set + Here we use assume Gaussian noises + flux: photometry + meds_sigs: median of the magnitude bin + stds_sigs: 1 standard deviation + """ + return flux, meds_sigs(flux), np.clip(stds_sigs(flux), a_min=0.001, a_max=None) + + +# the following functions are used to set the max time spent per object +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + raise TimeoutException + + +def absdiff(mags, obsphot, obsphot_unc): + """abs difference in photometry""" + + return np.abs(mags - obsphot) + + +def chi2dof(mags, obsphot, obsphot_unc, individual=False): + """reduced chi^2""" + + if individual: + return ((mags - obsphot) / obsphot_unc) ** 2 + else: + chi2 = np.nansum(((mags - obsphot) / obsphot_unc) ** 2, axis=1) + return chi2 / np.sum(np.isfinite(obsphot)) + + +def chidof(mags, obsphot, obsphot_unc, individual=False): + """reduced chi^2""" + + if individual: + return (mags - obsphot) / obsphot_unc + else: + chi = np.nansum((mags - obsphot) / obsphot_unc, axis=1) + return chi / np.sum(np.isfinite(obsphot)) + + +def gauss_approx_missingband(obs, run_params, sbi_params, max_neighbors=200): + """nearest neighbor approximation of missing bands; + see sec. 4.1.2 for details + """ + use_res = False + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + invalid_mask = np.copy(obs["missing_mask"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + y_obs_valid_only = y_obs[~invalid_mask] + valid_idx = np.where(~invalid_mask)[0] + not_valid_idx = np.where(invalid_mask)[0] + + look_in_training = y_train[:, valid_idx] + chi2_nei = chi2dof( + mags=look_in_training, obsphot=y_obs[valid_idx], obsphot_unc=sig_obs[valid_idx] + ) + + _chi2_thres = run_params["ini_chi2"] * 1 + cnt = 0 + use_res = True + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + cnt += 1 + + if _chi2_thres > run_params["max_chi2"]: + use_res = False + chi2_selected = y_train[:, valid_idx] + chi2_selected = chi2_selected[:max_neighbors] + guess_ndata = y_train[:, not_valid_idx] + guess_ndata = guess_ndata[:max_neighbors] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + print( + "_chi2_thres {} > max_chi2 {}".format( + _chi2_thres, run_params["max_chi2"] + ), + len(guess_ndata), + ) + else: + chi2_selected = y_train[:, valid_idx][idx_chi2_selected] + # get distribution of the missing band + guess_ndata = y_train[:, not_valid_idx][idx_chi2_selected] + dists = np.linalg.norm(y_obs_valid_only - chi2_selected, axis=1) + neighs_weights = 1 / dists + + kdes = [] + for i in range(guess_ndata.shape[1]): + _kde = stats.gaussian_kde(guess_ndata.T[i], 0.2, weights=neighs_weights) + kdes.append(_kde) + + return kdes, use_res, idx_chi2_selected + + +def sbi_missingband(obs, run_params, sbi_params, seconditer=False): + """used when observations have missing data; + see sec. 4.1.2 of for details + """ + + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("sbi missing bands") + ave_theta = [] + + max_neighbors = 200 + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + invalid_mask = np.copy(obs["missing_mask"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + y_obs_valid_only = y_obs[~invalid_mask] + valid_idx = np.where(~invalid_mask)[0] + not_valid_idx = np.where(invalid_mask)[0] + st = time.time() + + # ------------------------------------------------ + # nearest neighbor approximation of missing bands; + # see sec. 4.1 for details + look_in_training = y_train[:, valid_idx] + chi2_nei = chi2dof( + mags=look_in_training, obsphot=y_obs[valid_idx], obsphot_unc=sig_obs[valid_idx] + ) + chi_nei = chidof( + mags=look_in_training, obsphot=y_obs[valid_idx], obsphot_unc=sig_obs[valid_idx] + ) + + _chi2_thres = run_params["ini_chi2"] * 1 + cnt = 0 + use_res = True + while _chi2_thres <= run_params["max_chi2"]: + # idx_chi2_selected = np.where(chi2_nei[redshift_idx] <= _chi2_thres)[0] + # if len(idx_chi2_selected) >= 100 and np.abs(np.median(chi_nei[redshift_idx][idx_chi2_selected])) < 0.25: + + # let's not allow any matches with giant overall offsets + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[ + 0 + ] # & (np.abs(chi_nei) < 0.25))[0] + if ( + len(idx_chi2_selected) >= max_neighbors + ): # and np.abs(np.median(chi_nei[idx_chi2_selected])) < 0.25: + break + else: + _chi2_thres += 5 + cnt += 1 + + if _chi2_thres > run_params["max_chi2"]: + use_res = False + chi2_selected = y_train[:, valid_idx] + chi2_selected = chi2_selected[:max_neighbors] + guess_ndata = y_train[:, not_valid_idx] + guess_ndata = guess_ndata[:max_neighbors] + + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + print( + "_chi2_thres {} > max_chi2 {}".format( + _chi2_thres, run_params["max_chi2"] + ), + len(guess_ndata), + ) + + chi2_selected = y_train[:, valid_idx][idx_chi2_selected] + # get distribution of the missing band + + guess_ndata = y_train[:, not_valid_idx][idx_chi2_selected] + + dists = np.linalg.norm(y_obs_valid_only - chi2_selected, axis=1) + neighs_weights = 1 / dists + + kdes = [] + for i in range(guess_ndata.shape[1]): + kde = stats.gaussian_kde(guess_ndata.T[i], 0.2, weights=neighs_weights) + kdes.append(kde) + # import pdb; pdb.set_trace() + # ------------------------------------------------ + + nbands = y_train.shape[1] // 2 # total number of bands + not_valid_idx_unc = not_valid_idx + nbands + + all_x = [] + cnt = 0 + cnt_timeout = 0 + timeout_flag = False + # ------------------------------------------------ + # draw monte carlo samples from the nearest neighbor approximation + # later we will average over the monte-carlo posterior samples to attain the final posterior estimation + while cnt < run_params["nmc"]: + signal.alarm( + 0 + ) # run_params["tmax_per_obj"]) # max time spent on one object in sec -- disabled for now + try: + x = np.copy(observed) + + for j, idx in enumerate(not_valid_idx): + x[not_valid_idx[j]] = np.random.choice(guess_ndata.T[j]) + # let's just randomly sample the neighbors instead of unpredictable toy noise model + x[22:][not_valid_idx[j]] = y_train[idx_chi2_selected][ + np.random.choice(range(len(guess_ndata.T[j]))) + ][22:][not_valid_idx[j]] + # x[not_valid_idx_unc[j]] = toy_noise( + # flux=x[not_valid_idx[j]], + # meds_sigs=sbi_params["toynoise_meds_sigs"][idx], + # stds_sigs=sbi_params["toynoise_stds_sigs"][idx], + # verbose=run_params["verbose"], + # )[1] + + # the noise model in the training isn't quite right + # Pan-STARRS in particular seems a little off + # we'll have to re-train at some point, but for now just pull + # uncertainties from the training sample + for idx, fname in zip(valid_idx, obs["filternames"][valid_idx]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + x[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][idx] + + all_x.append(x) + + # if we can't get one posterior sample in one second, we should move along + # to the next MC sample + do_continue = False + for tmax, npost in zip( + [1, run_params["tmax_per_iter"]], [1, run_params["nposterior"]] + ): + signal.alarm(tmax) # max time spent on one object in sec + try: + noiseless_theta = hatp_x_y.sample( + (npost,), + x=torch.as_tensor(x.astype(np.float32)).to(device), + show_progress_bars=False, + ) + except TimeoutException: + signal.alarm(0) + do_continue = True + break + + if do_continue: + continue + + signal.alarm(0) + + noiseless_theta = noiseless_theta.detach().numpy() + + ave_theta.append(noiseless_theta) + + cnt += 1 + if run_params["verbose"]: + if cnt % 10 == 0: + print("mc samples:", cnt) + + except TimeoutException: + cnt_timeout += 1 + else: + signal.alarm(0) + + # set max time + et = time.time() + elapsed_time = et - st # in secs + if elapsed_time / 60 > run_params["tmax_all"] or ( + cnt < run_params["nmc"] / 10 + and elapsed_time / 60 * 10 > run_params["tmax_all"] + ): + timeout_flag = True + use_res = False + break + # ------------------------------------------------ + + all_x = np.array(all_x) + all_x_flux = all_x.T[:nbands] + all_x_unc = all_x.T[nbands:] + y_guess = np.concatenate( + [np.median(all_x_flux, axis=1), np.median(all_x_unc, axis=1), [obs["redshift"]]] + ) + + return ave_theta, y_guess, use_res, timeout_flag, cnt + + +def lim_of_noisy_guass(obs, run_params, sbi_params): + """restrict the range over which we monte carlo the noise based on similar SEDs in the training set; + see sec. 4.1.1 for details + """ + + use_res = 1 + + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + noisy_mask = np.copy(obs["noisy_mask"]) + + noisy_idx = np.where(noisy_mask == True)[0] + not_noisy_idx = np.where(noisy_mask == False)[0] + + look_in_training = y_train[:, noisy_idx] + chi2_nei = chi2dof( + mags=look_in_training, obsphot=y_obs[noisy_idx], obsphot_unc=sig_obs[noisy_idx] + ) + + _chi2_thres = run_params["ini_chi2"] * 1 + while True: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 10: + chi2_nei_selected = y_train[idx_chi2_selected] + chi2_nei_selected = np.squeeze(chi2_nei_selected[:, noisy_idx]) + lims = [ + np.atleast_1d(np.min(chi2_nei_selected, axis=0)), + np.atleast_1d(np.max(chi2_nei_selected, axis=0)), + ] + if np.all((lims[0] - y_obs[noisy_idx]) < 0) and np.all( + (lims[1] - y_obs[noisy_idx]) > 0 + ): + break + _chi2_thres += 5 + if _chi2_thres > run_params["max_chi2"]: + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + print( + "Clipping the Gaussian, from which we MC noise, to be within the min & max of the magnitude at that band in the training set" + ) + use_res = 0 + # choose the args for clipping norm by the min & max of the magnitude at that band in the training set + lims = np.array( + [ + np.atleast_1d(np.min(y_train[:, noisy_idx], axis=0)), + np.atleast_1d(np.max(y_train[:, noisy_idx], axis=0)), + ] + ) + break + + return lims, use_res + + +def sbi_mcnoise(obs, run_params, sbi_params, max_neighbors=200): + """used when observations have out-of-distribution uncertainties; + see sec. 4.1.1 for details + """ + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("sbi mc noise") + + ave_theta = [] + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 # total number of bands + + noisy_mask = np.copy(obs["noisy_mask"]) + noisy_idx = np.where(noisy_mask == True)[0] + not_noisy_idx = np.where(noisy_mask == False)[0] + + # start time + st = time.time() + + lims, use_res = lim_of_noisy_guass( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + loc = y_obs[noisy_idx] + scale = sig_obs[noisy_idx] + + ### temporary for getting errors, because error model not good enough + chi2_nei = chi2dof(mags=y_train[:, :22], obsphot=y_obs, obsphot_unc=sig_obs) + + _chi2_thres = run_params["ini_chi2"] * 1 + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + + if _chi2_thres > run_params["max_chi2"]: + chi2_selected = y_train[:] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + else: + chi2_selected = y_train[idx_chi2_selected] + + cnt = 0 + cnt_timeout = 0 + timeout_flag = False + # ------------------------------------------------ + # draw monte carlo samples from a norm dist centered at x_obs and 1 sigma = 1 sigma uncertainty associated with x_obs + # later we will average over the those "noisy" posterior samples to attain the final posterior estimation + while cnt < run_params["nmc"]: + samp_y_guess = np.copy(observed) + samp_y_guess[noisy_idx] = stats.norm.rvs(loc=loc, scale=scale) + # ensure positive uncertainties + _nnflag = True + for ii, this_noisy_flux in enumerate(samp_y_guess[noisy_idx]): + # print(lims[0][ii], lims[1][ii]) + if this_noisy_flux > lims[0][ii] and this_noisy_flux < lims[1][ii]: + _nnflag &= True + else: + _nnflag &= False + + if _nnflag: + samp_y_guess[noisy_idx + nbands] = toy_noise( + flux=samp_y_guess[noisy_idx], + meds_sigs=sbi_params["toynoise_meds_sigs"][ii], + stds_sigs=sbi_params["toynoise_stds_sigs"][ii], + verbose=run_params["verbose"], + )[1] + # signal.alarm(run_params["tmax_per_obj"]) + + for idx, fname in enumerate(obs["filternames"]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + samp_y_guess[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][ + idx + ] + + do_continue = False + for tmax, npost in zip( + [run_params["tmax_per_iter"]], [run_params["nposterior"]] + ): + signal.alarm(tmax) # max time spent on one object in sec + try: + noiseless_theta = hatp_x_y.sample( + (run_params["nposterior"],), + x=torch.as_tensor(samp_y_guess).to(device), + show_progress_bars=False, + ) + except TimeoutException: + signal.alarm(0) + do_continue = True + break + + if do_continue: + continue + + noiseless_theta = noiseless_theta.detach().numpy() + + ave_theta.append(noiseless_theta) + + cnt += 1 + if run_params["verbose"]: + if cnt % 10 == 0: + print("mc samples:", cnt) + + # except TimeoutException: + # cnt_timeout += 1 + # else: + signal.alarm(0) + + # end time + et = time.time() + elapsed_time = et - st # in secs + if elapsed_time / 60 > run_params["tmax_all"] or ( + cnt < run_params["nmc"] / 10 + and elapsed_time / 60 * 10 > run_params["tmax_all"] + ): + timeout_flag = True + use_res = False + break + # ------------------------------------------------ + + return ave_theta, use_res, timeout_flag, cnt + + +def sbi_missing_and_noisy(obs, run_params, sbi_params): + """used when observations have missing data and out-of-distribution uncertainties. + fill in the missing bands first using the nearest neighbor approximation; + then mc the noisy bands + """ + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("sbi missing and noisy bands") + + ave_theta = [] + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 + + noisy_mask = np.copy(obs["noisy_mask"]) + noisy_idx = np.where(noisy_mask == True)[0] + not_noisy_idx = np.where(noisy_mask == False)[0] + + invalid_mask = np.copy(obs["missing_mask"]) + y_obs_valid_only = y_obs[~invalid_mask] + valid_idx = np.where(~invalid_mask)[0] + not_valid_idx = np.where(invalid_mask)[0] + not_valid_idx_unc = not_valid_idx + nbands + + # ------------------------------------------------ + kdes, use_res_missing, idx_chi2_selected = gauss_approx_missingband( + obs, run_params, sbi_params + ) + + # start time + st = time.time() + + lims, use_res_noisy = lim_of_noisy_guass( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + loc = y_obs[noisy_idx] + scale = sig_obs[noisy_idx] + + cnt = 0 + cnt_timeout = 0 + timeout_flag = False + while cnt < run_params["nmc"]: + + samp_y_guess = np.copy(observed) + + # first, fill in the missing bands + for j in range(len(not_valid_idx)): + # samp_y_guess[not_valid_idx[j]] = kdes[j].resample(size=1) + # samp_y_guess[not_valid_idx_unc[j]] = toy_noise( + # flux=samp_y_guess[not_valid_idx[j]], + # meds_sigs=sbi_params["toynoise_meds_sigs"][not_valid_idx[j]], + # stds_sigs=sbi_params["toynoise_stds_sigs"][not_valid_idx[j]], + # verbose=run_params["verbose"], + # )[1] + samp_y_guess[not_valid_idx[j]] = y_train[idx_chi2_selected][ + np.random.choice(range(len(idx_chi2_selected))) + ][not_valid_idx[j]] + samp_y_guess[not_valid_idx_unc[j]] = y_train[idx_chi2_selected][ + np.random.choice(range(len(idx_chi2_selected))) + ][not_valid_idx_unc[j]] + + # second, deal with OOD noise + samp_y_guess[noisy_idx] = stats.norm.rvs(loc=loc, scale=scale) + _nnflag = True + for ii, this_noisy_flux in enumerate(samp_y_guess[noisy_idx]): + if this_noisy_flux > lims[0][ii] and this_noisy_flux < lims[1][ii]: + _nnflag &= True + else: + _nnflag &= False + + if _nnflag: + samp_y_guess[noisy_idx[ii] + nbands] = y_train[idx_chi2_selected][ + np.random.choice(range(len(idx_chi2_selected))) + ][noisy_idx[ii] + nbands] + # samp_y_guess[noisy_idx + nbands] = toy_noise( + # flux=samp_y_guess[noisy_idx[ii]], + # meds_sigs=sbi_params["toynoise_meds_sigs"][noisy_idx[ii]], + # stds_sigs=sbi_params["toynoise_stds_sigs"][noisy_idx[ii]], + # verbose=run_params["verbose"], + # )[1] + + # the noise model in the training isn't quite right + # Pan-STARRS in particular seems a little off + # we'll have to re-train at some point, but for now just pull + # uncertainties from the training sample + for idx, fname in zip(valid_idx, obs["filternames"][valid_idx]): + # if 'PanSTARRS' in fname or '2MASS' in fname or 'SDSS' in fname or 'DES' in fname: + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + samp_y_guess[22:][idx] = y_train[idx_chi2_selected][chc][22:][idx] + + # if we can't get one posterior sample in one second, we should move along + # to the next MC sample + do_continue = False + for tmax, npost in zip( + [1, run_params["tmax_per_iter"]], [1, run_params["nposterior"]] + ): + signal.alarm(tmax) # max time spent on one object in sec + try: + noiseless_theta = hatp_x_y.sample( + (npost,), + x=torch.as_tensor(samp_y_guess).to(device), + show_progress_bars=False, + ) + except TimeoutException: + signal.alarm(0) + do_continue = True + break + + if do_continue: + continue + + signal.alarm(0) + noiseless_theta = noiseless_theta.detach().numpy() + + ave_theta.append(noiseless_theta) + + cnt += 1 + if run_params["verbose"]: + if cnt % 10 == 0: + print("mc samples:", cnt) + + # end time + et = time.time() + elapsed_time = et - st # in secs + if elapsed_time / 60 > run_params["tmax_all"] or ( + cnt < run_params["nmc"] / 10 + and elapsed_time / 60 * 10 > run_params["tmax_all"] + ): + timeout_flag = 1 + use_res = 0 + break + # ------------------------------------------------ + + if use_res_missing == 1 and use_res_noisy == 1 and timeout_flag == 0: + use_res = 1 + + return ave_theta, use_res, timeout_flag, cnt + + +def sbi_baseline(obs, run_params, sbi_params, max_neighbors=200): + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("baseline sbi") + + flags = { + "use_res": 0, # True if sbi++ succeeds; False if otherwise. + # below are for bookkeeping + "timeout": 0, + "use_res_missing": 0, # True if sbi++ for missing bands succeeds + "use_res_noisy": 0, # True if sbi++ for noisy bands succeeds + "noisy_data": False, # True if sbi++ (noisy data) is called + "missing_data": False, # True if sbi++ (missing data) is called + "nsamp_missing": -99, # number of MC samples drawn + "nsamp_noisy": -99, # number of MC samples drawn + } + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags"]) + sig_obs = np.copy(obs["mags_unc"]) + # copy the observed data to be used by sbi + # missing data, if any, will be filled in later + obs["mags_sbi"] = y_obs + obs["mags_unc_sbi"] = sig_obs + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 # total number of bands + + flags["use_res"] = 1 + flags["timeout"] = False + + ### temporary for getting errors, because error model not good enough + chi2_nei = chi2dof(mags=y_train[:, :22], obsphot=y_obs, obsphot_unc=sig_obs) + + _chi2_thres = run_params["ini_chi2"] * 1 + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + + if _chi2_thres > run_params["max_chi2"]: + chi2_selected = y_train[:] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + else: + chi2_selected = y_train[idx_chi2_selected] + + # ------------------------------------------------ + # call baseline sbi to draw posterior samples + signal.alarm(run_params["tmax_per_obj"]) # max time spent on one object in sec + + x = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + + for idx, fname in enumerate(obs["filternames"]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + x[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][idx] + + try: + ave_theta = hatp_x_y.sample( + (run_params["np_baseline"],), + x=torch.as_tensor(x.astype(np.float32)).to(device), + show_progress_bars=False, + ) + ave_theta = ave_theta.detach().numpy() + except TimeoutException: + flags["timeout"] = True + ave_theta = [np.nan] + if run_params["verbose"]: + print("timeout!") + else: + signal.alarm(0) + # ------------------------------------------------ + + return ave_theta, obs, flags + + +def sbi_pp(obs, run_params, sbi_params, max_neighbors=200): + """wrapper for sbi++; this should be the only function needed to be called outside this scipt under normal circumstances + + obs: a dictionary at least containing + - "mags": observed photometry, unit must match the training set + - "mags_unc": observed uncertainty, unit must match the training set + run_params: a dictionary at least containing + - "" + + """ + signal.signal(signal.SIGALRM, timeout_handler) + + flags = { + "use_res": 0, # True if sbi++ succeeds; False if otherwise. + # below are for bookkeeping + "timeout": 0, + "use_res_missing": 0, # True if sbi++ for missing bands succeeds + "use_res_noisy": 0, # True if sbi++ for noisy bands succeeds + "noisy_data": False, # True if sbi++ (noisy data) is called + "missing_data": False, # True if sbi++ (missing data) is called + "nsamp_missing": -99, # number of MC samples drawn + "nsamp_noisy": -99, # number of MC samples drawn + } + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags"]) + sig_obs = np.copy(obs["mags_unc"]) + # copy the observed data to be used by sbi + # missing data, if any, will be filled in later + obs["mags_sbi"] = y_obs + obs["mags_unc_sbi"] = sig_obs + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 # total number of bands + + # decide if we need to deal with missing bands + obs["missing_mask"] = np.isnan(y_obs) + missing_mask = np.isnan(y_obs) # idx of missing bands + # decide if we need to deal with noisy bands + noisy_mask = np.zeros_like(y_obs, dtype=bool) + for j in range(nbands): + _toynoise = toy_noise( + flux=y_obs[j], + meds_sigs=sbi_params["toynoise_meds_sigs"][j], + stds_sigs=sbi_params["toynoise_stds_sigs"][j], + verbose=run_params["verbose"], + ) + noisy_mask[j] = (sig_obs[j] - _toynoise[1]) / _toynoise[2] >= run_params[ + "noisy_sig" + ] + # if noisy_mask[j]: + # import pdb; pdb.set_trace() + noisy_mask &= np.isfinite(y_obs) # idx of noisy bands + obs["noisy_mask"] = noisy_mask + + ave_theta = [np.nan] + if np.any(missing_mask): + flags["missing_data"] = True + if np.any(noisy_mask): + flags["noisy_data"] = True + + if not flags["missing_data"] and not flags["noisy_data"]: + flags["use_res"] = 1 + flags["timeout"] = False + if run_params["verbose"]: + print("baseline sbi") + + ### temporary for getting errors, because error model not good enough + chi2_nei = chi2dof(mags=y_train[:, :22], obsphot=y_obs, obsphot_unc=sig_obs) + + _chi2_thres = run_params["ini_chi2"] * 1 + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + + if _chi2_thres > run_params["max_chi2"]: + chi2_selected = y_train[:] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + else: + chi2_selected = y_train[idx_chi2_selected] + + signal.alarm(run_params["tmax_per_obj"]) # max time spent on one object in sec + + x = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + + for idx, fname in enumerate(obs["filternames"]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + x[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][idx] + + try: + ave_theta = hatp_x_y.sample( + (run_params["np_baseline"],), + x=torch.as_tensor(x.astype(np.float32)).to(device), + show_progress_bars=False, + ) + ave_theta = ave_theta.detach().numpy() + except TimeoutException: + flags["timeout"] = True + ave_theta = [np.nan] + else: + signal.alarm(0) + + return ave_theta, obs, flags + + if flags["missing_data"] and flags["noisy_data"]: + ( + ave_theta, + flags["use_res"], + flags["timeout"], + flags["nsamp_noisy"], + ) = sbi_missing_and_noisy(obs=obs, run_params=run_params, sbi_params=sbi_params) + if flags["timeout"]: + for i, mask in enumerate( + [ + (obs["bands"] == "WISE_W3") | (obs["bands"] == "WISE_W4"), + (obs["bands"] == "WISE_W1") | (obs["bands"] == "WISE_W2"), + (obs["bands"] == "GALEX_NUV") | (obs["bands"] == "GALEX_FUV"), + ] + ): + print("timeout! trying without some filters") + if i == 2: + run_params["tmax_all"] *= 3 + obs["missing_mask"][mask] = True + ( + ave_theta, + flags["use_res"], + flags["timeout"], + flags["nsamp_noisy"], + ) = sbi_missing_and_noisy( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + if not flags["timeout"]: + break + + # separate cases + if flags["missing_data"] and not flags["noisy_data"]: + res = sbi_missingband( + obs=obs, run_params=run_params, sbi_params=sbi_params, seconditer=True + ) + + # if things timed out, then we should try dropping + # some problematic filters + # I think this is mostly deprecated now + (ave_theta, flags["use_res"], flags["nsamp_noisy"], flags["timeout"], cnt) = res + if flags["timeout"]: + for i, mask in enumerate( + [ + (obs["bands"] == "WISE_W3") | (obs["bands"] == "WISE_W4"), + (obs["bands"] == "WISE_W1") | (obs["bands"] == "WISE_W2"), + (obs["bands"] == "GALEX_NUV") | (obs["bands"] == "GALEX_FUV"), + ] + ): + print("timeout! trying without some filters") + if i == 2: + run_params["tmax_all"] *= 3 + + obs["missing_mask"][mask] = True + ( + ave_theta, + flags["use_res"], + flags["nsamp_noisy"], + flags["timeout"], + cnt, + ) = sbi_missingband( + obs=obs, + run_params=run_params, + sbi_params=sbi_params, + seconditer=True, + ) + if not flags["timeout"]: + break + + if len(res) != 5: + raise RuntimeError( + "couldnt get good chi2 for nearest neighbors. Aborting" + ) + + ( + ave_theta, + y_guess, + flags["use_res_missing"], + flags["timeout"], + flags["nsamp_missing"], + ) = res + + flags["use_res"] = flags["use_res_missing"] * 1 + + if not flags["missing_data"] and flags["noisy_data"]: + # mc the noisy bands + ( + ave_theta, + flags["use_res_noisy"], + flags["timeout"], + ### flags["nsamp_noisy"], + cnt, + ) = sbi_mcnoise(obs=obs, run_params=run_params, sbi_params=sbi_params) + flags["use_res"] = flags["use_res_noisy"] * 1 + + ave_theta = np.array(ave_theta) + try: + ave_theta = np.concatenate(ave_theta) + except Exception as e: + pass + + return ave_theta, obs, flags diff --git a/app/host/SBI/sbi_pp_fordebugging.py b/app/host/SBI/sbi_pp_fordebugging.py new file mode 100644 index 00000000..b52f6f70 --- /dev/null +++ b/app/host/SBI/sbi_pp_fordebugging.py @@ -0,0 +1,1476 @@ +# $ conda activate sbi_env +import copy +import os +import signal +import sys +import time +import warnings + +import pylab as plt +from astropy.stats import sigma_clipped_stats +from prospect.fitting import fit_model as fit_model_prospect +from prospect.fitting import lnprobfn +from prospect.io import write_results as writer +from prospect.io.write_results import write_h5_header +from prospect.io.write_results import write_obs_to_h5 +from prospect.models import priors +from prospect.models import SpecModel +from prospect.models.sedmodel import PolySpecModel +from prospect.models.templates import TemplateLibrary +from prospect.models.transforms import logsfr_ratios_to_sfrs +from prospect.models.transforms import zred_to_agebins +from prospect.sources import CSPSpecBasis +from prospect.sources import FastStepBasis + +# import importlib +# importlib.import_module("host.prospector.build_model") +# importlib.import_module("host.prospector.build_obs") + +os.environ["KMP_DUPLICATE_LIB_OK"] = "True" +from operator import itemgetter +import numpy as np +from numpy.random import normal, uniform +from scipy import stats +from scipy.interpolate import interp1d + +# torch +import torch + +torch.set_num_threads(1) +# torch.multiprocessing.set_start_method('forkserver',force=True) +# spawn +import torch.nn as nn +import torch.nn.functional as F +from sbi import utils as Ut +from sbi import inference as Inference + +from host.models import AperturePhotometry, Transient, Filter +from django.db.models import Q +from host.host_utils import get_dust_maps +import extinction +from prospect.utils.obsutils import fix_obs +from host.photometric_calibration import mJy_to_maggies +from astropy.cosmology import WMAP9 as cosmo + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" +all_filters = Filter.objects.filter(~Q(name="DES_i") & ~Q(name="DES_Y")) + + +def build_all(**kwargs): + return ( + build_obs(**kwargs), + build_model(**kwargs), + build_sps(**kwargs), + build_noise(**kwargs), + ) + + +def build_obs(**extras): ##transient, aperture_type): + + """ + This functions is required by prospector and should return + a dictionary defined by + https://prospect.readthedocs.io/en/latest/dataformat.html. + + """ + filters = [] + for filter in all_filters: + + filters.append(filter.transmission_curve()) + + obs_data = dict( + wavelength=None, + spectrum=None, + unc=None, + mask=None, + ##redshift=z, + maggies=np.ones(len(all_filters)), # np.array(flux_maggies), + maggies_unc=np.ones(len(all_filters)), + filters=filters, + ) + obs_data["phot_wave"] = np.array([f.wave_effective for f in obs_data["filters"]]) + obs_data["phot_mask"] = [True] * len(obs_data["filters"]) + + return fix_obs(obs_data) + + +def build_model(obs=None, **extras): + """prospector-alpha""" + fit_order = [ + "zred", + "logmass", + "logzsol", + "logsfr_ratios", + "dust2", + "dust_index", + "dust1_fraction", + "log_fagn", + "log_agn_tau", + "gas_logz", + "duste_qpah", + "duste_umin", + "log_duste_gamma", + ] + + # ------------- + # MODEL_PARAMS + model_params = {} + + # --- BASIC PARAMETERS --- + model_params["zred"] = { + "N": 1, + "isfree": True, + "init": 0.5, + "prior": priors.FastUniform(a=0, b=0.2), + } + + model_params["logmass"] = { + "N": 1, + "isfree": True, + "init": 8.0, + "units": "Msun", + "prior": priors.FastUniform(a=7.0, b=12.5), + } + + model_params["logzsol"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"$\log (Z/Z_\odot)$", + "prior": priors.FastUniform(a=-1.98, b=0.19), + } + + model_params["imf_type"] = { + "N": 1, + "isfree": False, + "init": 1, # 1 = chabrier + "units": None, + "prior": None, + } + model_params["add_igm_absorption"] = {"N": 1, "isfree": False, "init": True} + model_params["add_agb_dust_model"] = {"N": 1, "isfree": False, "init": True} + model_params["pmetals"] = {"N": 1, "isfree": False, "init": -99} + + # --- SFH --- + nbins_sfh = 7 + model_params["sfh"] = {"N": 1, "isfree": False, "init": 3} + model_params["logsfr_ratios"] = { + "N": 6, + "isfree": True, + "init": 0.0, + "prior": priors.FastTruncatedEvenStudentTFreeDeg2( + hw=np.ones(6) * 5.0, sig=np.ones(6) * 0.3 + ), + } + + # add redshift scaling to agebins, such that + # t_max = t_univ + def zred_to_agebins(zred=None, **extras): + amin = 7.1295 + nbins_sfh = 7 + tuniv = cosmo.age(zred)[0].value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + + def logsfr_ratios_to_masses( + logmass=None, logsfr_ratios=None, agebins=None, **extras + ): + """This converts from an array of log_10(SFR_j / SFR_{j+1}) and a value of + log10(\Sum_i M_i) to values of M_i. j=0 is the most recent bin in lookback + time. + """ + nbins = agebins.shape[0] + sratios = 10 ** np.clip(logsfr_ratios, -100, 100) + dt = 10 ** agebins[:, 1] - 10 ** agebins[:, 0] + coeffs = np.array( + [ + (1.0 / np.prod(sratios[:i])) + * (np.prod(dt[1 : i + 1]) / np.prod(dt[:i])) + for i in range(nbins) + ] + ) + m1 = (10**logmass) / coeffs.sum() + + return m1 * coeffs + + model_params["mass"] = { + "N": 7, + "isfree": False, + "init": 1e6, + "units": r"M$_\odot$", + "depends_on": logsfr_ratios_to_masses, + } + + model_params["agebins"] = { + "N": 7, + "isfree": False, + "init": zred_to_agebins(np.atleast_1d(0.5)), + "prior": None, + "depends_on": zred_to_agebins, + } + + # --- Dust Absorption --- + model_params["dust_type"] = { + "N": 1, + "isfree": False, + "init": 4, + "units": "FSPS index", + } + model_params["dust1_fraction"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.0, b=2.0, mu=1.0, sig=0.3), + } + + model_params["dust2"] = { + "N": 1, + "isfree": True, + "init": 0.0, + "units": "", + "prior": priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0), + } + + def to_dust1(dust1_fraction=None, dust1=None, dust2=None, **extras): + return dust1_fraction * dust2 + + model_params["dust1"] = { + "N": 1, + "isfree": False, + "depends_on": to_dust1, + "init": 0.0, + "units": "optical depth towards young stars", + "prior": None, + } + model_params["dust_index"] = { + "N": 1, + "isfree": True, + "init": 0.7, + "units": "", + "prior": priors.FastUniform(a=-1.0, b=0.2), + } + + # --- Nebular Emission --- + model_params["add_neb_emission"] = {"N": 1, "isfree": False, "init": True} + model_params["add_neb_continuum"] = {"N": 1, "isfree": False, "init": True} + model_params["gas_logz"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"log Z/Z_\odot", + "prior": priors.FastUniform(a=-2.0, b=0.5), + } + model_params["gas_logu"] = { + "N": 1, + "isfree": False, + "init": -1.0, + "units": r"Q_H/N_H", + "prior": priors.FastUniform(a=-4, b=-1), + } + + # --- AGN dust --- + model_params["add_agn_dust"] = {"N": 1, "isfree": False, "init": True} + + model_params["log_fagn"] = { + "N": 1, + "isfree": True, + "init": -7.0e-5, + "prior": priors.FastUniform(a=-5.0, b=-4.9), + } + + def to_fagn(log_fagn=None, **extras): + return 10**log_fagn + + model_params["fagn"] = {"N": 1, "isfree": False, "init": 0, "depends_on": to_fagn} + + model_params["log_agn_tau"] = { + "N": 1, + "isfree": True, + "init": np.log10(20.0), + "prior": priors.FastUniform(a=np.log10(15.0), b=np.log10(15.1)), + } + + def to_agn_tau(log_agn_tau=None, **extras): + return 10**log_agn_tau + + model_params["agn_tau"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_agn_tau, + } + + # --- Dust Emission --- + model_params["duste_qpah"] = { + "N": 1, + "isfree": True, + "init": 2.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=2.0, sig=2.0), + } + + model_params["duste_umin"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=1.0, sig=10.0), + } + + model_params["log_duste_gamma"] = { + "N": 1, + "isfree": True, + "init": -2.0, + "prior": priors.FastTruncatedNormal(a=-2.1, b=-1.9, mu=-2.0, sig=1.0), + } + + def to_duste_gamma(log_duste_gamma=None, **extras): + return 10**log_duste_gamma + + model_params["duste_gamma"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_duste_gamma, + } + + # ---- Units ---- + model_params["peraa"] = {"N": 1, "isfree": False, "init": False} + + model_params["mass_units"] = {"N": 1, "isfree": False, "init": "mformed"} + + tparams = {} + for i in fit_order: + tparams[i] = model_params[i] + for i in list(model_params.keys()): + if i not in fit_order: + tparams[i] = model_params[i] + model_params = tparams + + return PolySpecModel(model_params) + + +def build_sps(zcontinuous=2, compute_vega_mags=False, **extras): + sps = FastStepBasis(zcontinuous=zcontinuous, compute_vega_mags=compute_vega_mags) + return sps + + +def build_noise(**extras): + return None, None + + +# prior +def prior_from_train(ll_or_ul, x_train): + """We will only need the lower & upper limits to be passed to sbi as 'priors' + Note that I have not checked how this affects the sbi functions that are not used in the script. + Here we simply read in the bounds from the training set + """ + + assert ll_or_ul in ["ll", "ul"] + + if ll_or_ul == "ll": + res = [] + for i in range(x_train.shape[1]): + res.append(np.min(x_train.T[i])) + else: + res = [] + for i in range(x_train.shape[1]): + res.append(np.max(x_train.T[i])) + + return res + + +def toy_noise(flux, meds_sigs, stds_sigs, verbose=False, **extra): + """toy noise; must be the same as the noise model used when generating the training set + Here we use assume Gaussian noises + flux: photometry + meds_sigs: median of the magnitude bin + stds_sigs: 1 standard deviation + """ + return flux, meds_sigs(flux), np.clip(stds_sigs(flux), a_min=0.001, a_max=None) + + +# the following functions are used to set the max time spent per object +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + raise TimeoutException + + +def absdiff(mags, obsphot, obsphot_unc): + """abs difference in photometry""" + + return np.abs(mags - obsphot) + + +def chi2dof(mags, obsphot, obsphot_unc, individual=False): + """reduced chi^2""" + + if individual: + return ((mags - obsphot) / obsphot_unc) ** 2 + else: + chi2 = np.nansum(((mags - obsphot) / obsphot_unc) ** 2, axis=1) + return chi2 / np.sum(np.isfinite(obsphot)) + + +def chidof(mags, obsphot, obsphot_unc, individual=False): + """reduced chi^2""" + + if individual: + return (mags - obsphot) / obsphot_unc + else: + chi = np.nansum((mags - obsphot) / obsphot_unc, axis=1) + return chi / np.sum(np.isfinite(obsphot)) + + +def gauss_approx_missingband(obs, run_params, sbi_params, max_neighbors=200): + """nearest neighbor approximation of missing bands; + see sec. 4.1.2 for details + """ + use_res = False + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + invalid_mask = np.copy(obs["missing_mask"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + y_obs_valid_only = y_obs[~invalid_mask] + valid_idx = np.where(~invalid_mask)[0] + not_valid_idx = np.where(invalid_mask)[0] + + look_in_training = y_train[:, valid_idx] + chi2_nei = chi2dof( + mags=look_in_training, obsphot=y_obs[valid_idx], obsphot_unc=sig_obs[valid_idx] + ) + + _chi2_thres = run_params["ini_chi2"] * 1 + cnt = 0 + use_res = True + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + cnt += 1 + + if _chi2_thres > run_params["max_chi2"]: + use_res = False + chi2_selected = y_train[:, valid_idx] + chi2_selected = chi2_selected[:max_neighbors] + guess_ndata = y_train[:, not_valid_idx] + guess_ndata = guess_ndata[:max_neighbors] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + print( + "_chi2_thres {} > max_chi2 {}".format( + _chi2_thres, run_params["max_chi2"] + ), + len(guess_ndata), + ) + else: + chi2_selected = y_train[:, valid_idx][idx_chi2_selected] + # get distribution of the missing band + guess_ndata = y_train[:, not_valid_idx][idx_chi2_selected] + dists = np.linalg.norm(y_obs_valid_only - chi2_selected, axis=1) + neighs_weights = 1 / dists + + kdes = [] + for i in range(guess_ndata.shape[1]): + _kde = stats.gaussian_kde(guess_ndata.T[i], 0.2, weights=neighs_weights) + kdes.append(_kde) + + return kdes, use_res, idx_chi2_selected + + +def sbi_missingband(obs, run_params, sbi_params, seconditer=False): + """used when observations have missing data; + see sec. 4.1.2 of for details + """ + + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("sbi missing bands") + # hack! + sps = build_sps() + ave_theta = [] + + max_neighbors = 200 + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + invalid_mask = np.copy(obs["missing_mask"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + y_obs_valid_only = y_obs[~invalid_mask] + valid_idx = np.where(~invalid_mask)[0] + not_valid_idx = np.where(invalid_mask)[0] + st = time.time() + + # ------------------------------------------------ + # nearest neighbor approximation of missing bands; + # see sec. 4.1 for details + look_in_training = y_train[:, valid_idx] + chi2_nei = chi2dof( + mags=look_in_training, obsphot=y_obs[valid_idx], obsphot_unc=sig_obs[valid_idx] + ) + chi_nei = chidof( + mags=look_in_training, obsphot=y_obs[valid_idx], obsphot_unc=sig_obs[valid_idx] + ) + + _chi2_thres = run_params["ini_chi2"] * 1 + cnt = 0 + use_res = True + while _chi2_thres <= run_params["max_chi2"]: + # idx_chi2_selected = np.where(chi2_nei[redshift_idx] <= _chi2_thres)[0] + # if len(idx_chi2_selected) >= 100 and np.abs(np.median(chi_nei[redshift_idx][idx_chi2_selected])) < 0.25: + + # let's not allow any matches with giant overall offsets + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[ + 0 + ] # & (np.abs(chi_nei) < 0.25))[0] + if ( + len(idx_chi2_selected) >= max_neighbors + ): # and np.abs(np.median(chi_nei[idx_chi2_selected])) < 0.25: + break + else: + _chi2_thres += 5 + cnt += 1 + + if _chi2_thres > run_params["max_chi2"]: + use_res = False + chi2_selected = y_train[:, valid_idx] + # chi2_selected = chi2_selected[redshift_idx][:100] + chi2_selected = chi2_selected[:max_neighbors] + guess_ndata = y_train[:, not_valid_idx] + guess_ndata = guess_ndata[:max_neighbors] + + # if not seconditer: + # idx_chi2_selected = np.argsort(chi2_nei) + # diffs = absdiff( + # mags=look_in_training, + # obsphot=y_obs[valid_idx], + # obsphot_unc=sig_obs[valid_idx], + # ) + # diffs_best = np.sum(diffs[idx_chi2_selected[0:100]], axis=0) + # worst_band = np.where(diffs_best == np.max(diffs_best))[0] + # obs["missing_mask"][worst_band] = True + # print("Failed to find sufficient number of nearest neighbors!") + # print(f"Trying again after dropping band {worst_band[0]}") + # obs["sbi_flag"] = "chi2 fail" + # return obs + + # idx_chi2_selected = np.argsort(chi2_nei[redshift_idx])[0:100] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + print( + "_chi2_thres {} > max_chi2 {}".format( + _chi2_thres, run_params["max_chi2"] + ), + len(guess_ndata), + ) + + ### sometimes one bad photometry point messes up everything + # std_out,iBad = np.array([]),np.array([],dtype=int) + # for i in idx_chi2_selected[0:100]: + # std_out = np.append(std_out,[sigma_clipped_stats(y_obs[valid_idx]-y_train[redshift_idx][i][0:22][valid_idx[0:22]])[2]]) + # iBad = np.append(iBad,np.where(np.abs(y_obs[valid_idx]-y_train[redshift_idx][i][0:22][valid_idx[0:22]]) > 3*std_out[-1])[0]) + # hack! + # if len(iBad): + # iBad,counts = np.unique(iBad,return_counts=True) + # iBad = iBad[counts >= 10]; counts = counts[counts >= 10] + # iBad = iBad[np.argsort(counts)[::-1]][0:2] ## remove max two outliers, choose the ones that are most commonly outliers + # invalid_mask[valid_idx[iBad]] = True + # y_obs_valid_only = y_obs[~invalid_mask] + # valid_idx = np.where(~invalid_mask)[0] + # not_valid_idx = np.where(invalid_mask)[0] + + # chi2_selected = y_train[:, valid_idx][redshift_idx][idx_chi2_selected] + # get distribution of the missing band + # guess_ndata = y_train[:, not_valid_idx][redshift_idx][idx_chi2_selected] + chi2_selected = y_train[:, valid_idx][idx_chi2_selected] + # get distribution of the missing band + + guess_ndata = y_train[:, not_valid_idx][idx_chi2_selected] + + dists = np.linalg.norm(y_obs_valid_only - chi2_selected, axis=1) + neighs_weights = 1 / dists + + kdes = [] + for i in range(guess_ndata.shape[1]): + kde = stats.gaussian_kde(guess_ndata.T[i], 0.2, weights=neighs_weights) + kdes.append(kde) + # import pdb; pdb.set_trace() + # ------------------------------------------------ + + nbands = y_train.shape[1] // 2 # total number of bands + not_valid_idx_unc = not_valid_idx + nbands + + all_x = [] + cnt = 0 + cnt_timeout = 0 + timeout_flag = False + # ------------------------------------------------ + # draw monte carlo samples from the nearest neighbor approximation + # later we will average over the monte-carlo posterior samples to attain the final posterior estimation + # hack! + while cnt < run_params["nmc"]: + ### hack! + signal.alarm( + 0 + ) # run_params["tmax_per_obj"]) # max time spent on one object in sec + try: + x = np.copy(observed) + # D. Jones edit + # idx_neighbor = np.random.choice(range(len(guess_ndata.T[0]))) + # while abs(y_train[idx_chi2_selected][idx_neighbor][-1] - observed[-1]) > 0.05: + # idx_neighbor = np.random.choice(range(len(guess_ndata.T[0]))) + + for j, idx in enumerate(not_valid_idx): ##range(len(not_valid_idx)): + x[not_valid_idx[j]] = np.random.choice( + guess_ndata.T[j] + ) # np.clip(kdes[j].resample(size=1),0,23) ### I think these crazy non-detections are destroying the fit? + # hack! + # x[not_valid_idx[j]] = kdes[j].resample(size=1) ##np.clip(kdes[j].resample(size=1),0,23) ### I think these crazy non-detections are destroying the fit? + # x[22:][not_valid_idx[j]] = y_train[idx_chi2_selected][idx_neighbor][22:][not_valid_idx[j]] + + # let's just randomly sample the neighbors instead of unpredictable toy noise model + x[22:][not_valid_idx[j]] = y_train[idx_chi2_selected][ + np.random.choice(range(len(guess_ndata.T[j]))) + ][22:][not_valid_idx[j]] + # x[not_valid_idx_unc[j]] = toy_noise( + # flux=x[not_valid_idx[j]], + # meds_sigs=sbi_params["toynoise_meds_sigs"][idx], + # stds_sigs=sbi_params["toynoise_stds_sigs"][idx], + # verbose=run_params["verbose"], + # )[1] + + # the noise model in the training isn't quite right + # Pan-STARRS in particular seems a little off + # we'll have to re-train at some point, but for now just pull + # uncertainties from the training sample + for idx, fname in zip(valid_idx, obs["filternames"][valid_idx]): + # if 'PanSTARRS' in fname or '2MASS' in fname or 'SDSS' in fname or 'DES' in fname: + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + # hack! + # x[22:-1][idx] = y_train[idx_chi2_selected][0][22:-1][idx] + x[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][idx] + # x[22:-1][idx] = y_train[idx_chi2_selected][idx_neighbor][22:-1][idx] + + # import pdb; pdb.set_trace() + # x = y_train[idx_chi2_selected][2150] ###guess_ndata.T[:,idx_neighbor] + + all_x.append(x) + + # ixtest = np.where(chi2_nei <= 100)[0] + # ixtest = np.where(chi2_nei == np.min(chi2_nei))[0] + # plt.errorbar(obs['wavelengths'],obs['mags'],obs['mags_unc'],fmt='.',zorder=100) + # plt.errorbar(obs['wavelengths'],obs['mags_sbi'],obs['mags_unc_sbi'],fmt='.') + # plt.errorbar(obs['wavelengths'],x[0:22],yerr=x[22:-1],fmt='.') + ##plt.errorbar(obs['wavelengths'],y_train[ixtest[4]][:22],yerr=y_train[ixtest[4]][22:-1],fmt='.') + # plt.errorbar(obs['wavelengths'],y_train[idx_chi2_selected[30]][:22],yerr=y_train[idx_chi2_selected[30]][22:-1],fmt='.',zorder=101) + # plt.ylim([21,16]) + # plt.savefig('tmp.png',dpi=200) + # x = y_train[idx_chi2_selected[30]] + # x[1] += np.log10(0.7290672783420541) + # import pdb; pdb.set_trace() + + # if we can't get one posterior sample in one second, we should move along + # to the next MC sample + do_continue = False + for tmax, npost in zip( + [1, run_params["tmax_per_iter"]], [1, run_params["nposterior"]] + ): + signal.alarm(tmax) # max time spent on one object in sec + try: + noiseless_theta = hatp_x_y.sample( + (npost,), + x=torch.as_tensor(x.astype(np.float32)).to(device), + show_progress_bars=False, + ) + except TimeoutException: + signal.alarm(0) + do_continue = True + break + + if do_continue: + continue + + signal.alarm(0) + + noiseless_theta = noiseless_theta.detach().numpy() + # if np.median(noiseless_theta[:,1]) > 9: + + ave_theta.append(noiseless_theta) + + ### debug! + # transient = Transient.objects.get(name='PS15bwh') + #### all these lines + if not "hi": + print("getting photometry") + observations = build_obs() # transient, "global") + model_components = build_model(obs) + model_mfrac = copy.deepcopy(model_components) # ["model"]) + # sps = build_sps() + for i in range(1): + print(i) + if i == 0: + ( + best_spec, + best_phot, + mfrac, + ) = model_components.predict( # ["model"] + noiseless_theta[i, :], + obs=observations, + sps=sps, # model_components["sps"] + ) + noiseless_theta[i, 1] -= np.log10(mfrac) + ( + best_spec, + best_phot, + mfrac, + ) = model_components.predict( # ["model"] + noiseless_theta[i, :], + obs=observations, + sps=sps, # model_components["sps"] + ) + else: + ( + best_spec, + best_phot_s, + mfrac, + ) = model_components.predict( # ["model"] + noiseless_theta[i, :], + obs=observations, + sps=sps, # model_components["sps"] + ) + noiseless_theta[i, 1] -= np.log10(mfrac) + ( + best_spec, + best_phot, + mfrac, + ) = model_components.predict( # ["model"] + noiseless_theta[i, :], + obs=observations, + sps=sps, # model_components["sps"] + ) + best_phot = np.vstack([best_phot, best_phot_s]) + print(-2.5 * np.log10(best_phot)) # np.median(best_phot,axis=0))) + import pdb + + pdb.set_trace() + # avg_theta = np.median(noiseless_theta,axis=0) + # avg_theta[1] -= np.log10(mfrac) + # best_spec2, best_phot2, mfrac2 = model_mfrac.predict( + # avg_theta, obs=observations, sps=build_sps() + # ) + + # theta_test = np.array([ 0.12240195, 8.47703558, -1.34368912, -3.72383187, -2.20004498, + # 3.03045638, 1.43847368, 2.05204451, 0.95743611, 0.55960846, + # 0.22913056, 1.28790968, -2.26064742, 1.67065051, -0.15686861, + # 1.51209086, 3.13108879, -0.39720158]) + # theta_test = np.array([ 0.08959557, 9.91042752, -0.16129237, -0.05564067, -0.47920846, 0.04121112, 0.03795635, 0.49733082, 0.02108837, 0.42442192, 0.17738165, 0.52207115, -0.82690121, 1.12715198, -1.3099881, 1.17192278, 10.65062175, -2.48147237]) + # obs, model, sps, noise = build_all(**run_params) + # best_spec, best_phot, mfrac = model.predict(theta_test, obs=observations, sps=build_sps()) + + # age_interp, allsfhs_interp, allMWA, allsfrs = getSFH( + # chain, theta_index=theta_index, rtn_chains=True, zred=zred + # ) + + ### (Pdb) sbi_params['theta_train'][idx_chi2_selected][0] + # array([ 0.11929193, 10.39623445, -1.32188065, 0.18913014, 0.16209094, + # 0.06441716, -0.19440235, 0.11935996, 0.37152376, 0.34494525, + # -0.42501956, 0.51024255, -2.06271797, 1.37400889, -1.21739342, + # 3.32965967, 1.28220919, -1.79931691]) + # (Pdb) noiseless_theta[0] + # array([ 0.11942666, 10.399073 , -1.2320234 , 0.11172969, 0.34717456, + # 0.3892678 , -0.6431147 , 0.24748906, -0.5234739 , 0.27822632, + # -0.52583283, 1.187651 , -1.2531726 , 1.9812987 , -1.2108788 , + # 2.0470366 , 2.0370784 , -2.9912286 ], dtype=float32) + + # import pdb; pdb.set_trace() + + cnt += 1 + # print(x) + if run_params["verbose"]: + if cnt % 10 == 0: + print("mc samples:", cnt) + + except TimeoutException: + cnt_timeout += 1 + else: + signal.alarm(0) + + # set max time + et = time.time() + elapsed_time = et - st # in secs + if elapsed_time / 60 > run_params["tmax_all"] or ( + cnt < run_params["nmc"] / 10 + and elapsed_time / 60 * 10 > run_params["tmax_all"] + ): + timeout_flag = True + use_res = False + break + # ------------------------------------------------ + + all_x = np.array(all_x) + # import pdb; pdb.set_trace() + all_x_flux = all_x.T[:nbands] + all_x_unc = all_x.T[nbands:] + y_guess = np.concatenate( + [np.median(all_x_flux, axis=1), np.median(all_x_unc, axis=1), [obs["redshift"]]] + ) + + return ave_theta, y_guess, use_res, timeout_flag, cnt + + +def lim_of_noisy_guass(obs, run_params, sbi_params): + """restrict the range over which we monte carlo the noise based on similar SEDs in the training set; + see sec. 4.1.1 for details + """ + + use_res = 1 + + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + noisy_mask = np.copy(obs["noisy_mask"]) + + noisy_idx = np.where(noisy_mask == True)[0] + not_noisy_idx = np.where(noisy_mask == False)[0] + + look_in_training = y_train[:, noisy_idx] + chi2_nei = chi2dof( + mags=look_in_training, obsphot=y_obs[noisy_idx], obsphot_unc=sig_obs[noisy_idx] + ) + + _chi2_thres = run_params["ini_chi2"] * 1 + while True: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 10: + chi2_nei_selected = y_train[idx_chi2_selected] + chi2_nei_selected = np.squeeze(chi2_nei_selected[:, noisy_idx]) + lims = [ + np.atleast_1d(np.min(chi2_nei_selected, axis=0)), + np.atleast_1d(np.max(chi2_nei_selected, axis=0)), + ] + if np.all((lims[0] - y_obs[noisy_idx]) < 0) and np.all( + (lims[1] - y_obs[noisy_idx]) > 0 + ): + break + _chi2_thres += 5 + if _chi2_thres > run_params["max_chi2"]: + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + print( + "Clipping the Gaussian, from which we MC noise, to be within the min & max of the magnitude at that band in the training set" + ) + use_res = 0 + # choose the args for clipping norm by the min & max of the magnitude at that band in the training set + lims = np.array( + [ + np.atleast_1d(np.min(y_train[:, noisy_idx], axis=0)), + np.atleast_1d(np.max(y_train[:, noisy_idx], axis=0)), + ] + ) + break + + return lims, use_res + + +def sbi_mcnoise(obs, run_params, sbi_params, max_neighbors=200): + """used when observations have out-of-distribution uncertainties; + see sec. 4.1.1 for details + """ + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("sbi mc noise") + + ave_theta = [] + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 # total number of bands + + noisy_mask = np.copy(obs["noisy_mask"]) + noisy_idx = np.where(noisy_mask == True)[0] + not_noisy_idx = np.where(noisy_mask == False)[0] + + # start time + st = time.time() + + lims, use_res = lim_of_noisy_guass( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + loc = y_obs[noisy_idx] + scale = sig_obs[noisy_idx] + + ### temporary for getting errors, because error model not good enough + chi2_nei = chi2dof(mags=y_train[:, :22], obsphot=y_obs, obsphot_unc=sig_obs) + + _chi2_thres = run_params["ini_chi2"] * 1 + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + + if _chi2_thres > run_params["max_chi2"]: + chi2_selected = y_train[:] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + else: + chi2_selected = y_train[idx_chi2_selected] + + cnt = 0 + cnt_timeout = 0 + timeout_flag = False + # ------------------------------------------------ + # draw monte carlo samples from a norm dist centered at x_obs and 1 sigma = 1 sigma uncertainty associated with x_obs + # later we will average over the those "noisy" posterior samples to attain the final posterior estimation + while cnt < run_params["nmc"]: + samp_y_guess = np.copy(observed) + samp_y_guess[noisy_idx] = stats.norm.rvs(loc=loc, scale=scale) + # ensure positive uncertainties + _nnflag = True + for ii, this_noisy_flux in enumerate(samp_y_guess[noisy_idx]): + # print(lims[0][ii], lims[1][ii]) + if this_noisy_flux > lims[0][ii] and this_noisy_flux < lims[1][ii]: + _nnflag &= True + else: + _nnflag &= False + + if _nnflag: + samp_y_guess[noisy_idx + nbands] = toy_noise( + flux=samp_y_guess[noisy_idx], + meds_sigs=sbi_params["toynoise_meds_sigs"][ii], + stds_sigs=sbi_params["toynoise_stds_sigs"][ii], + verbose=run_params["verbose"], + )[1] + # signal.alarm(run_params["tmax_per_obj"]) + + for idx, fname in enumerate(obs["filternames"]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + samp_y_guess[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][ + idx + ] + + do_continue = False + for tmax, npost in zip( + [run_params["tmax_per_iter"]], [run_params["nposterior"]] + ): + signal.alarm(tmax) # max time spent on one object in sec + try: + noiseless_theta = hatp_x_y.sample( + (run_params["nposterior"],), + x=torch.as_tensor(samp_y_guess).to(device), + show_progress_bars=False, + ) + except TimeoutException: + signal.alarm(0) + do_continue = True + break + + if do_continue: + continue + + noiseless_theta = noiseless_theta.detach().numpy() + + ave_theta.append(noiseless_theta) + + cnt += 1 + if run_params["verbose"]: + if cnt % 10 == 0: + print("mc samples:", cnt) + + # except TimeoutException: + # cnt_timeout += 1 + # else: + signal.alarm(0) + + # end time + et = time.time() + elapsed_time = et - st # in secs + if elapsed_time / 60 > run_params["tmax_all"] or ( + cnt < run_params["nmc"] / 10 + and elapsed_time / 60 * 10 > run_params["tmax_all"] + ): + timeout_flag = True + use_res = False + break + # ------------------------------------------------ + + return ave_theta, use_res, timeout_flag, cnt + + +def sbi_missing_and_noisy(obs, run_params, sbi_params): + """used when observations have missing data and out-of-distribution uncertainties. + fill in the missing bands first using the nearest neighbor approximation; + then mc the noisy bands + """ + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("sbi missing and noisy bands") + + ave_theta = [] + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags_sbi"]) + sig_obs = np.copy(obs["mags_unc_sbi"]) + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 + + noisy_mask = np.copy(obs["noisy_mask"]) + noisy_idx = np.where(noisy_mask == True)[0] + not_noisy_idx = np.where(noisy_mask == False)[0] + + invalid_mask = np.copy(obs["missing_mask"]) + y_obs_valid_only = y_obs[~invalid_mask] + valid_idx = np.where(~invalid_mask)[0] + not_valid_idx = np.where(invalid_mask)[0] + not_valid_idx_unc = not_valid_idx + nbands + + # ------------------------------------------------ + kdes, use_res_missing, idx_chi2_selected = gauss_approx_missingband( + obs, run_params, sbi_params + ) + + # start time + st = time.time() + + lims, use_res_noisy = lim_of_noisy_guass( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + loc = y_obs[noisy_idx] + scale = sig_obs[noisy_idx] + + cnt = 0 + cnt_timeout = 0 + timeout_flag = False + while cnt < run_params["nmc"]: + + samp_y_guess = np.copy(observed) + + # first, fill in the missing bands + for j in range(len(not_valid_idx)): + # samp_y_guess[not_valid_idx[j]] = kdes[j].resample(size=1) + # samp_y_guess[not_valid_idx_unc[j]] = toy_noise( + # flux=samp_y_guess[not_valid_idx[j]], + # meds_sigs=sbi_params["toynoise_meds_sigs"][not_valid_idx[j]], + # stds_sigs=sbi_params["toynoise_stds_sigs"][not_valid_idx[j]], + # verbose=run_params["verbose"], + # )[1] + samp_y_guess[not_valid_idx[j]] = y_train[idx_chi2_selected][ + np.random.choice(range(len(idx_chi2_selected))) + ][not_valid_idx[j]] + samp_y_guess[not_valid_idx_unc[j]] = y_train[idx_chi2_selected][ + np.random.choice(range(len(idx_chi2_selected))) + ][not_valid_idx_unc[j]] + + # second, deal with OOD noise + samp_y_guess[noisy_idx] = stats.norm.rvs(loc=loc, scale=scale) + _nnflag = True + for ii, this_noisy_flux in enumerate(samp_y_guess[noisy_idx]): + if this_noisy_flux > lims[0][ii] and this_noisy_flux < lims[1][ii]: + _nnflag &= True + else: + _nnflag &= False + + if _nnflag: + samp_y_guess[noisy_idx[ii] + nbands] = y_train[idx_chi2_selected][ + np.random.choice(range(len(idx_chi2_selected))) + ][noisy_idx[ii] + nbands] + # samp_y_guess[noisy_idx + nbands] = toy_noise( + # flux=samp_y_guess[noisy_idx[ii]], + # meds_sigs=sbi_params["toynoise_meds_sigs"][noisy_idx[ii]], + # stds_sigs=sbi_params["toynoise_stds_sigs"][noisy_idx[ii]], + # verbose=run_params["verbose"], + # )[1] + + # the noise model in the training isn't quite right + # Pan-STARRS in particular seems a little off + # we'll have to re-train at some point, but for now just pull + # uncertainties from the training sample + for idx, fname in zip(valid_idx, obs["filternames"][valid_idx]): + # if 'PanSTARRS' in fname or '2MASS' in fname or 'SDSS' in fname or 'DES' in fname: + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + samp_y_guess[22:][idx] = y_train[idx_chi2_selected][chc][22:][idx] + + # if we can't get one posterior sample in one second, we should move along + # to the next MC sample + do_continue = False + for tmax, npost in zip( + [1, run_params["tmax_per_iter"]], [1, run_params["nposterior"]] + ): + signal.alarm(tmax) # max time spent on one object in sec + try: + noiseless_theta = hatp_x_y.sample( + (npost,), + x=torch.as_tensor(samp_y_guess).to(device), + show_progress_bars=False, + ) + except TimeoutException: + signal.alarm(0) + do_continue = True + break + + if do_continue: + continue + + signal.alarm(0) + noiseless_theta = noiseless_theta.detach().numpy() + + ave_theta.append(noiseless_theta) + + cnt += 1 + if run_params["verbose"]: + if cnt % 10 == 0: + print("mc samples:", cnt) + + # end time + et = time.time() + elapsed_time = et - st # in secs + if elapsed_time / 60 > run_params["tmax_all"] or ( + cnt < run_params["nmc"] / 10 + and elapsed_time / 60 * 10 > run_params["tmax_all"] + ): + timeout_flag = 1 + use_res = 0 + break + # ------------------------------------------------ + + if use_res_missing == 1 and use_res_noisy == 1 and timeout_flag == 0: + use_res = 1 + + return ave_theta, use_res, timeout_flag, cnt + + +def sbi_baseline(obs, run_params, sbi_params, max_neighbors=200): + signal.signal(signal.SIGALRM, timeout_handler) + + if run_params["verbose"]: + print("baseline sbi") + + flags = { + "use_res": 0, # True if sbi++ succeeds; False if otherwise. + # below are for bookkeeping + "timeout": 0, + "use_res_missing": 0, # True if sbi++ for missing bands succeeds + "use_res_noisy": 0, # True if sbi++ for noisy bands succeeds + "noisy_data": False, # True if sbi++ (noisy data) is called + "missing_data": False, # True if sbi++ (missing data) is called + "nsamp_missing": -99, # number of MC samples drawn + "nsamp_noisy": -99, # number of MC samples drawn + } + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags"]) + sig_obs = np.copy(obs["mags_unc"]) + # copy the observed data to be used by sbi + # missing data, if any, will be filled in later + obs["mags_sbi"] = y_obs + obs["mags_unc_sbi"] = sig_obs + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 # total number of bands + + flags["use_res"] = 1 + flags["timeout"] = False + + ### temporary for getting errors, because error model not good enough + chi2_nei = chi2dof(mags=y_train[:, :22], obsphot=y_obs, obsphot_unc=sig_obs) + + _chi2_thres = run_params["ini_chi2"] * 1 + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + + if _chi2_thres > run_params["max_chi2"]: + chi2_selected = y_train[:] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + else: + chi2_selected = y_train[idx_chi2_selected] + + # ------------------------------------------------ + # call baseline sbi to draw posterior samples + signal.alarm(run_params["tmax_per_obj"]) # max time spent on one object in sec + + x = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + + for idx, fname in enumerate(obs["filternames"]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + x[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][idx] + + try: + ave_theta = hatp_x_y.sample( + (run_params["np_baseline"],), + x=torch.as_tensor(x.astype(np.float32)).to(device), + show_progress_bars=False, + ) + ave_theta = ave_theta.detach().numpy() + except TimeoutException: + flags["timeout"] = True + ave_theta = [np.nan] + if run_params["verbose"]: + print("timeout!") + else: + signal.alarm(0) + # ------------------------------------------------ + + return ave_theta, obs, flags + + +def sbi_pp(obs, run_params, sbi_params, max_neighbors=200): + """wrapper for sbi++; this should be the only function needed to be called outside this scipt under normal circumstances + + obs: a dictionary at least containing + - "mags": observed photometry, unit must match the training set + - "mags_unc": observed uncertainty, unit must match the training set + run_params: a dictionary at least containing + - "" + + """ + signal.signal(signal.SIGALRM, timeout_handler) + + flags = { + "use_res": 0, # True if sbi++ succeeds; False if otherwise. + # below are for bookkeeping + "timeout": 0, + "use_res_missing": 0, # True if sbi++ for missing bands succeeds + "use_res_noisy": 0, # True if sbi++ for noisy bands succeeds + "noisy_data": False, # True if sbi++ (noisy data) is called + "missing_data": False, # True if sbi++ (missing data) is called + "nsamp_missing": -99, # number of MC samples drawn + "nsamp_noisy": -99, # number of MC samples drawn + } + + hatp_x_y = sbi_params["hatp_x_y"] + y_train = sbi_params["y_train"] + + y_obs = np.copy(obs["mags"]) + sig_obs = np.copy(obs["mags_unc"]) + # copy the observed data to be used by sbi + # missing data, if any, will be filled in later + obs["mags_sbi"] = y_obs + obs["mags_unc_sbi"] = sig_obs + observed = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + nbands = y_train.shape[1] // 2 # total number of bands + + # decide if we need to deal with missing bands + obs["missing_mask"] = np.isnan(y_obs) + missing_mask = np.isnan(y_obs) # idx of missing bands + # decide if we need to deal with noisy bands + noisy_mask = np.zeros_like(y_obs, dtype=bool) + for j in range(nbands): + _toynoise = toy_noise( + flux=y_obs[j], + meds_sigs=sbi_params["toynoise_meds_sigs"][j], + stds_sigs=sbi_params["toynoise_stds_sigs"][j], + verbose=run_params["verbose"], + ) + noisy_mask[j] = (sig_obs[j] - _toynoise[1]) / _toynoise[2] >= run_params[ + "noisy_sig" + ] + # if noisy_mask[j]: + # import pdb; pdb.set_trace() + noisy_mask &= np.isfinite(y_obs) # idx of noisy bands + obs["noisy_mask"] = noisy_mask + + ave_theta = [np.nan] + if np.any(missing_mask): + flags["missing_data"] = True + if np.any(noisy_mask): + flags["noisy_data"] = True + + if not flags["missing_data"] and not flags["noisy_data"]: + flags["use_res"] = 1 + flags["timeout"] = False + if run_params["verbose"]: + print("baseline sbi") + + ### temporary for getting errors, because error model not good enough + chi2_nei = chi2dof(mags=y_train[:, :22], obsphot=y_obs, obsphot_unc=sig_obs) + + _chi2_thres = run_params["ini_chi2"] * 1 + while _chi2_thres <= run_params["max_chi2"]: + idx_chi2_selected = np.where(chi2_nei <= _chi2_thres)[0] + if len(idx_chi2_selected) >= 30: + break + else: + _chi2_thres += 5 + + if _chi2_thres > run_params["max_chi2"]: + chi2_selected = y_train[:] + idx_chi2_selected = np.argsort(chi2_nei)[0:max_neighbors] + if run_params["verbose"]: + print("Failed to find sufficient number of nearest neighbors!") + else: + chi2_selected = y_train[idx_chi2_selected] + + signal.alarm(run_params["tmax_per_obj"]) # max time spent on one object in sec + + x = np.concatenate([y_obs, sig_obs, [obs["redshift"]]]) + + for idx, fname in enumerate(obs["filternames"]): + chc = np.random.choice(range(len(y_train[idx_chi2_selected]))) + x[22:-1][idx] = y_train[idx_chi2_selected][chc][22:-1][idx] + + try: + ave_theta = hatp_x_y.sample( + (run_params["np_baseline"],), + x=torch.as_tensor(x.astype(np.float32)).to(device), + show_progress_bars=False, + ) + ave_theta = ave_theta.detach().numpy() + except TimeoutException: + flags["timeout"] = True + ave_theta = [np.nan] + else: + signal.alarm(0) + + return ave_theta, obs, flags + + if flags["missing_data"] and flags["noisy_data"]: + ( + ave_theta, + flags["use_res"], + flags["timeout"], + flags["nsamp_noisy"], + ) = sbi_missing_and_noisy(obs=obs, run_params=run_params, sbi_params=sbi_params) + if flags["timeout"]: + for i, mask in enumerate( + [ + (obs["bands"] == "WISE_W3") | (obs["bands"] == "WISE_W4"), + (obs["bands"] == "WISE_W1") | (obs["bands"] == "WISE_W2"), + (obs["bands"] == "GALEX_NUV") | (obs["bands"] == "GALEX_FUV"), + ] + ): + print("timeout! trying without some filters") + if i == 2: + run_params["tmax_all"] *= 3 + obs["missing_mask"][mask] = True + ( + ave_theta, + flags["use_res"], + flags["timeout"], + flags["nsamp_noisy"], + ) = sbi_missing_and_noisy( + obs=obs, run_params=run_params, sbi_params=sbi_params + ) + if not flags["timeout"]: + break + + # separate cases + if flags["missing_data"] and not flags["noisy_data"]: + res = sbi_missingband( + obs=obs, run_params=run_params, sbi_params=sbi_params, seconditer=True + ) + # if len(res) != 5: + # if sbi_flag == "chi2 fail": + # obs["missing_mask"] = res["missing_mask"][:] + # res = sbi_missingband( + # obs=obs, run_params=run_params, sbi_params=sbi_params, seconditer=True + # ) + # else: + # if things timed out, then we should try dropping + # some problematic filters + (ave_theta, flags["use_res"], flags["nsamp_noisy"], flags["timeout"], cnt) = res + if flags["timeout"]: + for i, mask in enumerate( + [ + (obs["bands"] == "WISE_W3") | (obs["bands"] == "WISE_W4"), + (obs["bands"] == "WISE_W1") | (obs["bands"] == "WISE_W2"), + (obs["bands"] == "GALEX_NUV") | (obs["bands"] == "GALEX_FUV"), + ] + ): + print("timeout! trying without some filters") + if i == 2: + run_params["tmax_all"] *= 3 + + obs["missing_mask"][mask] = True + ( + ave_theta, + flags["use_res"], + flags["nsamp_noisy"], + flags["timeout"], + cnt, + ) = sbi_missingband( + obs=obs, + run_params=run_params, + sbi_params=sbi_params, + seconditer=True, + ) + if not flags["timeout"]: + break + + if len(res) != 5: + raise RuntimeError( + "couldnt get good chi2 for nearest neighbors. Aborting" + ) + + ( + ave_theta, + y_guess, + flags["use_res_missing"], + flags["timeout"], + flags["nsamp_missing"], + ) = res + + flags["use_res"] = flags["use_res_missing"] * 1 + + if not flags["missing_data"] and flags["noisy_data"]: + # mc the noisy bands + ( + ave_theta, + flags["use_res_noisy"], + flags["timeout"], + ### flags["nsamp_noisy"], + cnt, + ) = sbi_mcnoise(obs=obs, run_params=run_params, sbi_params=sbi_params) + flags["use_res"] = flags["use_res_noisy"] * 1 + + ave_theta = np.array(ave_theta) + try: + ave_theta = np.concatenate(ave_theta) + except Exception as e: + pass + + return ave_theta, obs, flags diff --git a/app/host/SBI/snr_vs_mag.py b/app/host/SBI/snr_vs_mag.py new file mode 100644 index 00000000..3ea6ac0d --- /dev/null +++ b/app/host/SBI/snr_vs_mag.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# D. Jones - 5/26/23 +""" +Create S/N vs. mag files for every filter +Just ASCII file output should be fine +""" +import h5py +import numpy as np +from host.models import * +from scipy.stats import binned_statistic + + +def main(): + + for f in Filter.objects.all(): + phot = AperturePhotometry.objects.filter(filter=f, magnitude__isnull=False) + mag = np.array(phot.values_list("magnitude", flat=True)) + if not len(mag): + continue + mag_error = np.array(phot.values_list("magnitude_error", flat=True)) + flux = np.array(phot.values_list("flux", flat=True)) + flux_error = np.array(phot.values_list("flux_error", flat=True)) + + magbins = np.arange(np.min(mag), np.max(mag), 0.25) + snr_binned = binned_statistic( + mag, flux / flux_error, bins=magbins, statistic="mean" + ).statistic + count_binned = binned_statistic( + mag, flux / flux_error, bins=magbins, statistic="count" + ).statistic + + ### we need some minimum number of counts + ### and we need to interpolate over the missing points + mincount = 5 + magbins = (magbins[1:] + magbins[:-1]) / 2.0 + idx = np.where(count_binned > mincount)[0] + idx_valid = np.arange(np.min(idx), np.max(idx) + 1, 1) + + snr_bins_interp = np.interp(magbins[idx_valid], magbins[idx], snr_binned[idx]) + magbins = magbins[idx_valid] + + with open(f"host/SBI/snrfiles/{f.name}_magvsnr.txt", "w") as fout: + for m, s in zip(magbins, snr_bins_interp): + print(f"{m:.3f} {s:.3f}", file=fout) + + return diff --git a/app/host/SBI/snrfiles/2MASS_H_magvsnr.txt b/app/host/SBI/snrfiles/2MASS_H_magvsnr.txt new file mode 100644 index 00000000..0ca38da7 --- /dev/null +++ b/app/host/SBI/snrfiles/2MASS_H_magvsnr.txt @@ -0,0 +1,38 @@ +10.251 102.154 +10.501 88.819 +10.751 81.025 +11.001 66.689 +11.251 60.840 +11.501 57.260 +11.751 38.349 +12.001 36.082 +12.251 36.400 +12.501 31.461 +12.751 25.081 +13.001 24.932 +13.251 23.476 +13.501 18.236 +13.751 17.191 +14.001 17.036 +14.251 13.051 +14.501 11.815 +14.751 11.906 +15.001 8.395 +15.251 8.130 +15.501 7.529 +15.751 6.889 +16.001 5.267 +16.251 4.883 +16.501 3.658 +16.751 3.944 +17.001 3.034 +17.251 2.283 +17.501 2.528 +17.751 1.964 +18.001 1.278 +18.251 1.744 +18.501 1.456 +18.751 0.732 +19.001 0.891 +19.251 1.053 +19.501 0.818 diff --git a/app/host/SBI/snrfiles/2MASS_J_magvsnr.txt b/app/host/SBI/snrfiles/2MASS_J_magvsnr.txt new file mode 100644 index 00000000..f02bc061 --- /dev/null +++ b/app/host/SBI/snrfiles/2MASS_J_magvsnr.txt @@ -0,0 +1,40 @@ +11.136 83.192 +11.386 89.193 +11.636 64.847 +11.886 59.652 +12.136 63.543 +12.386 52.373 +12.636 36.116 +12.886 37.328 +13.136 33.797 +13.386 28.133 +13.636 23.833 +13.886 23.819 +14.136 20.900 +14.386 17.309 +14.636 17.818 +14.886 16.679 +15.136 13.865 +15.386 12.714 +15.636 9.538 +15.886 8.468 +16.136 7.098 +16.386 6.917 +16.636 5.224 +16.886 5.280 +17.136 3.906 +17.386 3.517 +17.636 3.691 +17.886 2.803 +18.136 2.097 +18.386 2.255 +18.636 2.118 +18.886 1.783 +19.136 1.128 +19.386 1.180 +19.636 1.103 +19.886 1.026 +20.136 0.896 +20.386 0.859 +20.636 0.821 +20.886 0.783 diff --git a/app/host/SBI/snrfiles/2MASS_K_magvsnr.txt b/app/host/SBI/snrfiles/2MASS_K_magvsnr.txt new file mode 100644 index 00000000..dfeedb70 --- /dev/null +++ b/app/host/SBI/snrfiles/2MASS_K_magvsnr.txt @@ -0,0 +1,41 @@ +10.204 69.532 +10.454 69.387 +10.704 52.577 +10.954 48.825 +11.204 47.694 +11.454 31.483 +11.704 26.719 +11.954 29.255 +12.204 28.379 +12.454 21.182 +12.704 19.389 +12.954 18.060 +13.204 17.080 +13.454 12.613 +13.704 12.310 +13.954 12.281 +14.204 10.886 +14.454 8.578 +14.704 7.389 +14.954 5.941 +15.204 6.674 +15.454 4.904 +15.704 4.210 +15.954 3.483 +16.204 2.765 +16.454 3.453 +16.704 2.106 +16.954 2.134 +17.204 2.141 +17.454 1.454 +17.704 1.757 +17.954 1.399 +18.204 1.219 +18.454 1.038 +18.704 0.929 +18.954 0.819 +19.204 0.821 +19.454 0.797 +19.704 0.773 +19.954 0.536 +20.204 0.299 diff --git a/app/host/SBI/snrfiles/DES_g_magvsnr.txt b/app/host/SBI/snrfiles/DES_g_magvsnr.txt new file mode 100644 index 00000000..ac213267 --- /dev/null +++ b/app/host/SBI/snrfiles/DES_g_magvsnr.txt @@ -0,0 +1,33 @@ +14.095 1498.834 +14.345 1351.754 +14.595 1539.793 +14.845 1354.735 +15.095 1127.177 +15.345 825.908 +15.595 665.150 +15.845 842.000 +16.095 728.931 +16.345 676.029 +16.595 562.362 +16.845 488.038 +17.095 383.174 +17.345 394.547 +17.595 349.924 +17.845 320.220 +18.095 242.899 +18.345 192.111 +18.595 193.220 +18.845 169.726 +19.095 142.447 +19.345 155.718 +19.595 130.075 +19.845 111.698 +20.095 98.763 +20.345 92.329 +20.595 75.412 +20.845 72.376 +21.095 69.339 +21.345 56.024 +21.595 45.311 +21.845 44.467 +22.095 32.316 diff --git a/app/host/SBI/snrfiles/DES_r_magvsnr.txt b/app/host/SBI/snrfiles/DES_r_magvsnr.txt new file mode 100644 index 00000000..3a64ffc7 --- /dev/null +++ b/app/host/SBI/snrfiles/DES_r_magvsnr.txt @@ -0,0 +1,35 @@ +13.220 1380.660 +13.470 1515.118 +13.720 1440.689 +13.970 1075.392 +14.220 1171.863 +14.470 1122.357 +14.720 743.579 +14.970 847.772 +15.220 701.530 +15.470 607.197 +15.720 638.522 +15.970 581.440 +16.220 401.855 +16.470 350.811 +16.720 509.118 +16.970 344.474 +17.220 283.598 +17.470 204.505 +17.720 224.727 +17.970 158.058 +18.220 173.284 +18.470 165.233 +18.720 150.001 +18.970 109.567 +19.220 96.969 +19.470 110.299 +19.720 74.490 +19.970 85.434 +20.220 79.614 +20.470 73.795 +20.720 46.062 +20.970 45.486 +21.220 41.995 +21.470 38.503 +21.720 25.180 diff --git a/app/host/SBI/snrfiles/DES_z_magvsnr.txt b/app/host/SBI/snrfiles/DES_z_magvsnr.txt new file mode 100644 index 00000000..46badb51 --- /dev/null +++ b/app/host/SBI/snrfiles/DES_z_magvsnr.txt @@ -0,0 +1,34 @@ +12.541 745.456 +12.791 851.234 +13.041 877.348 +13.291 673.375 +13.541 603.870 +13.791 552.570 +14.041 539.094 +14.291 485.266 +14.541 386.892 +14.791 328.524 +15.041 298.223 +15.291 266.563 +15.541 305.988 +15.791 228.101 +16.041 200.223 +16.291 211.366 +16.541 177.799 +16.791 139.803 +17.041 132.773 +17.291 124.820 +17.541 88.054 +17.791 88.806 +18.041 93.157 +18.291 64.252 +18.541 68.874 +18.791 48.970 +19.041 45.735 +19.291 51.471 +19.541 35.820 +19.791 35.549 +20.041 35.278 +20.291 26.431 +20.541 25.123 +20.791 24.644 diff --git a/app/host/SBI/snrfiles/GALEX_FUV_magvsnr.txt b/app/host/SBI/snrfiles/GALEX_FUV_magvsnr.txt new file mode 100644 index 00000000..467ebe2b --- /dev/null +++ b/app/host/SBI/snrfiles/GALEX_FUV_magvsnr.txt @@ -0,0 +1,35 @@ +17.728 24.130 +17.978 25.067 +18.228 26.003 +18.478 28.711 +18.728 20.557 +18.978 18.112 +19.228 19.044 +19.478 15.839 +19.728 14.159 +19.978 16.936 +20.228 8.754 +20.478 12.591 +20.728 11.202 +20.978 11.820 +21.228 6.099 +21.478 6.566 +21.728 7.327 +21.978 4.138 +22.228 5.171 +22.478 4.367 +22.728 3.428 +22.978 2.250 +23.228 3.982 +23.478 1.970 +23.728 2.820 +23.978 5.000 +24.228 1.573 +24.478 2.968 +24.728 1.808 +24.978 2.375 +25.228 1.693 +25.478 1.010 +25.728 1.313 +25.978 1.615 +26.228 1.918 diff --git a/app/host/SBI/snrfiles/GALEX_NUV_magvsnr.txt b/app/host/SBI/snrfiles/GALEX_NUV_magvsnr.txt new file mode 100644 index 00000000..518a1950 --- /dev/null +++ b/app/host/SBI/snrfiles/GALEX_NUV_magvsnr.txt @@ -0,0 +1,41 @@ +17.132 104.949 +17.382 66.782 +17.632 41.550 +17.882 87.020 +18.132 39.955 +18.382 51.707 +18.632 43.578 +18.882 38.728 +19.132 42.767 +19.382 33.679 +19.632 32.440 +19.882 33.081 +20.132 33.721 +20.382 34.662 +20.632 14.149 +20.882 14.985 +21.132 18.325 +21.382 11.576 +21.632 8.577 +21.882 8.195 +22.132 16.709 +22.382 10.025 +22.632 9.107 +22.882 14.359 +23.132 4.603 +23.382 8.216 +23.632 8.824 +23.882 5.238 +24.132 1.962 +24.382 4.152 +24.632 3.315 +24.882 2.478 +25.132 2.786 +25.382 1.187 +25.632 1.225 +25.882 1.263 +26.132 1.301 +26.382 1.340 +26.632 1.378 +26.882 1.416 +27.132 0.286 diff --git a/app/host/SBI/snrfiles/PanSTARRS_g_magvsnr.txt b/app/host/SBI/snrfiles/PanSTARRS_g_magvsnr.txt new file mode 100644 index 00000000..05d6aadd --- /dev/null +++ b/app/host/SBI/snrfiles/PanSTARRS_g_magvsnr.txt @@ -0,0 +1,42 @@ +13.083 1816.397 +13.333 1804.632 +13.583 1692.624 +13.833 1580.616 +14.083 1238.591 +14.333 1113.054 +14.583 934.062 +14.833 1182.527 +15.083 946.046 +15.333 681.405 +15.583 808.538 +15.833 651.426 +16.083 572.737 +16.333 502.708 +16.583 482.651 +16.833 428.387 +17.083 356.152 +17.333 306.005 +17.583 228.385 +17.833 294.120 +18.083 230.318 +18.333 165.480 +18.583 143.292 +18.833 155.063 +19.083 114.239 +19.333 95.372 +19.583 98.135 +19.833 81.517 +20.083 69.577 +20.333 55.760 +20.583 46.804 +20.833 39.828 +21.083 34.009 +21.333 30.690 +21.583 27.400 +21.833 26.572 +22.083 15.079 +22.333 14.387 +22.583 13.695 +22.833 13.002 +23.083 12.310 +23.333 11.618 diff --git a/app/host/SBI/snrfiles/PanSTARRS_i_magvsnr.txt b/app/host/SBI/snrfiles/PanSTARRS_i_magvsnr.txt new file mode 100644 index 00000000..c56d88a2 --- /dev/null +++ b/app/host/SBI/snrfiles/PanSTARRS_i_magvsnr.txt @@ -0,0 +1,38 @@ +12.265 3263.131 +12.515 2500.063 +12.765 2768.057 +13.015 2803.113 +13.265 2195.514 +13.515 2092.378 +13.765 2526.691 +14.015 1568.585 +14.265 1240.408 +14.515 1788.709 +14.765 1278.834 +15.015 1309.244 +15.265 877.523 +15.515 866.484 +15.765 837.605 +16.015 814.952 +16.265 560.307 +16.515 524.885 +16.765 501.233 +17.015 451.916 +17.265 368.273 +17.515 346.580 +17.765 249.223 +18.015 244.013 +18.265 233.799 +18.515 160.425 +18.765 177.281 +19.015 101.225 +19.265 113.575 +19.515 86.084 +19.765 79.387 +20.015 64.853 +20.265 52.716 +20.515 55.557 +20.765 38.368 +21.015 40.815 +21.265 39.449 +21.515 34.699 diff --git a/app/host/SBI/snrfiles/PanSTARRS_r_magvsnr.txt b/app/host/SBI/snrfiles/PanSTARRS_r_magvsnr.txt new file mode 100644 index 00000000..02d0d546 --- /dev/null +++ b/app/host/SBI/snrfiles/PanSTARRS_r_magvsnr.txt @@ -0,0 +1,40 @@ +12.358 2631.265 +12.608 2907.569 +12.858 2617.053 +13.108 2326.538 +13.358 2606.377 +13.608 1765.747 +13.858 1556.996 +14.108 1669.835 +14.358 1408.436 +14.608 1108.200 +14.858 1387.527 +15.108 992.778 +15.358 1116.654 +15.608 758.910 +15.858 639.770 +16.108 755.693 +16.358 593.008 +16.608 447.057 +16.858 382.013 +17.108 381.960 +17.358 369.867 +17.608 300.776 +17.858 231.685 +18.108 200.068 +18.358 218.117 +18.608 160.979 +18.858 149.169 +19.108 140.760 +19.358 92.559 +19.608 98.273 +19.858 75.177 +20.108 57.984 +20.358 63.477 +20.608 56.354 +20.858 35.442 +21.108 39.512 +21.358 24.827 +21.608 19.557 +21.858 36.308 +22.108 22.217 diff --git a/app/host/SBI/snrfiles/PanSTARRS_y_magvsnr.txt b/app/host/SBI/snrfiles/PanSTARRS_y_magvsnr.txt new file mode 100644 index 00000000..953ff904 --- /dev/null +++ b/app/host/SBI/snrfiles/PanSTARRS_y_magvsnr.txt @@ -0,0 +1,44 @@ +12.039 902.098 +12.289 823.899 +12.539 776.054 +12.789 890.464 +13.039 639.816 +13.289 728.412 +13.539 588.615 +13.789 439.420 +14.039 535.072 +14.289 407.813 +14.539 383.396 +14.789 328.718 +15.039 367.699 +15.289 238.024 +15.539 252.160 +15.789 200.699 +16.039 157.084 +16.289 167.880 +16.539 141.717 +16.789 107.414 +17.039 103.129 +17.289 82.735 +17.539 71.061 +17.789 65.141 +18.039 45.932 +18.289 52.535 +18.539 38.323 +18.789 28.730 +19.039 29.652 +19.289 22.043 +19.539 19.934 +19.789 20.146 +20.039 12.408 +20.289 13.348 +20.539 8.204 +20.789 12.074 +21.039 10.782 +21.289 9.570 +21.539 8.358 +21.789 7.147 +22.039 5.935 +22.289 4.723 +22.539 3.449 +22.789 2.175 diff --git a/app/host/SBI/snrfiles/PanSTARRS_z_magvsnr.txt b/app/host/SBI/snrfiles/PanSTARRS_z_magvsnr.txt new file mode 100644 index 00000000..7adeb0de --- /dev/null +++ b/app/host/SBI/snrfiles/PanSTARRS_z_magvsnr.txt @@ -0,0 +1,37 @@ +12.239 1816.051 +12.489 1579.939 +12.739 1498.990 +12.989 1771.086 +13.239 1134.047 +13.489 1366.779 +13.739 1034.712 +13.989 732.335 +14.239 1009.289 +14.489 714.963 +14.739 825.701 +14.989 634.964 +15.239 507.439 +15.489 516.615 +15.739 465.358 +15.989 403.622 +16.239 291.560 +16.489 313.410 +16.739 268.340 +16.989 230.473 +17.239 191.684 +17.489 153.529 +17.739 138.912 +17.989 119.081 +18.239 105.602 +18.489 98.028 +18.739 79.078 +18.989 62.257 +19.239 52.037 +19.489 42.552 +19.739 42.116 +19.989 33.592 +20.239 31.027 +20.489 21.277 +20.739 17.550 +20.989 24.915 +21.239 19.468 diff --git a/app/host/SBI/snrfiles/SDSS_g_magvsnr.txt b/app/host/SBI/snrfiles/SDSS_g_magvsnr.txt new file mode 100644 index 00000000..752a9027 --- /dev/null +++ b/app/host/SBI/snrfiles/SDSS_g_magvsnr.txt @@ -0,0 +1,34 @@ +14.171 320.459 +14.421 277.370 +14.671 250.209 +14.921 223.221 +15.171 206.050 +15.421 177.197 +15.671 167.197 +15.921 144.895 +16.171 126.240 +16.421 114.309 +16.671 101.213 +16.921 90.979 +17.171 74.832 +17.421 70.992 +17.671 64.106 +17.921 57.258 +18.171 50.049 +18.421 42.308 +18.671 38.576 +18.921 34.578 +19.171 30.876 +19.421 27.397 +19.671 25.134 +19.921 21.651 +20.171 19.707 +20.421 17.083 +20.671 14.270 +20.921 12.810 +21.171 11.349 +21.421 9.581 +21.671 8.652 +21.921 7.605 +22.171 6.557 +22.421 5.692 diff --git a/app/host/SBI/snrfiles/SDSS_i_magvsnr.txt b/app/host/SBI/snrfiles/SDSS_i_magvsnr.txt new file mode 100644 index 00000000..2808d971 --- /dev/null +++ b/app/host/SBI/snrfiles/SDSS_i_magvsnr.txt @@ -0,0 +1,34 @@ +12.891 510.577 +13.141 469.398 +13.391 398.071 +13.641 365.095 +13.891 344.057 +14.141 292.944 +14.391 276.183 +14.641 253.859 +14.891 216.827 +15.141 197.841 +15.391 169.604 +15.641 151.986 +15.891 133.225 +16.141 128.252 +16.391 101.205 +16.641 94.192 +16.891 80.591 +17.141 73.671 +17.391 64.609 +17.641 56.486 +17.891 48.863 +18.141 45.051 +18.391 40.223 +18.641 36.843 +18.891 29.008 +19.141 27.686 +19.391 23.375 +19.641 20.744 +19.891 18.410 +20.141 15.534 +20.391 14.094 +20.641 12.655 +20.891 11.263 +21.141 9.872 diff --git a/app/host/SBI/snrfiles/SDSS_r_magvsnr.txt b/app/host/SBI/snrfiles/SDSS_r_magvsnr.txt new file mode 100644 index 00000000..05f2ec34 --- /dev/null +++ b/app/host/SBI/snrfiles/SDSS_r_magvsnr.txt @@ -0,0 +1,35 @@ +13.464 426.609 +13.714 383.957 +13.964 336.035 +14.214 315.580 +14.464 276.271 +14.714 247.280 +14.964 221.881 +15.214 199.554 +15.464 174.376 +15.714 155.712 +15.964 139.547 +16.214 122.530 +16.464 109.725 +16.714 95.082 +16.964 87.702 +17.214 74.689 +17.464 68.954 +17.714 59.335 +17.964 50.392 +18.214 47.793 +18.464 41.991 +18.714 36.341 +18.964 33.509 +19.214 28.201 +19.464 26.647 +19.714 22.223 +19.964 19.113 +20.214 17.072 +20.464 15.032 +20.714 12.883 +20.964 12.683 +21.214 10.437 +21.464 9.592 +21.714 8.748 +21.964 6.734 diff --git a/app/host/SBI/snrfiles/SDSS_u_magvsnr.txt b/app/host/SBI/snrfiles/SDSS_u_magvsnr.txt new file mode 100644 index 00000000..1211fafc --- /dev/null +++ b/app/host/SBI/snrfiles/SDSS_u_magvsnr.txt @@ -0,0 +1,32 @@ +15.472 100.880 +15.722 70.351 +15.972 83.389 +16.222 62.216 +16.472 59.710 +16.722 53.628 +16.972 55.468 +17.222 42.721 +17.472 38.782 +17.722 35.022 +17.972 27.915 +18.222 29.218 +18.472 28.836 +18.722 24.175 +18.972 22.590 +19.222 18.918 +19.472 14.276 +19.722 12.107 +19.972 13.035 +20.222 10.369 +20.472 7.561 +20.722 7.829 +20.972 6.816 +21.222 6.271 +21.472 5.207 +21.722 4.501 +21.972 3.371 +22.222 3.240 +22.472 3.049 +22.722 2.859 +22.972 2.066 +23.222 1.592 diff --git a/app/host/SBI/snrfiles/SDSS_z_magvsnr.txt b/app/host/SBI/snrfiles/SDSS_z_magvsnr.txt new file mode 100644 index 00000000..14b09577 --- /dev/null +++ b/app/host/SBI/snrfiles/SDSS_z_magvsnr.txt @@ -0,0 +1,35 @@ +12.587 344.298 +12.837 347.120 +13.087 281.064 +13.337 262.167 +13.587 226.162 +13.837 190.157 +14.087 200.053 +14.337 183.171 +14.587 163.299 +14.837 175.851 +15.087 123.774 +15.337 108.292 +15.587 110.638 +15.837 85.414 +16.087 70.256 +16.337 63.651 +16.587 53.619 +16.837 50.238 +17.087 43.748 +17.337 33.677 +17.587 29.818 +17.837 28.311 +18.087 23.905 +18.337 20.654 +18.587 23.096 +18.837 14.009 +19.087 15.178 +19.337 11.158 +19.587 8.050 +19.837 10.615 +20.087 5.601 +20.337 5.399 +20.587 6.099 +20.837 3.018 +21.087 5.603 diff --git a/app/host/SBI/snrfiles/WISE_W1_magvsnr.txt b/app/host/SBI/snrfiles/WISE_W1_magvsnr.txt new file mode 100644 index 00000000..a3c44685 --- /dev/null +++ b/app/host/SBI/snrfiles/WISE_W1_magvsnr.txt @@ -0,0 +1,46 @@ +9.955 896.428 +10.205 868.423 +10.455 580.398 +10.705 819.889 +10.955 571.307 +11.205 477.974 +11.455 496.780 +11.705 432.305 +11.955 388.522 +12.205 318.152 +12.455 284.307 +12.705 319.957 +12.955 291.065 +13.205 170.694 +13.455 142.471 +13.705 245.832 +13.955 119.919 +14.205 139.959 +14.455 112.439 +14.705 94.710 +14.955 66.742 +15.205 86.793 +15.455 78.766 +15.705 68.093 +15.955 63.871 +16.205 48.698 +16.455 30.150 +16.705 48.499 +16.955 18.986 +17.205 27.107 +17.455 21.494 +17.705 26.835 +17.955 30.779 +18.205 17.344 +18.455 7.609 +18.705 7.996 +18.955 8.383 +19.205 10.515 +19.455 7.607 +19.705 4.653 +19.955 4.879 +20.205 5.105 +20.455 5.330 +20.705 5.556 +20.955 5.782 +21.205 3.417 diff --git a/app/host/SBI/snrfiles/WISE_W2_magvsnr.txt b/app/host/SBI/snrfiles/WISE_W2_magvsnr.txt new file mode 100644 index 00000000..f5004e81 --- /dev/null +++ b/app/host/SBI/snrfiles/WISE_W2_magvsnr.txt @@ -0,0 +1,46 @@ +9.589 569.064 +9.839 566.647 +10.089 564.231 +10.339 476.468 +10.589 437.646 +10.839 379.485 +11.089 304.809 +11.339 302.738 +11.589 256.046 +11.839 252.251 +12.089 116.095 +12.339 204.962 +12.589 170.444 +12.839 181.098 +13.089 105.873 +13.339 93.013 +13.589 134.440 +13.839 99.905 +14.089 63.594 +14.339 72.594 +14.589 69.651 +14.839 35.510 +15.089 41.823 +15.339 53.339 +15.589 36.689 +15.839 29.656 +16.089 25.770 +16.339 26.362 +16.589 18.262 +16.839 19.280 +17.089 15.059 +17.339 18.528 +17.589 18.100 +17.839 14.429 +18.089 10.376 +18.339 3.785 +18.589 5.034 +18.839 6.282 +19.089 5.212 +19.339 4.833 +19.589 4.455 +19.839 3.528 +20.089 2.600 +20.339 2.936 +20.589 3.272 +20.839 2.582 diff --git a/app/host/SBI/snrfiles/WISE_W3_magvsnr.txt b/app/host/SBI/snrfiles/WISE_W3_magvsnr.txt new file mode 100644 index 00000000..7e515c3d --- /dev/null +++ b/app/host/SBI/snrfiles/WISE_W3_magvsnr.txt @@ -0,0 +1,42 @@ +6.708 123.838 +6.958 172.067 +7.208 220.297 +7.458 268.526 +7.708 718.484 +7.958 427.705 +8.208 281.304 +8.458 334.133 +8.708 137.463 +8.958 177.734 +9.208 133.900 +9.458 96.406 +9.708 155.706 +9.958 99.294 +10.208 137.089 +10.458 129.070 +10.708 48.637 +10.958 91.261 +11.208 56.861 +11.458 65.960 +11.708 54.875 +11.958 55.912 +12.208 54.285 +12.458 53.958 +12.708 22.026 +12.958 30.829 +13.208 14.193 +13.458 13.210 +13.708 17.521 +13.958 10.946 +14.208 14.874 +14.458 7.505 +14.708 6.917 +14.958 9.443 +15.208 6.776 +15.458 4.108 +15.708 5.635 +15.958 4.907 +16.208 2.644 +16.458 2.634 +16.708 2.261 +16.958 2.909 diff --git a/app/host/SBI/snrfiles/WISE_W4_magvsnr.txt b/app/host/SBI/snrfiles/WISE_W4_magvsnr.txt new file mode 100644 index 00000000..8244284d --- /dev/null +++ b/app/host/SBI/snrfiles/WISE_W4_magvsnr.txt @@ -0,0 +1,44 @@ +5.363 150.200 +5.613 164.116 +5.863 178.032 +6.113 111.280 +6.363 116.695 +6.613 85.341 +6.863 90.811 +7.113 99.952 +7.363 37.822 +7.613 44.851 +7.863 25.629 +8.113 26.360 +8.363 23.568 +8.613 25.448 +8.863 26.994 +9.113 22.456 +9.363 28.704 +9.613 22.412 +9.863 13.099 +10.113 14.707 +10.363 15.735 +10.613 12.176 +10.863 8.584 +11.113 10.425 +11.363 7.485 +11.613 7.685 +11.863 6.615 +12.113 5.545 +12.363 5.060 +12.613 4.973 +12.863 4.023 +13.113 3.073 +13.363 2.698 +13.613 2.126 +13.863 4.472 +14.113 1.762 +14.363 1.467 +14.613 1.173 +14.863 1.677 +15.113 1.330 +15.363 1.101 +15.613 1.011 +15.863 0.920 +16.113 0.830 diff --git a/app/host/SBI/train_sbi.py b/app/host/SBI/train_sbi.py new file mode 100644 index 00000000..099ae6cc --- /dev/null +++ b/app/host/SBI/train_sbi.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python +# D. Jones - 5/26/23 +# modified by B. Wang - 6/7/23 +"""Implementation of SBI++ training for Blast""" +import math +import os +import pickle + +import h5py +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from astropy.cosmology import WMAP9 as cosmo +from django.db.models import Q +from django_cron import CronJobBase +from django_cron import Schedule +from host.models import * +from host.photometric_calibration import ab_mag_to_mJy +from host.photometric_calibration import mJy_to_maggies +from host.prospector import build_model +from host.prospector import build_obs +from prospect.fitting import fit_model as fit_model_prospect +from prospect.io import write_results as writer +from prospect.models import priors +from prospect.models import SpecModel +from prospect.models.sedmodel import PolySpecModel +from prospect.models.templates import TemplateLibrary +from prospect.sources import CSPSpecBasis +from prospect.sources import FastStepBasis +from prospect.utils.obsutils import fix_obs +from sbi import inference as Inference +from sbi import utils as Ut + +_outfile = os.environ.get("OUTFILE") + +# torch + +all_filters = Filter.objects.filter(~Q(name="DES_i") & ~Q(name="DES_Y")) +massmet = np.loadtxt("host/SBI/priors/gallazzi_05_massmet.txt") + +nhidden = 500 # architecture +nblocks = 15 # architecture + +fanpe = "host/SBI/SBI_model.pt" # name for the .pt file where the trained model will be saved +fsumm = "host/SBI/SBI_model_summary.p" # name for the .p file where the training summary will be saved; useful if want to check the convergence, etc. + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + + +def maggies_to_asinh(x): + """asinh magnitudes""" + a = 2.50 * np.log10(np.e) + mu = 35.0 + return -a * math.asinh((x / 2.0) * np.exp(mu / a)) + mu + + +def build_obs(**extras): ##transient, aperture_type): + + """ + This functions is required by prospector and should return + a dictionary defined by + https://prospect.readthedocs.io/en/latest/dataformat.html. + + """ + filters = [] + for filter in all_filters: + + filters.append(filter.transmission_curve()) + + obs_data = dict( + wavelength=None, + spectrum=None, + unc=None, + mask=None, + ##redshift=z, + maggies=np.ones(len(all_filters)), # np.array(flux_maggies), + maggies_unc=np.ones(len(all_filters)), + filters=filters, + ) + obs_data["phot_wave"] = np.array([f.wave_effective for f in obs_data["filters"]]) + obs_data["phot_mask"] = [True] * len(obs_data["filters"]) + + return fix_obs(obs_data) + + +def build_model(obs=None, **extras): + """prospector-alpha""" + fit_order = [ + "zred", + "logmass", + "logzsol", + "logsfr_ratios", + "dust2", + "dust_index", + "dust1_fraction", + "log_fagn", + "log_agn_tau", + "gas_logz", + "duste_qpah", + "duste_umin", + "log_duste_gamma", + ] + + # ------------- + # MODEL_PARAMS + model_params = {} + + # --- BASIC PARAMETERS --- + model_params["zred"] = { + "N": 1, + "isfree": True, + "init": 0.5, + "prior": priors.FastUniform(a=0, b=0.2), + } + + model_params["logmass"] = { + "N": 1, + "isfree": True, + "init": 8.0, + "units": "Msun", + "prior": priors.FastUniform(a=7.0, b=12.5), + } + + model_params["logzsol"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"$\log (Z/Z_\odot)$", + "prior": priors.FastUniform(a=-1.98, b=0.19), + } + + model_params["imf_type"] = { + "N": 1, + "isfree": False, + "init": 1, # 1 = chabrier + "units": None, + "prior": None, + } + model_params["add_igm_absorption"] = {"N": 1, "isfree": False, "init": True} + model_params["add_agb_dust_model"] = {"N": 1, "isfree": False, "init": True} + model_params["pmetals"] = {"N": 1, "isfree": False, "init": -99} + + # --- SFH --- + nbins_sfh = 7 + model_params["sfh"] = {"N": 1, "isfree": False, "init": 3} + model_params["logsfr_ratios"] = { + "N": 6, + "isfree": True, + "init": 0.0, + "prior": priors.FastTruncatedEvenStudentTFreeDeg2( + hw=np.ones(6) * 5.0, sig=np.ones(6) * 0.3 + ), + } + + # add redshift scaling to agebins, such that + # t_max = t_univ + def zred_to_agebins(zred=None, **extras): + amin = 7.1295 + nbins_sfh = 7 + tuniv = cosmo.age(zred)[0].value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + + def logsfr_ratios_to_masses( + logmass=None, logsfr_ratios=None, agebins=None, **extras + ): + """This converts from an array of log_10(SFR_j / SFR_{j+1}) and a value of + log10(\Sum_i M_i) to values of M_i. j=0 is the most recent bin in lookback + time. + """ + nbins = agebins.shape[0] + sratios = 10 ** np.clip(logsfr_ratios, -100, 100) + dt = 10 ** agebins[:, 1] - 10 ** agebins[:, 0] + coeffs = np.array( + [ + (1.0 / np.prod(sratios[:i])) + * (np.prod(dt[1 : i + 1]) / np.prod(dt[:i])) + for i in range(nbins) + ] + ) + m1 = (10**logmass) / coeffs.sum() + + return m1 * coeffs + + model_params["mass"] = { + "N": 7, + "isfree": False, + "init": 1e6, + "units": r"M$_\odot$", + "depends_on": logsfr_ratios_to_masses, + } + + model_params["agebins"] = { + "N": 7, + "isfree": False, + "init": zred_to_agebins(np.atleast_1d(0.5)), + "prior": None, + "depends_on": zred_to_agebins, + } + + # --- Dust Absorption --- + model_params["dust_type"] = { + "N": 1, + "isfree": False, + "init": 4, + "units": "FSPS index", + } + model_params["dust1_fraction"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.0, b=2.0, mu=1.0, sig=0.3), + } + + model_params["dust2"] = { + "N": 1, + "isfree": True, + "init": 0.0, + "units": "", + "prior": priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0), + } + + def to_dust1(dust1_fraction=None, dust1=None, dust2=None, **extras): + return dust1_fraction * dust2 + + model_params["dust1"] = { + "N": 1, + "isfree": False, + "depends_on": to_dust1, + "init": 0.0, + "units": "optical depth towards young stars", + "prior": None, + } + model_params["dust_index"] = { + "N": 1, + "isfree": True, + "init": 0.7, + "units": "", + "prior": priors.FastUniform(a=-1.0, b=0.2), + } + + # --- Nebular Emission --- + model_params["add_neb_emission"] = {"N": 1, "isfree": False, "init": True} + model_params["add_neb_continuum"] = {"N": 1, "isfree": False, "init": True} + model_params["gas_logz"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"log Z/Z_\odot", + "prior": priors.FastUniform(a=-2.0, b=0.5), + } + model_params["gas_logu"] = { + "N": 1, + "isfree": False, + "init": -1.0, + "units": r"Q_H/N_H", + "prior": priors.FastUniform(a=-4, b=-1), + } + + # --- AGN dust --- + model_params["add_agn_dust"] = {"N": 1, "isfree": False, "init": True} + + model_params["log_fagn"] = { + "N": 1, + "isfree": True, + "init": -7.0e-5, + "prior": priors.FastUniform(a=-5.0, b=-4.9), + } + + def to_fagn(log_fagn=None, **extras): + return 10**log_fagn + + model_params["fagn"] = {"N": 1, "isfree": False, "init": 0, "depends_on": to_fagn} + + model_params["log_agn_tau"] = { + "N": 1, + "isfree": True, + "init": np.log10(20.0), + "prior": priors.FastUniform(a=np.log10(15.0), b=np.log10(15.1)), + } + + def to_agn_tau(log_agn_tau=None, **extras): + return 10**log_agn_tau + + model_params["agn_tau"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_agn_tau, + } + + # --- Dust Emission --- + model_params["duste_qpah"] = { + "N": 1, + "isfree": True, + "init": 2.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=2.0, sig=2.0), + } + + model_params["duste_umin"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=1.0, sig=10.0), + } + + model_params["log_duste_gamma"] = { + "N": 1, + "isfree": True, + "init": -2.0, + "prior": priors.FastTruncatedNormal(a=-2.1, b=-1.9, mu=-2.0, sig=1.0), + } + + def to_duste_gamma(log_duste_gamma=None, **extras): + return 10**log_duste_gamma + + model_params["duste_gamma"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_duste_gamma, + } + + # ---- Units ---- + model_params["peraa"] = {"N": 1, "isfree": False, "init": False} + + model_params["mass_units"] = {"N": 1, "isfree": False, "init": "mformed"} + + tparams = {} + for i in fit_order: + tparams[i] = model_params[i] + for i in list(model_params.keys()): + if i not in fit_order: + tparams[i] = model_params[i] + model_params = tparams + + return PolySpecModel(model_params) + + +def build_sps(zcontinuous=2, compute_vega_mags=False, **extras): + sps = FastStepBasis(zcontinuous=zcontinuous, compute_vega_mags=compute_vega_mags) + return sps + + +def build_noise(**extras): + return None, None + + +def scale(mass): + upper_84 = np.interp(mass, massmet[:, 0], massmet[:, 3]) + lower_16 = np.interp(mass, massmet[:, 0], massmet[:, 2]) + return upper_84 - lower_16 + + +def loc(mass): + return np.interp(mass, massmet[:, 0], massmet[:, 1]) + + +def draw_thetas(flat=False): + + if flat: + zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample() + logmass = priors.FastUniform(a=7.0, b=12.5).sample() + logzsol = priors.FastUniform(a=-1.98, b=0.19).sample() + + logsfrratio_0 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_1 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_2 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_3 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_4 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_5 = priors.FastUniform(a=-5.0, b=5.0).sample() + + dust2 = priors.FastUniform(a=0.0, b=4.0).sample() + dust_index = priors.FastUniform(a=-1.0, b=0.4).sample() + dust1_fraction = priors.FastUniform(a=0.0, b=2.0).sample() + log_fagn = priors.FastUniform(a=-5.0, b=np.log10(3.0)).sample() + log_agn_tau = priors.FastUniform(a=np.log10(5.0), b=np.log10(150.0)).sample() + gas_logz = priors.FastUniform(a=-2.0, b=0.5).sample() + duste_qpah = priors.FastUniform(a=0.0, b=7.0).sample() + duste_umin = priors.FastUniform(a=0.1, b=25.0).sample() + log_duste_gamma = priors.FastUniform(a=-4.0, b=0.0).sample() + + else: + zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample() + logmass = priors.FastUniform(a=7.0, b=12.5).sample() + logzsol = priors.FastTruncatedNormal( + a=-1.98, b=0.19, mu=loc(logmass), sig=scale(logmass) + ).sample() + + logsfrratio_0 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_1 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_2 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_3 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_4 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_5 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + + dust2 = priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0).sample() + dust_index = priors.FastUniform(a=-1.0, b=0.4).sample() + dust1_fraction = priors.FastTruncatedNormal( + a=0.0, b=2.0, mu=1.0, sig=0.3 + ).sample() + log_fagn = priors.FastUniform(a=-5.0, b=np.log10(3.0)).sample() + log_agn_tau = priors.FastUniform(a=np.log10(5.0), b=np.log10(150.0)).sample() + gas_logz = priors.FastUniform(a=-2.0, b=0.5).sample() + duste_qpah = priors.FastTruncatedNormal(a=0.0, b=7.0, mu=2.0, sig=2.0).sample() + duste_umin = priors.FastTruncatedNormal( + a=0.1, b=25.0, mu=1.0, sig=10.0 + ).sample() + log_duste_gamma = priors.FastTruncatedNormal( + a=-4.0, b=0.0, mu=-2.0, sig=1.0 + ).sample() + + return np.array( + [ + zred, + logmass, + logzsol, + logsfrratio_0, + logsfrratio_1, + logsfrratio_2, + logsfrratio_3, + logsfrratio_4, + logsfrratio_5, + dust2, + dust_index, + dust1_fraction, + log_fagn, + log_agn_tau, + gas_logz, + duste_qpah, + duste_umin, + log_duste_gamma, + ] + ) + + +class TrainSBI(CronJobBase): + + RUN_EVERY_MINS = 3 + + schedule = Schedule(run_every_mins=RUN_EVERY_MINS) + code = "host.SBI.train_SBI.TrainSBI" + + def build_all(self, **kwargs): + return ( + build_obs(**kwargs), + build_model(**kwargs), + build_sps(**kwargs), + build_noise(**kwargs), + ) + + def do(self, do_phot=True, do_train=False): + + # parameters + needed_size = 150000 + run_params = {"ichunk": 0, "needed_size": needed_size} + run_params["add_duste"] = True + run_params["add_igm"] = True + run_params["add_neb"] = True + run_params["dynesty"] = False + run_params["optmization"] = False + run_params["emcee"] = False + + if do_phot: + obs, model, sps, noise = self.build_all(**run_params) + + run_params["sps_libraries"] = sps.ssp.libraries + run_params["param_file"] = __file__ + + # get the minimum, maximum magnitudes + cat_min, cat_max, cat_full = {}, {}, {} + for f in all_filters: + mag, snr = np.loadtxt( + f"host/SBI/snrfiles/{f.name}_magvsnr.txt", unpack=True + ) + + cat_min[f.name] = np.min(mag) + cat_max[f.name] = np.max(mag) + cat_full[f.name] = (mag, snr) + + ### start putting together the synthetic data + list_thetas = [] + list_mfrac = [] + list_phot = [] + while len(list_phot) < needed_size: + if not len(list_phot) % 3: + theta = draw_thetas(flat=True) + else: + theta = draw_thetas(flat=False) + + # call prospector + # generate the model SED at given theta + spec, phot, mfrac = model.predict(theta, obs=obs, sps=sps) + + predicted_mags = -2.5 * np.log10(phot) + theta[1] += np.log10(mfrac) + + flag = True + for i, f in enumerate(all_filters): + # probably gonna have some unit issues here + # expand the magnitude range a bit + flag &= (predicted_mags[i] >= cat_min[f.name] - 1) & ( + predicted_mags[i] <= cat_max[f.name] + 2 + ) + + # if all phot is within valid range, we can proceed + if not flag: + continue + + list_thetas.append(theta) + list_mfrac.append(mfrac) + + # simulate the noised-up photometry + list_phot_single = np.array([]) + list_phot_errs_single = np.array([]) + for i, f in enumerate(all_filters): + snr = np.interp( + predicted_mags[i], cat_full[f.name][0], cat_full[f.name][1] + ) + phot_err = phot[i] / snr + phot_random = np.random.normal(phot[i], phot_err) + phot_random_mags = maggies_to_asinh(phot_random) + phot_err_mags = 2.5 / np.log(10) * phot_err / phot[i] + + list_phot_single = np.append(list_phot_single, [phot_random_mags]) + list_phot_errs_single = np.append( + list_phot_errs_single, [phot_err_mags] + ) + # adding in the redshift here + list_phot.append( + np.concatenate( + (list_phot_single, list_phot_errs_single, [theta[0]]), + ) + ) + print(len(list_phot)) + + save_phot = True + if save_phot: + hf_phot = h5py.File(_outfile, "w") + hf_phot.create_dataset("wphot", data=obs["phot_wave"]) + hf_phot.create_dataset("phot", data=list_phot) + hf_phot.create_dataset("mfrac", data=list_mfrac) + hf_phot.create_dataset("theta", data=list_thetas) + + try: + hf_phot.close() + except (AttributeError): + pass + + if do_train: + data = h5py.File("host/SBI/sbi_phot_1.h5", "r") + list_thetas, list_phot, list_mfrac, list_phot_wave = ( + np.array(data["theta"]), + np.array(data["phot"]), + np.array(data["mfrac"]), + np.array(data["wphot"]), + ) + for i in range(2, 11): + data2 = h5py.File(f"host/SBI/sbi_phot_{i}.h5", "r") + list_thetas = np.append(list_thetas, np.array(data2["theta"]), axis=0) + list_phot = np.append(list_phot, np.array(data2["phot"]), axis=0) + list_mfrac = np.append(list_mfrac, np.array(data2["mfrac"]), axis=0) + list_phot_wave = np.append( + list_phot_wave, np.array(data2["wphot"]), axis=0 + ) + hf_phot = h5py.File("host/SBI/sbi_phot.h5", "w") + hf_phot.create_dataset("wphot", data=list_phot_wave) + hf_phot.create_dataset("phot", data=list_phot) + hf_phot.create_dataset("mfrac", data=list_mfrac) + hf_phot.create_dataset("theta", data=list_thetas) + hf_phot.close() + + x_train = np.array(list_thetas) + y_train = np.array(list_phot) + # now do the training + anpe = Inference.SNPE( + density_estimator=Ut.posterior_nn( + "maf", hidden_features=nhidden, num_transforms=nblocks + ), + device=device, + ) + # because we append_simulations, training set == prior + anpe.append_simulations( + torch.as_tensor(x_train.astype(np.float32), device="cpu"), + torch.as_tensor(y_train.astype(np.float32), device="cpu"), + ) + p_x_y_estimator = anpe.train() + + # save trained ANPE + torch.save(p_x_y_estimator.state_dict(), fanpe) + + # save training summary + pickle.dump(anpe._summary, open(fsumm, "wb")) + print(anpe._summary) + + print("Finished.") + + +if __name__ == "__main__": + ts = TrainSBI() + ts.main() diff --git a/app/host/SBI/train_sbi_local.py b/app/host/SBI/train_sbi_local.py new file mode 100644 index 00000000..9a8204fb --- /dev/null +++ b/app/host/SBI/train_sbi_local.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python +# D. Jones - 5/26/23 +# modified by B. Wang - 6/7/23 +"""Implementation of SBI++ training for Blast""" +import math +import os +import pickle + +import h5py +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from astropy.cosmology import WMAP9 as cosmo +from django.db.models import Q +from django_cron import CronJobBase +from django_cron import Schedule +from host.models import * +from host.photometric_calibration import ab_mag_to_mJy +from host.photometric_calibration import mJy_to_maggies +from host.prospector import build_model +from host.prospector import build_obs +from prospect.fitting import fit_model as fit_model_prospect +from prospect.io import write_results as writer +from prospect.models import priors +from prospect.models import SpecModel +from prospect.models.sedmodel import PolySpecModel +from prospect.models.templates import TemplateLibrary +from prospect.sources import CSPSpecBasis +from prospect.sources import FastStepBasis +from prospect.utils.obsutils import fix_obs +from sbi import inference as Inference +from sbi import utils as Ut + +_outfile = os.environ.get("OUTFILE") + +# torch + +all_filters = Filter.objects.filter(~Q(name="DES_i") & ~Q(name="DES_Y")) +massmet = np.loadtxt("host/SBI/priors/gallazzi_05_massmet.txt") + +nhidden = 500 # architecture +nblocks = 15 # architecture + +fanpe = "host/SBI/SBI_model.pt" # name for the .pt file where the trained model will be saved +fsumm = "host/SBI/SBI_model_summary.p" # name for the .p file where the training summary will be saved; useful if want to check the convergence, etc. + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + + +def maggies_to_asinh(x): + """asinh magnitudes""" + a = 2.50 * np.log10(np.e) + mu = 35.0 + return -a * math.asinh((x / 2.0) * np.exp(mu / a)) + mu + + +def build_obs(**extras): ##transient, aperture_type): + + """ + This functions is required by prospector and should return + a dictionary defined by + https://prospect.readthedocs.io/en/latest/dataformat.html. + + """ + filters = [] + for filter in all_filters: + + filters.append(filter.transmission_curve()) + + obs_data = dict( + wavelength=None, + spectrum=None, + unc=None, + mask=None, + ##redshift=z, + maggies=np.ones(len(all_filters)), # np.array(flux_maggies), + maggies_unc=np.ones(len(all_filters)), + filters=filters, + ) + obs_data["phot_wave"] = np.array([f.wave_effective for f in obs_data["filters"]]) + obs_data["phot_mask"] = [True] * len(obs_data["filters"]) + + return fix_obs(obs_data) + + +def build_model(obs=None, **extras): + """prospector-alpha""" + fit_order = [ + "zred", + "logmass", + "logzsol", + "logsfr_ratios", + "dust2", + "dust_index", + "dust1_fraction", + "log_fagn", + "log_agn_tau", + "gas_logz", + "duste_qpah", + "duste_umin", + "log_duste_gamma", + ] + + # ------------- + # MODEL_PARAMS + model_params = {} + + # --- BASIC PARAMETERS --- + model_params["zred"] = { + "N": 1, + "isfree": True, + "init": 0.5, + "prior": priors.FastUniform(a=0, b=0.2), + } + + model_params["logmass"] = { + "N": 1, + "isfree": True, + "init": 8.0, + "units": "Msun", + "prior": priors.FastUniform(a=7.0, b=12.5), + } + + model_params["logzsol"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"$\log (Z/Z_\odot)$", + "prior": priors.FastUniform(a=-1.98, b=0.19), + } + + model_params["imf_type"] = { + "N": 1, + "isfree": False, + "init": 1, # 1 = chabrier + "units": None, + "prior": None, + } + model_params["add_igm_absorption"] = {"N": 1, "isfree": False, "init": True} + model_params["add_agb_dust_model"] = {"N": 1, "isfree": False, "init": True} + model_params["pmetals"] = {"N": 1, "isfree": False, "init": -99} + + # --- SFH --- + nbins_sfh = 7 + model_params["sfh"] = {"N": 1, "isfree": False, "init": 3} + model_params["logsfr_ratios"] = { + "N": 6, + "isfree": True, + "init": 0.0, + "prior": priors.FastTruncatedEvenStudentTFreeDeg2( + hw=np.ones(6) * 5.0, sig=np.ones(6) * 0.3 + ), + } + + # add redshift scaling to agebins, such that + # t_max = t_univ + def zred_to_agebins(zred=None, **extras): + amin = 7.1295 + nbins_sfh = 7 + tuniv = cosmo.age(zred)[0].value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + + def logsfr_ratios_to_masses( + logmass=None, logsfr_ratios=None, agebins=None, **extras + ): + """This converts from an array of log_10(SFR_j / SFR_{j+1}) and a value of + log10(\Sum_i M_i) to values of M_i. j=0 is the most recent bin in lookback + time. + """ + nbins = agebins.shape[0] + sratios = 10 ** np.clip(logsfr_ratios, -100, 100) + dt = 10 ** agebins[:, 1] - 10 ** agebins[:, 0] + coeffs = np.array( + [ + (1.0 / np.prod(sratios[:i])) + * (np.prod(dt[1 : i + 1]) / np.prod(dt[:i])) + for i in range(nbins) + ] + ) + m1 = (10**logmass) / coeffs.sum() + + return m1 * coeffs + + model_params["mass"] = { + "N": 7, + "isfree": False, + "init": 1e6, + "units": r"M$_\odot$", + "depends_on": logsfr_ratios_to_masses, + } + + model_params["agebins"] = { + "N": 7, + "isfree": False, + "init": zred_to_agebins(np.atleast_1d(0.5)), + "prior": None, + "depends_on": zred_to_agebins, + } + + # --- Dust Absorption --- + model_params["dust_type"] = { + "N": 1, + "isfree": False, + "init": 4, + "units": "FSPS index", + } + model_params["dust1_fraction"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.0, b=2.0, mu=1.0, sig=0.3), + } + + model_params["dust2"] = { + "N": 1, + "isfree": True, + "init": 0.0, + "units": "", + "prior": priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0), + } + + def to_dust1(dust1_fraction=None, dust1=None, dust2=None, **extras): + return dust1_fraction * dust2 + + model_params["dust1"] = { + "N": 1, + "isfree": False, + "depends_on": to_dust1, + "init": 0.0, + "units": "optical depth towards young stars", + "prior": None, + } + model_params["dust_index"] = { + "N": 1, + "isfree": True, + "init": 0.7, + "units": "", + "prior": priors.FastUniform(a=-1.0, b=0.2), + } + + # --- Nebular Emission --- + model_params["add_neb_emission"] = {"N": 1, "isfree": False, "init": True} + model_params["add_neb_continuum"] = {"N": 1, "isfree": False, "init": True} + model_params["gas_logz"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"log Z/Z_\odot", + "prior": priors.FastUniform(a=-2.0, b=0.5), + } + model_params["gas_logu"] = { + "N": 1, + "isfree": False, + "init": -1.0, + "units": r"Q_H/N_H", + "prior": priors.FastUniform(a=-4, b=-1), + } + + # --- AGN dust --- + model_params["add_agn_dust"] = {"N": 1, "isfree": False, "init": True} + + model_params["log_fagn"] = { + "N": 1, + "isfree": True, + "init": -7.0e-5, + "prior": priors.FastUniform(a=-5.0, b=-4.9), + } + + def to_fagn(log_fagn=None, **extras): + return 10**log_fagn + + model_params["fagn"] = {"N": 1, "isfree": False, "init": 0, "depends_on": to_fagn} + + model_params["log_agn_tau"] = { + "N": 1, + "isfree": True, + "init": np.log10(20.0), + "prior": priors.FastUniform(a=np.log10(15.0), b=np.log10(15.1)), + } + + def to_agn_tau(log_agn_tau=None, **extras): + return 10**log_agn_tau + + model_params["agn_tau"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_agn_tau, + } + + # --- Dust Emission --- + model_params["duste_qpah"] = { + "N": 1, + "isfree": True, + "init": 2.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=2.0, sig=2.0), + } + + model_params["duste_umin"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=1.0, sig=10.0), + } + + model_params["log_duste_gamma"] = { + "N": 1, + "isfree": True, + "init": -2.0, + "prior": priors.FastTruncatedNormal(a=-2.1, b=-1.9, mu=-2.0, sig=1.0), + } + + def to_duste_gamma(log_duste_gamma=None, **extras): + return 10**log_duste_gamma + + model_params["duste_gamma"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_duste_gamma, + } + + # ---- Units ---- + model_params["peraa"] = {"N": 1, "isfree": False, "init": False} + + model_params["mass_units"] = {"N": 1, "isfree": False, "init": "mformed"} + + tparams = {} + for i in fit_order: + tparams[i] = model_params[i] + for i in list(model_params.keys()): + if i not in fit_order: + tparams[i] = model_params[i] + model_params = tparams + + return PolySpecModel(model_params) + + +def build_sps(zcontinuous=2, compute_vega_mags=False, **extras): + sps = FastStepBasis(zcontinuous=zcontinuous, compute_vega_mags=compute_vega_mags) + return sps + + +def build_noise(**extras): + return None, None + + +def scale(mass): + upper_84 = np.interp(mass, massmet[:, 0], massmet[:, 3]) + lower_16 = np.interp(mass, massmet[:, 0], massmet[:, 2]) + return upper_84 - lower_16 + + +def loc(mass): + return np.interp(mass, massmet[:, 0], massmet[:, 1]) + + +def draw_thetas(flat=False): + + if flat: + zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample() + ### go 1.5 dex less for local + logmass = priors.FastUniform(a=5.5, b=11.0).sample() + logzsol = priors.FastUniform(a=-1.98, b=0.19).sample() + + logsfrratio_0 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_1 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_2 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_3 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_4 = priors.FastUniform(a=-5.0, b=5.0).sample() + logsfrratio_5 = priors.FastUniform(a=-5.0, b=5.0).sample() + + dust2 = priors.FastUniform(a=0.0, b=4.0).sample() + dust_index = priors.FastUniform(a=-1.0, b=0.4).sample() + dust1_fraction = priors.FastUniform(a=0.0, b=2.0).sample() + log_fagn = priors.FastUniform(a=-5.0, b=np.log10(3.0)).sample() + log_agn_tau = priors.FastUniform(a=np.log10(5.0), b=np.log10(150.0)).sample() + gas_logz = priors.FastUniform(a=-2.0, b=0.5).sample() + duste_qpah = priors.FastUniform(a=0.0, b=7.0).sample() + duste_umin = priors.FastUniform(a=0.1, b=25.0).sample() + log_duste_gamma = priors.FastUniform(a=-4.0, b=0.0).sample() + + else: + zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample() + logmass = priors.FastUniform(a=5.5, b=11.0).sample() + logzsol = priors.FastTruncatedNormal( + a=-1.98, b=0.19, mu=loc(logmass), sig=scale(logmass) + ).sample() + + logsfrratio_0 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_1 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_2 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_3 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_4 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + logsfrratio_5 = priors.FastTruncatedEvenStudentTFreeDeg2( + hw=5.0, sig=0.3 + ).sample() + + dust2 = priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0).sample() + dust_index = priors.FastUniform(a=-1.0, b=0.4).sample() + dust1_fraction = priors.FastTruncatedNormal( + a=0.0, b=2.0, mu=1.0, sig=0.3 + ).sample() + log_fagn = priors.FastUniform(a=-5.0, b=np.log10(3.0)).sample() + log_agn_tau = priors.FastUniform(a=np.log10(5.0), b=np.log10(150.0)).sample() + gas_logz = priors.FastUniform(a=-2.0, b=0.5).sample() + duste_qpah = priors.FastTruncatedNormal(a=0.0, b=7.0, mu=2.0, sig=2.0).sample() + duste_umin = priors.FastTruncatedNormal( + a=0.1, b=25.0, mu=1.0, sig=10.0 + ).sample() + log_duste_gamma = priors.FastTruncatedNormal( + a=-4.0, b=0.0, mu=-2.0, sig=1.0 + ).sample() + + return np.array( + [ + zred, + logmass, + logzsol, + logsfrratio_0, + logsfrratio_1, + logsfrratio_2, + logsfrratio_3, + logsfrratio_4, + logsfrratio_5, + dust2, + dust_index, + dust1_fraction, + log_fagn, + log_agn_tau, + gas_logz, + duste_qpah, + duste_umin, + log_duste_gamma, + ] + ) + + +class TrainSBI(CronJobBase): + + RUN_EVERY_MINS = 3 + + schedule = Schedule(run_every_mins=RUN_EVERY_MINS) + code = "host.SBI.train_SBI.TrainSBI" + + def build_all(self, **kwargs): + return ( + build_obs(**kwargs), + build_model(**kwargs), + build_sps(**kwargs), + build_noise(**kwargs), + ) + + def do(self, do_phot=True, do_train=False): + + # parameters + needed_size = 150000 + run_params = {"ichunk": 0, "needed_size": needed_size} + run_params["add_duste"] = True + run_params["add_igm"] = True + run_params["add_neb"] = True + run_params["dynesty"] = False + run_params["optmization"] = False + run_params["emcee"] = False + + if do_phot: + obs, model, sps, noise = self.build_all(**run_params) + + run_params["sps_libraries"] = sps.ssp.libraries + run_params["param_file"] = __file__ + + # get the minimum, maximum magnitudes + cat_min, cat_max, cat_full = {}, {}, {} + for f in all_filters: + mag, snr = np.loadtxt( + f"host/SBI/snrfiles/{f.name}_magvsnr.txt", unpack=True + ) + + cat_min[f.name] = np.min(mag) + cat_max[f.name] = np.max(mag) + cat_full[f.name] = (mag, snr) + + ### start putting together the synthetic data + list_thetas = [] + list_mfrac = [] + list_phot = [] + while len(list_phot) < needed_size: + if not len(list_phot) % 3: + theta = draw_thetas(flat=True) + else: + theta = draw_thetas(flat=False) + + # call prospector + # generate the model SED at given theta + spec, phot, mfrac = model.predict(theta, obs=obs, sps=sps) + predicted_mags = -2.5 * np.log10(phot) + theta[1] += np.log10(mfrac) + + flag = True + for i, f in enumerate(all_filters): + # probably gonna have some unit issues here + # expand the magnitude range a bit + flag &= (predicted_mags[i] >= cat_min[f.name] - 1) & ( + predicted_mags[i] <= cat_max[f.name] + 2 + ) + + # if all phot is within valid range, we can proceed + if not flag: + continue + + list_thetas.append(theta) + list_mfrac.append(mfrac) + + # simulate the noised-up photometry + list_phot_single = np.array([]) + list_phot_errs_single = np.array([]) + for i, f in enumerate(all_filters): + snr = np.interp( + predicted_mags[i], cat_full[f.name][0], cat_full[f.name][1] + ) + phot_err = phot[i] / snr + phot_random = np.random.normal(phot[i], phot_err) + phot_random_mags = maggies_to_asinh(phot_random) + phot_err_mags = 2.5 / np.log(10) * phot_err / phot[i] + + list_phot_single = np.append(list_phot_single, [phot_random_mags]) + list_phot_errs_single = np.append( + list_phot_errs_single, [phot_err_mags] + ) + # adding in the redshift here + list_phot.append( + np.concatenate( + (list_phot_single, list_phot_errs_single, [theta[0]]), + ) + ) + print(len(list_phot)) + + save_phot = True + if save_phot: + hf_phot = h5py.File(_outfile, "w") + hf_phot.create_dataset("wphot", data=obs["phot_wave"]) + hf_phot.create_dataset("phot", data=list_phot) + hf_phot.create_dataset("mfrac", data=list_mfrac) + hf_phot.create_dataset("theta", data=list_thetas) + + try: + hf_phot.close() + except (AttributeError): + pass + + if do_train: + data = h5py.File("host/SBI/sbi_local_phot_1.h5", "r") + list_thetas, list_phot, list_mfrac, list_phot_wave = ( + np.array(data["theta"]), + np.array(data["phot"]), + np.array(data["mfrac"]), + np.array(data["wphot"]), + ) + for i in range(2, 11): + data2 = h5py.File(f"host/SBI/sbi_local_phot_{i}.h5", "r") + list_thetas = np.append(list_thetas, np.array(data2["theta"]), axis=0) + list_phot = np.append(list_phot, np.array(data2["phot"]), axis=0) + list_mfrac = np.append(list_mfrac, np.array(data2["mfrac"]), axis=0) + list_phot_wave = np.append( + list_phot_wave, np.array(data2["wphot"]), axis=0 + ) + hf_phot = h5py.File("host/SBI/sbi_local_phot.h5", "w") + hf_phot.create_dataset("wphot", data=list_phot_wave) + hf_phot.create_dataset("phot", data=list_phot) + hf_phot.create_dataset("mfrac", data=list_mfrac) + hf_phot.create_dataset("theta", data=list_thetas) + hf_phot.close() + + x_train = np.array(list_thetas) + y_train = np.array(list_phot) + # now do the training + anpe = Inference.SNPE( + density_estimator=Ut.posterior_nn( + "maf", hidden_features=nhidden, num_transforms=nblocks + ), + device=device, + ) + # because we append_simulations, training set == prior + anpe.append_simulations( + torch.as_tensor(x_train.astype(np.float32), device="cpu"), + torch.as_tensor(y_train.astype(np.float32), device="cpu"), + ) + p_x_y_estimator = anpe.train() + + # save trained ANPE + torch.save(p_x_y_estimator.state_dict(), fanpe) + + # save training summary + pickle.dump(anpe._summary, open(fsumm, "wb")) + print(anpe._summary) + + print("Finished.") + + +if __name__ == "__main__": + ts = TrainSBI() + ts.main() diff --git a/app/host/base_tasks.py b/app/host/base_tasks.py index fee31575..c98218ac 100644 --- a/app/host/base_tasks.py +++ b/app/host/base_tasks.py @@ -3,6 +3,7 @@ from abc import abstractproperty from time import process_time +from billiard.exceptions import SoftTimeLimitExceeded from django.utils import timezone from .models import Status @@ -186,13 +187,14 @@ def select_register_item(self): register = self.find_register_items_meeting_prerequisites() return self._select_highest_priority(register) if register.exists() else None - def run_process(self): + def run_process(self, task_register_item=None): """ Runs task runner process. """ # self.task = Task.objects.get(name__exact=self.task_name) - task_register_item = self.select_register_item() + if task_register_item is None: + task_register_item = self.select_register_item() processing_status = Status.objects.get(message__exact="processing") if task_register_item is not None: @@ -202,6 +204,9 @@ def run_process(self): start_time = process_time() try: status_message = self._run_process(transient) + except SoftTimeLimitExceeded: + status_message = "time limit exceeded" + raise except: status_message = self._failed_status_message() raise @@ -304,5 +309,11 @@ def initialise_all_tasks_status(transient): not_processed = Status.objects.get(message__exact="not processed") for task in tasks: - task_status = TaskRegister(task=task, transient=transient) - update_status(task_status, not_processed) + task_status = TaskRegister.objects.filter(task=task, transient=transient) + if not len(task_status): + task_status = TaskRegister(task=task, transient=transient) + ### if the task already exists, let's not change it + ### because bad things seem to happen.... + update_status(task_status, not_processed) + else: + task_status = task_status[0] diff --git a/app/host/cutouts.py b/app/host/cutouts.py index d164dbdb..920ba853 100644 --- a/app/host/cutouts.py +++ b/app/host/cutouts.py @@ -1,8 +1,11 @@ import os import re +import time +from io import BytesIO import astropy.table as at import astropy.units as u +import astropy.utils.data import numpy as np import pandas as pd import requests @@ -24,11 +27,50 @@ from .models import Cutout from .models import Filter +# from host import SkyServer + + +def getRADecBox(ra, dec, size): + RAboxsize = DECboxsize = size + + # get the maximum 1.0/cos(DEC) term: used for RA cut + minDec = dec - 0.5 * DECboxsize + if minDec <= -90.0: + minDec = -89.9 + maxDec = dec + 0.5 * DECboxsize + if maxDec >= 90.0: + maxDec = 89.9 + + invcosdec = max( + 1.0 / np.cos(dec * np.pi / 180.0), + 1.0 / np.cos(minDec * np.pi / 180.0), + 1.0 / np.cos(maxDec * np.pi / 180.0), + ) + + ramin = ra - 0.5 * RAboxsize * invcosdec + ramax = ra + 0.5 * RAboxsize * invcosdec + decmin = dec - 0.5 * DECboxsize + decmax = dec + 0.5 * DECboxsize + + if ra < 0.0: + ra += 360.0 + if ra >= 360.0: + ra -= 360.0 + + if ramin != None: + if (ra - ramin) < -180: + ramin -= 360.0 + ramax -= 360.0 + elif (ra - ramin) > 180: + ramin += 360.0 + ramax += 360.0 + return (ramin, ramax, decmin, decmax) + def download_and_save_cutouts( transient, fov=Quantity(0.1, unit="deg"), - media_root=settings.MEDIA_ROOT, + media_root=settings.CUTOUT_ROOT, overwrite=settings.CUTOUT_OVERWRITE, ): """ @@ -56,24 +98,30 @@ def download_and_save_cutouts( path_to_fits = save_dir + f"{filter.name}.fits" file_exists = os.path.exists(path_to_fits) - if file_exists and overwrite == "False": - cutout_name = f"{transient.name}_{filter.name}" - cutout_object = Cutout(name=cutout_name, filter=filter, transient=transient) - cutout_object.fits.name = path_to_fits - cutout_object.save() - else: + fits = None + if not file_exists or not overwrite == "False": fits = cutout(transient.sky_coord, filter, fov=fov) if fits: save_dir = f"{media_root}/{transient.name}/{filter.survey.name}/" os.makedirs(save_dir, exist_ok=True) path_to_fits = save_dir + f"{filter.name}.fits" fits.writeto(path_to_fits, overwrite=True) - cutout_name = f"{transient.name}_{filter.name}" + + if file_exists or fits: + cutout_name = f"{transient.name}_{filter.name}" + cutout_object = Cutout.objects.filter( + name=cutout_name, filter=filter, transient=transient + ) + + if not cutout_object.exists(): cutout_object = Cutout( name=cutout_name, filter=filter, transient=transient ) - cutout_object.fits.name = path_to_fits - cutout_object.save() + else: + cutout_object = cutout_object[0] + + cutout_object.fits.name = path_to_fits + cutout_object.save() def panstarrs_image_filename(position, image_size=None, filter=None): @@ -96,7 +144,11 @@ def panstarrs_image_filename(position, image_size=None, filter=None): f"&size={image_size}&format=fits&filters={filter}" ) - filename_table = pd.read_csv(url, delim_whitespace=True)["filename"] + ### was having SSL errors with pandas, so let's run it through requests + ### optionally, can edit to do this in an unsafe way + r = requests.get(url, stream=True) + r.raw.decode_content = True + filename_table = pd.read_csv(r.raw, sep="\s+")["filename"] return filename_table[0] if len(filename_table) > 0 else None @@ -146,6 +198,7 @@ def panstarrs_cutout(position, image_size=None, filter=None): ------- :cutout : :class:`~astropy.io.fits.HDUList` or None """ + filename = panstarrs_image_filename(position, image_size=image_size, filter=filter) if filename is not None: service = "https://ps1images.stsci.edu/cgi-bin/fitscut.cgi?" @@ -153,7 +206,13 @@ def panstarrs_cutout(position, image_size=None, filter=None): f"{service}ra={position.ra.degree}&dec={position.dec.degree}" f"&size={image_size}&format=fits&red={filename}" ) - fits_image = fits.open(fits_url) + try: + r = requests.get(fits_url, stream=True) + except Exception as e: + time.sleep(5) + r = requests.get(fits_url, stream=True) + fits_image = fits.open(BytesIO(r.content)) + else: fits_image = None @@ -181,6 +240,12 @@ def galex_cutout(position, image_size=None, filter=None): & (obs["filters"] == filter) & (obs["distance"] == 0) ] + + # avoid masked regions + center = SkyCoord(obs["s_ra"], obs["s_dec"], unit=u.deg) + sep = position.separation(center).deg + obs = obs[sep < 0.55] + if len(obs) > 1: obs = obs[obs["t_exptime"] == max(obs["t_exptime"])] @@ -190,8 +255,11 @@ def galex_cutout(position, image_size=None, filter=None): fits_image = fits.open( obs["dataURL"][0] .replace("-exp.fits.gz", "-int.fits.gz") + .replace("-gsp.fits.gz", "-int.fits.gz") .replace("-rr.fits.gz", "-int.fits.gz") - .replace("-xd-mcat.fits.gz", f"-{filter[0].lower()}d-int.fits.gz") + .replace("-cnt.fits.gz", "-int.fits.gz") + .replace("-xd-mcat.fits.gz", f"-{filter[0].lower()}d-int.fits.gz"), + cache=None, ) wcs = WCS(fits_image[0].header) @@ -235,11 +303,24 @@ def WISE_cutout(position, image_size=None, filter=None): if t.startswith("https"): url = t[:] break - data = at.Table.read(r.text, format="ascii.csv") + + # remove the AWS crap messing up the CSV format + line_out = "" + for line in r.text.split("\n"): + try: + idx1 = line.index("{") + except ValueError: + line_out += line[:] + "\n" + continue + idx2 = line.index("}") + newline = line[0 : idx1 + 1] + line[idx2:] + "\n" + line_out += newline + + data = at.Table.read(line_out, format="ascii.csv") exptime = data["t_exptime"][0] if url is not None: - fits_image = fits.open(url) + fits_image = fits.open(url, cache=None) wcs = WCS(fits_image[0].header) cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs) @@ -284,13 +365,24 @@ def DES_cutout(position, image_size=None, filter=None): if len(valid_urls): # we need both the depth and the image - - fits_image = fits.open(valid_urls[0].replace("-depth-", "-image-")) + time.sleep(1) + try: + fits_image = fits.open( + valid_urls[0].replace("-depth-", "-image-"), cache=None + ) + except Exception as e: + ### found some bad links... + return None if np.shape(fits_image[0].data)[0] == 1 or np.shape(fits_image[0].data)[1] == 1: # no idea what's happening here but this is a mess return None - depth_image = fits.open(valid_urls[0]) + try: + depth_image = fits.open(valid_urls[0]) + except Exception as e: + # wonder if there's some issue with other tasks clearing the cache + time.sleep(5) + depth_image = fits.open(valid_urls[0]) wcs_depth = WCS(depth_image[0].header) xc, yc = wcs_depth.wcs_world2pix(position.ra.deg, position.dec.deg, 0) @@ -300,13 +392,14 @@ def DES_cutout(position, image_size=None, filter=None): exptime = depth_image[0].data[0][0] else: exptime = depth_image[0].data[int(yc), int(xc)] - - wcs = WCS(fits_image[0].header) - cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs) - fits_image[0].data = cutout.data - fits_image[0].header.update(cutout.wcs.to_header()) - fits_image[0].header["EXPTIME"] = exptime - + if exptime == 0: + fits_image = None + else: + wcs = WCS(fits_image[0].header) + cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs) + fits_image[0].data = cutout.data + fits_image[0].header.update(cutout.wcs.to_header()) + fits_image[0].header["EXPTIME"] = exptime else: fits_image = None @@ -336,7 +429,7 @@ def TWOMASS_cutout(position, image_size=None, filter=None): if re.match(f"https://irsa.*{filter.lower()}i.*fits", line.split("]]>")[0]): fitsurl = line.split("]]")[0] - fits_image = fits.open(fitsurl) + fits_image = fits.open(fitsurl, cache=None) wcs = WCS(fits_image[0].header) if position.contained_by(wcs): @@ -371,32 +464,59 @@ def SDSS_cutout(position, image_size=None, filter=None): sdss_baseurl = "https://dr14.sdss.org/sas" print(position) - xid = SDSS.query_region(position, radius=0.05 * u.deg) - if xid is not None: - sc = SkyCoord(xid["ra"], xid["dec"], unit=u.deg) - sep = position.separation(sc).arcsec - iSep = np.where(sep == min(sep))[0][0] - - link = SDSS.IMAGING_URL_SUFFIX.format( - base=sdss_baseurl, - run=xid[iSep]["run"], - dr=14, - instrument="eboss", - rerun=xid[iSep]["rerun"], - camcol=xid[iSep]["camcol"], - field=xid[iSep]["field"], - band=filter, - ) - fits_image = fits.open(link) + # xid = SDSS.query_region(position, radius=0.05 * u.deg) + # ramin,ramax,decmin,decmax = getRADecBox(position.ra.deg,position.dec.deg,size=0.02) + # sqlquery = f"""SELECT DISTINCT p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field FROM PhotoObjAll AS p WHERE p.ra between {ramin} and {ramax} and p.dec between {decmin} and {decmax}""" + # import pdb; pdb.set_trace() + # xid = SkyServer.sqlSearch(sqlquery) + + url = f"https://dr12.sdss.org/fields/raDec?ra={position.ra.deg}&dec={position.dec.deg}" + print(url) + rt = requests.get(url) + + # a little latency so that we don't look like a bot to SDSS? + time.sleep(1) + if "Error: Couldn't find field covering" in rt.text: + return None + + # if not len(xid): + # return None + # if xid is not None and 'titleSkyserver_Errortitle' in xid.keys(): + # RuntimeWarning(f'SDSS query fail for position {position.ra.deg},{position.dec.deg}') + # return None + + # if len(xid) and xid is not None: + # sc = SkyCoord(xid["ra"], xid["dec"], unit=u.deg) + # sep = position.separation(sc).arcsec + # iSep = np.where(sep == min(sep))[0][0] + + regex = "
run<\/dt>.*
.*<\/dd>" + run = re.findall("
run
\n.*
([0-9]+)
", rt.text)[0] + rerun = re.findall("
rerun
\n.*
([0-9]+)
", rt.text)[0] + camcol = re.findall("
camcol
\n.*
([0-9]+)
", rt.text)[0] + field = re.findall("
field
\n.*
([0-9]+)
", rt.text)[0] + + link = SDSS.IMAGING_URL_SUFFIX.format( + base=sdss_baseurl, + run=int(run), + dr=14, + instrument="eboss", + rerun=int(rerun), + camcol=int(camcol), + field=int(field), + band=filter, + ) - wcs = WCS(fits_image[0].header) - cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs) - fits_image[0].data = cutout.data - fits_image[0].header.update(cutout.wcs.to_header()) + fits_image = fits.open(link, cache=None) - else: - fits_image = None + wcs = WCS(fits_image[0].header) + cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs) + fits_image[0].data = cutout.data + fits_image[0].header.update(cutout.wcs.to_header()) + + # else: + # fits_image = None return fits_image @@ -432,12 +552,14 @@ def cutout(transient, survey, fov=Quantity(0.1, unit="deg")): Image cutout in fits format or if the image cannot be download due to a `ReadTimeoutError` None will be returned. """ + # need to make sure the cache doesn't overfill + astropy.utils.data.clear_download_cache() num_pixels = int(fov.to(u.arcsec).value / survey.pixel_size_arcsec) if survey.image_download_method == "hips": try: fits = hips_cutout(transient, survey, image_size=num_pixels) - except: + except Exception as e: print(f"Conection timed out, could not download {survey.name} data") fits = None else: diff --git a/app/host/fixtures/example/2010ag.yaml b/app/host/fixtures/example/2010ag.yaml index 03cdc032..31dbd717 100644 --- a/app/host/fixtures/example/2010ag.yaml +++ b/app/host/fixtures/example/2010ag.yaml @@ -13,7 +13,7 @@ ra_deg: 255.97346 dec_deg: 31.50172 tasks_initialized: True - spectroscopic_class: SN 1a + spectroscopic_class: SN Ia host: [UGC 10679] - model: host.TaskRegister fields: diff --git a/app/host/fixtures/example/2010ai.yaml b/app/host/fixtures/example/2010ai.yaml index 5e90ff6a..2f6f9683 100644 --- a/app/host/fixtures/example/2010ai.yaml +++ b/app/host/fixtures/example/2010ai.yaml @@ -7,7 +7,7 @@ ra_deg: 194.85012 dec_deg: 27.99642 tasks_initialized: True - spectroscopic_class: SN 1a + spectroscopic_class: SN Ia - model: host.TaskRegister fields: transient: [2010ai] diff --git a/app/host/fixtures/example/2010h.yaml b/app/host/fixtures/example/2010h.yaml index 4c4a8334..e597dadf 100644 --- a/app/host/fixtures/example/2010h.yaml +++ b/app/host/fixtures/example/2010h.yaml @@ -4,6 +4,7 @@ redshift: 0.015355 ra_deg: 121.6004583 dec_deg: 1.0359861 + milkyway_dust_reddening: 0.026 - model: host.Transient fields: name: 2010h @@ -13,13 +14,14 @@ ra_deg: 121.6015 dec_deg: 1.03586 host: [IC 494] - spectroscopic_class: SN 1a + spectroscopic_class: SN Ia + milkyway_dust_reddening: 0.026 tasks_initialized: True - model: host.TaskRegister fields: transient: [2010h] task: [Transient MWEBV] - status: [not processed] + status: [processed] last_modified: 1999-01-23 01:23:45.678+00:00 last_processing_time_seconds: 45.0 - model: host.TaskRegister @@ -33,7 +35,7 @@ fields: transient: [2010h] task: [Host MWEBV] - status: [not processed] + status: [processed] last_modified: 1998-01-23 01:23:45.678+00:00 last_processing_time_seconds: 35.0 - model: host.TaskRegister @@ -54,7 +56,21 @@ fields: transient: [2010h] task: [Global aperture photometry] - status: [processed] + status: [not processed] + last_modified: 2000-01-23 01:23:45.678+00:00 + last_processing_time_seconds: 11.2323 +- model: host.TaskRegister + fields: + transient: [2010h] + task: [Validate global photometry] + status: [not processed] + last_modified: 2000-01-23 01:23:45.678+00:00 + last_processing_time_seconds: 11.2323 +- model: host.TaskRegister + fields: + transient: [2010h] + task: [Validate local photometry] + status: [not processed] last_modified: 2000-01-23 01:23:45.678+00:00 last_processing_time_seconds: 11.2323 - model: host.TaskRegister @@ -68,21 +84,21 @@ fields: transient: [2010h] task: [Local aperture photometry] - status: [processed] + status: [not processed] last_modified: 2000-01-23 01:23:45.678+00:00 last_processing_time_seconds: 0.03 - model: host.TaskRegister fields: transient: [2010h] task: [Global host SED inference] - status: [processed] + status: [not processed] last_modified: 2000-01-23 01:23:45.678+00:00 last_processing_time_seconds: 0.004 - model: host.TaskRegister fields: transient: [2010h] task: [Local host SED inference] - status: [processed] + status: [not processed] last_modified: 2000-01-23 01:23:45.678+00:00 last_processing_time_seconds: 100.0 - model: host.TaskRegister @@ -97,109 +113,109 @@ name: 2010h_PanSTARRS_g filter: [PanSTARRS_g] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_g.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_g.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_r filter: [PanSTARRS_r] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_r.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_r.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_i filter: [PanSTARRS_i] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_i.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_i.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_z filter: [PanSTARRS_z] transient: [2010h] - fits: ../cutout_cdn/2010H/PanSTARRS/PanSTARRS_z.fits + fits: ../data/cutout_cdn/2010H/PanSTARRS/PanSTARRS_z.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_y filter: [PanSTARRS_y] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_y.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_y.fits - model: host.Cutout fields: name: 2010h_SDSS_u filter: [SDSS_u] transient: [2010h] - fits: ../cutout_cdn/2010h/SDSS/SDSS_u.fits + fits: ../data/cutout_cdn/2010h/SDSS/SDSS_u.fits - model: host.Cutout fields: name: 2010h_SDSS_r filter: [SDSS_r] transient: [2010h] - fits: ../cutout_cdn/2010h/SDSS/SDSS_r.fits + fits: ../data/cutout_cdn/2010h/SDSS/SDSS_r.fits - model: host.Cutout fields: name: 2010h_SDSS_i filter: [SDSS_i] transient: [2010h] - fits: ../cutout_cdn/2010h/SDSS/SDSS_i.fits + fits: ../data/cutout_cdn/2010h/SDSS/SDSS_i.fits - model: host.Cutout fields: name: 2010H_SDSS_z filter: [SDSS_z] transient: [2010h] - fits: ../cutout_cdn/2010H/SDSS/SDSS_z.fits + fits: ../data/cutout_cdn/2010H/SDSS/SDSS_z.fits - model: host.Cutout fields: name: 2010h_GALEX_NUV filter: [GALEX_NUV] transient: [2010h] - fits: ../cutout_cdn/2010h/GALEX/GALEX_NUV.fits + fits: ../data/cutout_cdn/2010h/GALEX/GALEX_NUV.fits - model: host.Cutout fields: name: 2010h_GALEX_FUV filter: [GALEX_FUV] transient: [2010h] - fits: ../cutout_cdn/2010h/GALEX/GALEX_FUV.fits + fits: ../data/cutout_cdn/2010h/GALEX/GALEX_FUV.fits - model: host.Cutout fields: name: 2010h_2MASS_J filter: [2MASS_J] transient: [2010h] - fits: ../cutout_cdn/2010h/2MASS/2MASS_J.fits + fits: ../data/cutout_cdn/2010h/2MASS/2MASS_J.fits - model: host.Cutout fields: name: 2010h_2MASS_H filter: [2MASS_H] transient: [2010h] - fits: ../cutout_cdn/2010h/2MASS/2MASS_H.fits + fits: ../data/cutout_cdn/2010h/2MASS/2MASS_H.fits - model: host.Cutout fields: name: 2010h_2MASS_K filter: [2MASS_K] transient: [2010h] - fits: ../cutout_cdn/2010h/2MASS/2MASS_K.fits + fits: ../data/cutout_cdn/2010h/2MASS/2MASS_K.fits - model: host.Cutout fields: name: 2010h_WISE_W1 filter: [WISE_W1] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W1.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W1.fits - model: host.Cutout fields: name: 2010h_WISE_W2 filter: [WISE_W2] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W2.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W2.fits - model: host.Cutout fields: name: 2010h_WISE_W3 filter: [WISE_W3] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W3.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W3.fits - model: host.Cutout fields: name: 2010h_WISE_W4 filter: [WISE_W4] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W4.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W4.fits - model: host.Aperture fields: name: 2010h_PanSTARRS_g_global @@ -222,349 +238,3 @@ semi_major_axis_arcsec: 1.0 semi_minor_axis_arcsec: 1.0 type: local -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [PanSTARRS_g] - transient: [2010h] - flux: 66.48 - flux_error: 0.3 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [PanSTARRS_r] - transient: [2010h] - flux: 134.14 - flux_error: 0.35 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [PanSTARRS_i] - transient: [2010h] - flux: 199.2 - flux_error: 0.39 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [PanSTARRS_z] - transient: [2010h] - flux: 251.87 - flux_error: 0.87 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [PanSTARRS_y] - transient: [2010h] - flux: 318.33 - flux_error: 1.79 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [SDSS_u] - transient: [2010h] - flux: 24.5 - flux_error: 9.52 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [SDSS_r] - transient: [2010h] - flux: 265.28 - flux_error: 31.09 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [SDSS_i] - transient: [2010h] - flux: 406.41 - flux_error: 38.5 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [SDSS_z] - transient: [2010h] - flux: 537.61 - flux_error: 44.29 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [GALEX_NUV] - transient: [2010h] - flux: 1.27 - flux_error: 6.88 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [2MASS_J] - transient: [2010h] - flux: 1091.48 - flux_error: 130.38 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [2MASS_H] - transient: [2010h] - flux: 2183.8 - flux_error: 224.97 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [2MASS_K] - transient: [2010h] - flux: 2744.2 - flux_error: 321.52 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [WISE_W1] - transient: [2010h] - flux: 300.12 - flux_error: 76.95 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [WISE_W2] - transient: [2010h] - flux: 332.67 - flux_error: 164.67 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [WISE_W3] - transient: [2010h] - flux: 2095.17 - flux_error: 4138.78 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_local] - filter: [WISE_W4] - transient: [2010h] - flux: 1196.85 - flux_error: 99386.74 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [PanSTARRS_g] - transient: [2010h] - flux: 11486.21 - flux_error: 6.74 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [PanSTARRS_r] - transient: [2010h] - flux: 22544.18 - flux_error: 7.45 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [PanSTARRS_i] - transient: [2010h] - flux: 32834.65 - flux_error: 9.23 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [PanSTARRS_z] - transient: [2010h] - flux: 40192.33 - flux_error: 21.9 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [PanSTARRS_y] - transient: [2010h] - flux: 51028.82 - flux_error: 46.89 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [SDSS_u] - transient: [2010h] - flux: 3836.16 - flux_error: 124.09 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [SDSS_r] - transient: [2010h] - flux: 40749.25 - flux_error: 387.75 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [SDSS_i] - transient: [2010h] - flux: 60157.54 - flux_error: 472.23 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [SDSS_z] - transient: [2010h] - flux: 78708.5 - flux_error: 541.32 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [GALEX_NUV] - transient: [2010h] - flux: 171.44 - flux_error: 94.73 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [2MASS_J] - transient: [2010h] - flux: 131353.83 - flux_error: 1394.29 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [2MASS_H] - transient: [2010h] - flux: 260596.03 - flux_error: 2474.14 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [2MASS_K] - transient: [2010h] - flux: 318604.53 - flux_error: 3663.86 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [WISE_W1] - transient: [2010h] - flux: 69285.33 - flux_error: 1264.35 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [WISE_W2] - transient: [2010h] - flux: 78341.46 - flux_error: 3323.04 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [WISE_W3] - transient: [2010h] - flux: 516932.66 - flux_error: 109896.85 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.AperturePhotometry - fields: - aperture: [2010h_PanSTARRS_g_global] - filter: [WISE_W4] - transient: [2010h] - flux: 326671.97 - flux_error: 2666271.5 - magnitude: 0.0 - magnitude_error: 0.0 -- model: host.SEDFittingResult - fields: - transient: [2010h] - aperture: [2010h_PanSTARRS_g_global] - posterior: /sed_output/2010h/2010h_global.h5 - log_mass_16: 11.11623006468808 - log_mass_50: 11.119444597406906 - log_mass_84: 11.122195767038848 - log_sfr_16: -9.885957158543373 - log_sfr_50: -9.452641904357886 - log_sfr_84: -7.854718482318168 - log_ssfr_16: -21.005705687227554 - log_ssfr_50: -20.56957875030653 - log_ssfr_84: -18.975108348918212 - log_age_16: 3.3558538428424978 - log_age_50: 3.3848699023305824 - log_age_84: 3.4566111510622477 - log_tau_16: 0.10096442863244388 - log_tau_50: 0.1046197140354783 - log_tau_84: 0.1261118048050916 -- model: host.SEDFittingResult - fields: - transient: [2010h] - aperture: [2010h_local] - posterior: /sed_output/2010h/2010h_local.h5 - log_mass_16: 8.849994109986605 - log_mass_50: 8.894097969541852 - log_mass_84: 8.96004598530449 - log_sfr_16: -8.953639946834054 - log_sfr_50: -5.493737812812891 - log_sfr_84: -2.949760445521605 - log_ssfr_16: -17.851239889289612 - log_ssfr_50: -14.381920176924307 - log_ssfr_84: -11.890886258100974 - log_age_16: 2.7893040809653034 - log_age_50: 3.9156213648412592 - log_age_84: 6.416906088053504 - log_tau_16: 0.1255658907438466 - log_tau_50: 0.22027085216263398 - log_tau_84: 0.5961470357024149 diff --git a/app/host/fixtures/initial/setup_filter_data.yaml b/app/host/fixtures/initial/setup_filter_data.yaml index 67c064e2..2cda551f 100644 --- a/app/host/fixtures/initial/setup_filter_data.yaml +++ b/app/host/fixtures/initial/setup_filter_data.yaml @@ -12,6 +12,7 @@ wavelength_min_angstrom: 14787.38 wavelength_max_angstrom: 18231.02 vega_zero_point_jansky: 1024.00 + ab_offset: 1.3676 magnitude_zero_point_keyword: MAGZP image_pixel_units: 'counts' kcorrect_name: 'twomass_H' @@ -29,6 +30,7 @@ wavelength_min_angstrom: 10806.47 wavelength_max_angstrom: 14067.97 vega_zero_point_jansky: 1594.00 + ab_offset: 0.8949 magnitude_zero_point_keyword: MAGZP image_pixel_units: 'counts' kcorrect_name: 'twomass_J' @@ -46,6 +48,7 @@ wavelength_min_angstrom: 19543.69 wavelength_max_angstrom: 23552.40 vega_zero_point_jansky: 666.80 + ab_offset: 1.8378 magnitude_zero_point_keyword: MAGZP image_pixel_units: 'counts' kcorrect_name: 'twomass_Ks' @@ -63,6 +66,7 @@ wavelength_min_angstrom: 27540.97 wavelength_max_angstrom: 38723.88 vega_zero_point_jansky: 309.54 + ab_offset: 2.699 magnitude_zero_point: 20.5 image_pixel_units: 'counts/sec' kcorrect_name: 'wise_w1' @@ -80,6 +84,7 @@ wavelength_min_angstrom: 39633.26 wavelength_max_angstrom: 53413.60 vega_zero_point_jansky: 171.79 + ab_offset: 3.339 magnitude_zero_point: 19.5 image_pixel_units: 'counts/sec' kcorrect_name: 'wise_w2' @@ -97,6 +102,7 @@ wavelength_min_angstrom: 74430.44 wavelength_max_angstrom: 172613.43 vega_zero_point_jansky: 31.67 + ab_offset: 5.174 magnitude_zero_point: 18.0 image_pixel_units: 'counts/sec' kcorrect_name: 'wise_w3' @@ -114,6 +120,7 @@ wavelength_min_angstrom: 195200.83 wavelength_max_angstrom: 279107.24 vega_zero_point_jansky: 8.36 + ab_offset: 6.620 magnitude_zero_point: 13.0 image_pixel_units: 'counts/sec' kcorrect_name: 'wise_w4' @@ -131,6 +138,7 @@ wavelength_min_angstrom: 1692.52 wavelength_max_angstrom: 3006.67 vega_zero_point_jansky: 788.55 + ab_offset: 0 magnitude_zero_point: 20.08 image_pixel_units: 'counts/sec' kcorrect_name: 'galex_NUV' @@ -148,6 +156,7 @@ wavelength_min_angstrom: 1340.32 wavelength_max_angstrom: 1806.43 vega_zero_point_jansky: 520.73 + ab_offset: 0 magnitude_zero_point: 18.82 image_pixel_units: 'counts/sec' kcorrect_name: 'galex_FUV' @@ -165,6 +174,7 @@ wavelength_min_angstrom: 3943.40 wavelength_max_angstrom: 5593.27 vega_zero_point_jansky: 3893.00 + ab_offset: 0 magnitude_zero_point: 25.0 image_pixel_units: 'counts' kcorrect_name: 'none' @@ -182,6 +192,7 @@ wavelength_min_angstrom: 5386.23 wavelength_max_angstrom: 7035.65 vega_zero_point_jansky: 3135.50 + ab_offset: 0 magnitude_zero_point: 25.0 image_pixel_units: 'counts' kcorrect_name: 'none' @@ -199,6 +210,7 @@ wavelength_min_angstrom: 6778.45 wavelength_max_angstrom: 8304.37 vega_zero_point_jansky: 2577.03 + ab_offset: 0 magnitude_zero_point: 25.0 image_pixel_units: 'counts' kcorrect_name: 'none' @@ -216,6 +228,7 @@ wavelength_min_angstrom: 8028.00 wavelength_max_angstrom: 9346.00 vega_zero_point_jansky: 2273.05 + ab_offset: 0 magnitude_zero_point: 25.0 image_pixel_units: 'counts' kcorrect_name: 'none' @@ -233,6 +246,7 @@ wavelength_min_angstrom: 9100.50 wavelength_max_angstrom: 10838.50 vega_zero_point_jansky: 2203.66 + ab_offset: 0 magnitude_zero_point: 25.0 image_pixel_units: 'counts' kcorrect_name: 'none' @@ -246,10 +260,11 @@ image_fwhm_arcsec: 1.26 vosa_id: SLOAN/SDSS.i image_download_method: custom - wavelength_eff_angstrom: 4671.78 - wavelength_min_angstrom: 3782.54 - wavelength_max_angstrom: 5549.26 + wavelength_eff_angstrom: 7470.60 + wavelength_min_angstrom: 6430.00 + wavelength_max_angstrom: 8630.00 vega_zero_point_jansky: 3971.19 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'sdss_i0' @@ -267,6 +282,7 @@ wavelength_min_angstrom: 5415.34 wavelength_max_angstrom: 6989.14 vega_zero_point_jansky: 3142.02 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'sdss_r0' @@ -284,6 +300,7 @@ wavelength_min_angstrom: 3048.28 wavelength_max_angstrom: 4028.23 vega_zero_point_jansky: 1628.72 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'sdss_u0' @@ -301,6 +318,7 @@ wavelength_min_angstrom: 7960.44 wavelength_max_angstrom: 10833.25 vega_zero_point_jansky: 2227.03 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'sdss_z0' @@ -318,6 +336,7 @@ wavelength_min_angstrom: 3797.64 wavelength_max_angstrom: 5553.04 vega_zero_point_jansky: 4023.57 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'sdss_g0' @@ -335,6 +354,7 @@ wavelength_min_angstrom: 3925.42 wavelength_max_angstrom: 5529.96 vega_zero_point_jansky: 3905.01 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'decam_g' @@ -352,6 +372,7 @@ wavelength_min_angstrom: 5519.97 wavelength_max_angstrom: 7233.72 vega_zero_point_jansky: 3027.32 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'decam_r' @@ -369,6 +390,7 @@ wavelength_min_angstrom: 6948.18 wavelength_max_angstrom: 8645.68 vega_zero_point_jansky: 2467.82 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'decam_i' @@ -386,6 +408,7 @@ wavelength_min_angstrom: 8341.20 wavelength_max_angstrom: 10074.26 vega_zero_point_jansky: 2247.33 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'decam_z' @@ -403,6 +426,7 @@ wavelength_min_angstrom: 9364.77 wavelength_max_angstrom: 10685.78 vega_zero_point_jansky: 2146.18 + ab_offset: 0 magnitude_zero_point: 22.5 image_pixel_units: 'counts/sec' kcorrect_name: 'decam_Y' diff --git a/app/host/fixtures/initial/setup_status.yaml b/app/host/fixtures/initial/setup_status.yaml index 5ecd781f..d1835b19 100644 --- a/app/host/fixtures/initial/setup_status.yaml +++ b/app/host/fixtures/initial/setup_status.yaml @@ -34,3 +34,19 @@ fields: message: phot valid failed type: error +- model: host.Status + fields: + message: not enough filters + type: error +- model: host.Status + fields: + message: time limit exceeded + type: error +- model: host.Status + fields: + message: blocked + type: error +- model: host.Status + fields: + message: redshift too high + type: error diff --git a/app/host/fixtures/test/test_2010h.yaml b/app/host/fixtures/test/test_2010h.yaml index 4c4a8334..e4af9c6b 100644 --- a/app/host/fixtures/test/test_2010h.yaml +++ b/app/host/fixtures/test/test_2010h.yaml @@ -97,109 +97,109 @@ name: 2010h_PanSTARRS_g filter: [PanSTARRS_g] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_g.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_g.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_r filter: [PanSTARRS_r] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_r.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_r.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_i filter: [PanSTARRS_i] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_i.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_i.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_z filter: [PanSTARRS_z] transient: [2010h] - fits: ../cutout_cdn/2010H/PanSTARRS/PanSTARRS_z.fits + fits: ../data/cutout_cdn/2010H/PanSTARRS/PanSTARRS_z.fits - model: host.Cutout fields: name: 2010h_PanSTARRS_y filter: [PanSTARRS_y] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_y.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_y.fits - model: host.Cutout fields: name: 2010h_SDSS_u filter: [SDSS_u] transient: [2010h] - fits: ../cutout_cdn/2010h/SDSS/SDSS_u.fits + fits: ../data/cutout_cdn/2010h/SDSS/SDSS_u.fits - model: host.Cutout fields: name: 2010h_SDSS_r filter: [SDSS_r] transient: [2010h] - fits: ../cutout_cdn/2010h/SDSS/SDSS_r.fits + fits: ../data/cutout_cdn/2010h/SDSS/SDSS_r.fits - model: host.Cutout fields: name: 2010h_SDSS_i filter: [SDSS_i] transient: [2010h] - fits: ../cutout_cdn/2010h/SDSS/SDSS_i.fits + fits: ../data/cutout_cdn/2010h/SDSS/SDSS_i.fits - model: host.Cutout fields: name: 2010H_SDSS_z filter: [SDSS_z] transient: [2010h] - fits: ../cutout_cdn/2010H/SDSS/SDSS_z.fits + fits: ../data/cutout_cdn/2010H/SDSS/SDSS_z.fits - model: host.Cutout fields: name: 2010h_GALEX_NUV filter: [GALEX_NUV] transient: [2010h] - fits: ../cutout_cdn/2010h/GALEX/GALEX_NUV.fits + fits: ../data/cutout_cdn/2010h/GALEX/GALEX_NUV.fits - model: host.Cutout fields: name: 2010h_GALEX_FUV filter: [GALEX_FUV] transient: [2010h] - fits: ../cutout_cdn/2010h/GALEX/GALEX_FUV.fits + fits: ../data/cutout_cdn/2010h/GALEX/GALEX_FUV.fits - model: host.Cutout fields: name: 2010h_2MASS_J filter: [2MASS_J] transient: [2010h] - fits: ../cutout_cdn/2010h/2MASS/2MASS_J.fits + fits: ../data/cutout_cdn/2010h/2MASS/2MASS_J.fits - model: host.Cutout fields: name: 2010h_2MASS_H filter: [2MASS_H] transient: [2010h] - fits: ../cutout_cdn/2010h/2MASS/2MASS_H.fits + fits: ../data/cutout_cdn/2010h/2MASS/2MASS_H.fits - model: host.Cutout fields: name: 2010h_2MASS_K filter: [2MASS_K] transient: [2010h] - fits: ../cutout_cdn/2010h/2MASS/2MASS_K.fits + fits: ../data/cutout_cdn/2010h/2MASS/2MASS_K.fits - model: host.Cutout fields: name: 2010h_WISE_W1 filter: [WISE_W1] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W1.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W1.fits - model: host.Cutout fields: name: 2010h_WISE_W2 filter: [WISE_W2] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W2.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W2.fits - model: host.Cutout fields: name: 2010h_WISE_W3 filter: [WISE_W3] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W3.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W3.fits - model: host.Cutout fields: name: 2010h_WISE_W4 filter: [WISE_W4] transient: [2010h] - fits: ../cutout_cdn/2010h/WISE/WISE_W4.fits + fits: ../data/cutout_cdn/2010h/WISE/WISE_W4.fits - model: host.Aperture fields: name: 2010h_PanSTARRS_g_global diff --git a/app/host/fixtures/test/test_2010h_onefilter.yaml b/app/host/fixtures/test/test_2010h_onefilter.yaml index 3ac9bb6a..dc416277 100644 --- a/app/host/fixtures/test/test_2010h_onefilter.yaml +++ b/app/host/fixtures/test/test_2010h_onefilter.yaml @@ -97,7 +97,7 @@ name: 2010h_PanSTARRS_g filter: [PanSTARRS_g] transient: [2010h] - fits: ../cutout_cdn/2010h/PanSTARRS/PanSTARRS_g.fits + fits: ../data/cutout_cdn/2010h/PanSTARRS/PanSTARRS_g.fits - model: host.Aperture fields: name: 2010h_PanSTARRS_g_global diff --git a/app/host/fixtures/test/test_cutout.yaml b/app/host/fixtures/test/test_cutout.yaml index 482a3481..473276ae 100644 --- a/app/host/fixtures/test/test_cutout.yaml +++ b/app/host/fixtures/test/test_cutout.yaml @@ -3,4 +3,4 @@ name: testone_WISE_W4 filter: [WISE_W4] transient: [2022testone] - fits: /../cutout_cdn/2010h/WISE/WISE_W4.fits + fits: /../data/cutout_cdn/2010h/WISE/WISE_W4.fits diff --git a/app/host/forms.py b/app/host/forms.py index 2ddd0b34..0810ea13 100644 --- a/app/host/forms.py +++ b/app/host/forms.py @@ -3,9 +3,16 @@ class TransientSearchForm(forms.Form): name = forms.CharField( - label="", widget=forms.TextInput(attrs={"placeholder": "e.g. 2022eqw"}) + label="", + widget=forms.TextInput( + attrs={"placeholder": "e.g. 2022eqw", "style": "width:10em"} + ), + required=False, ) + ### optional "status" is read by the transient_list view + status = forms.CharField(label="", initial="all", required=False) + class ImageGetForm(forms.Form): def __init__(self, *args, **kwargs): @@ -18,3 +25,16 @@ def __init__(self, *args, **kwargs): choices=choices, widget=forms.Select(attrs={"placeholder": "select cutout"}), ) + + +class TransientUploadForm(forms.Form): + tns_names = forms.CharField( + widget=forms.Textarea, + label="Transients by Name, using TNS to gather additional information", + required=False, + ) + full_info = forms.CharField( + widget=forms.Textarea, + label='Comma-separated: Name, RA, Dec, Redshift, Classification. RA/Dec must be decimal degrees and use "None" to indicate missing redshift or classification.', + required=False, + ) diff --git a/app/host/ghost.py b/app/host/ghost.py index 18dfbfa5..aa5e69b1 100644 --- a/app/host/ghost.py +++ b/app/host/ghost.py @@ -3,6 +3,7 @@ from astro_ghost.ghostHelperFunctions import getGHOST from astro_ghost.ghostHelperFunctions import getTransientHosts +from astro_ghost.photoz_helper import calc_photoz from astropy.coordinates import SkyCoord from django.conf import settings @@ -32,18 +33,25 @@ def run_ghost(transient, output_dir=settings.GHOST_OUTPUT_ROOT): try: float(transient.name) transient_name = "sn" + str(transient.name) - except: + except Exception as e: transient_name = transient.name + ### some issues with Pan-STARRS downloads host_data = getTransientHosts( - snCoord=[transient_position], - snName=[transient_name], + transientCoord=[transient_position], + transientName=[transient_name], verbose=1, savepath=output_dir, starcut="gentle", - # ascentMatch=False, + ascentMatch=False, ) + # sometimes photo-zs randomly fail + try: + host_data = calc_photoz(host_data) + except Exception as e: + pass + # clean up after GHOST... # dir_list = glob.glob('transients_*/*/*') # for dir in dir_list: os.remove(dir) @@ -51,13 +59,22 @@ def run_ghost(transient, output_dir=settings.GHOST_OUTPUT_ROOT): # for level in ['*/*/', '*/']: # dir_list = glob.glob('transients_' + level) # for dir in dir_list: os.rmdir(dir) - if len(host_data) == 0: host = None else: host = Host( ra_deg=host_data["raMean"][0], dec_deg=host_data["decMean"][0], - name=host_data["objName"][0], + name=host_data["TransientName"][0], ) + + if host_data["NED_redshift"][0] == host_data["NED_redshift"][0]: + host.redshift = host_data["NED_redshift"][0] + + if ( + "photo_z" in host_data.keys() + and host_data["photo_z"][0] == host_data["photo_z"][0] + ): + host.photometric_redshift = host_data["photo_z"][0] + return host diff --git a/app/host/host_utils.py b/app/host/host_utils.py index a5b8ebf4..9fd29cfa 100644 --- a/app/host/host_utils.py +++ b/app/host/host_utils.py @@ -32,6 +32,9 @@ from photutils.segmentation import detect_threshold from photutils.segmentation import SourceCatalog from photutils.utils import calc_total_error +from photutils.background import LocalBackground +from photutils.background import MeanBackground, SExtractorBackground +from astropy.stats import SigmaClip from .photometric_calibration import ab_mag_to_mJy from .photometric_calibration import flux_to_mag @@ -178,7 +181,7 @@ def do_aperture_photometry(image, sky_aperture, filter): # get the background try: - background = estimate_background(image) + background = estimate_background(image, filter.name) except ValueError: # indicates poor image data return { @@ -188,8 +191,44 @@ def do_aperture_photometry(image, sky_aperture, filter): "magnitude_error": None, } + # is the aperture inside the image? + bbox = sky_aperture.to_pixel(wcs).bbox + if ( + bbox.ixmin < 0 + or bbox.iymin < 0 + or bbox.ixmax > image_data.shape[1] + or bbox.iymax > image_data.shape[0] + ): + return { + "flux": None, + "flux_error": None, + "magnitude": None, + "magnitude_error": None, + } + + # if the image pixels are all zero, let's assume this is masked + # even GALEX FUV should have *something* + phot_table_maskcheck = aperture_photometry(image_data, sky_aperture, wcs=wcs) + if phot_table_maskcheck["aperture_sum"].value[0] == 0: + return { + "flux": None, + "flux_error": None, + "magnitude": None, + "magnitude_error": None, + } + background_subtracted_data = image_data - background.background + # I think we need a local background subtraction for WISE + ### the others haven't given major problems + if "WISE" in filter.name: + aper_pix = sky_aperture.to_pixel(wcs) + lbg = LocalBackground(aper_pix.a, aper_pix.a * 2) + local_background = lbg( + background_subtracted_data, aper_pix.positions[0], aper_pix.positions[1] + ) + background_subtracted_data -= local_background + if filter.image_pixel_units == "counts/sec": error = calc_total_error( background_subtracted_data, @@ -206,7 +245,31 @@ def do_aperture_photometry(image, sky_aperture, filter): background_subtracted_data, sky_aperture, wcs=wcs, error=error ) uncalibrated_flux = phot_table["aperture_sum"].value[0] - uncalibrated_flux_err = phot_table["aperture_sum_err"].value[0] + if "2MASS" not in filter.name: + uncalibrated_flux_err = phot_table["aperture_sum_err"].value[0] + else: + # 2MASS is annoying + # https://wise2.ipac.caltech.edu/staff/jarrett/2mass/3chan/noise/ + n_pix = ( + np.pi + * sky_aperture.a.value + * sky_aperture.b.value + * filter.pixel_size_arcsec**2.0 + ) + uncalibrated_flux_err = np.sqrt( + uncalibrated_flux / (10 * 6) + + 4 * n_pix * 1.7**2.0 * np.median(background.background_rms) ** 2 + ) + + # check for correlated errors + aprad, err_adjust = filter.correlation_model() + if aprad is not None: + image_aperture = sky_aperture.to_pixel(wcs) + + err_adjust_interp = np.interp( + (image_aperture.a + image_aperture.b) / 2.0, aprad, err_adjust + ) + uncalibrated_flux_err *= err_adjust_interp if filter.magnitude_zero_point_keyword is not None: zpt = image[0].header[filter.magnitude_zero_point_keyword] @@ -225,7 +288,6 @@ def do_aperture_photometry(image, sky_aperture, filter): flux, flux_error = None, None wave_eff = filter.transmission_curve().wave_effective - return { "flux": flux, "flux_error": flux_error, @@ -262,7 +324,7 @@ def check_local_radius(redshift, image_fwhm_arcsec): dadist * 1000 * (np.pi / 180.0 / 3600.0) ) # 2 kpc aperture radius is this many arcsec - return apr_arcsec > image_fwhm_arcsec + return "true" if apr_arcsec > image_fwhm_arcsec else "false" def check_global_contamination(global_aperture_phot, aperture_primary): @@ -313,7 +375,7 @@ def check_global_contamination(global_aperture_phot, aperture_primary): return is_contam -def select_cutout_aperture(cutouts): +def select_cutout_aperture(cutouts, choice=0): """ Select cutout for aperture """ @@ -330,7 +392,8 @@ def select_cutout_aperture(cutouts): "2MASS_H", ] - choice = 0 + #### choice = 0 + ### edited to allow initial offset filter_choice = filter_names[choice] while not cutouts.filter(filter__name=filter_choice).exists(): @@ -394,7 +457,7 @@ def select_aperture(transient): # return host_position -def estimate_background(image): +def estimate_background(image, filter_name=None): """ Estimates the background of an image Parameters @@ -408,10 +471,19 @@ def estimate_background(image): """ image_data = image[0].data box_size = int(0.1 * np.sqrt(image_data.size)) + + # GALEX needs mean, not median - median just always comes up with zero + if filter_name is not None and "GALEX" in filter_name: + bkg = MeanBackground(SigmaClip(sigma=3.0)) + else: + bkg = SExtractorBackground(sigma_clip=None) + try: - return Background2D(image_data, box_size=box_size) + return Background2D(image_data, box_size=box_size, bkg_estimator=bkg) except ValueError: - return Background2D(image_data, box_size=box_size, exclude_percentile=50) + return Background2D( + image_data, box_size=box_size, exclude_percentile=50, bkg_estimator=bkg + ) def construct_aperture(image, position): @@ -428,11 +500,9 @@ def construct_aperture(image, position): ### found an edge case where deblending isn't working how I'd like it to ### so if it's not finding the host, play with the default threshold - iter = 0 - source_separation_arcsec = 100 - while source_separation_arcsec > 5 and iter < 5: + def get_source_data(threshhold_sigma): catalog = build_source_catalog( - image, background, threshhold_sigma=5 * (iter + 1) + image, background, threshhold_sigma=threshhold_sigma ) source_data = match_source(position, catalog, wcs) @@ -441,8 +511,21 @@ def construct_aperture(image, position): ) source_position = SkyCoord(source_ra, source_dec, unit=u.deg) source_separation_arcsec = position.separation(source_position).arcsec + return source_data, source_separation_arcsec + iter = 0 + source_separation_arcsec = 100 + while source_separation_arcsec > 5 and iter < 5: + source_data, source_separation_arcsec = get_source_data(5 * (iter + 1)) iter += 1 + # look for sub-threshold sources + # if we still can't find the host + if source_separation_arcsec > 5: + source_data, source_separation_arcsec = get_source_data(2) + + # make sure we know this failed + if source_separation_arcsec > 5: + return None return elliptical_sky_aperture(source_data, wcs) @@ -456,7 +539,11 @@ def query_ned(position): redshift = result_table["Redshift"].value if len(redshift): - galaxy_data = {"redshift": redshift[0]} + pos = SkyCoord(result_table["RA"].value, result_table["DEC"].value, unit=u.deg) + sep = position.separation(pos).arcsec + iBest = np.where(sep == np.min(sep))[0][0] + + galaxy_data = {"redshift": redshift[iBest]} else: galaxy_data = {"redshift": None} diff --git a/app/host/migrations/0012_filter_ab_offset.py b/app/host/migrations/0012_filter_ab_offset.py new file mode 100644 index 00000000..7ddc228d --- /dev/null +++ b/app/host/migrations/0012_filter_ab_offset.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.9 on 2023-04-11 22:07 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("host", "0011_filter_magnitude_zero_point_keyword"), + ] + + operations = [ + migrations.AddField( + model_name="filter", + name="ab_offset", + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py b/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py new file mode 100644 index 00000000..94792993 --- /dev/null +++ b/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.9 on 2023-05-26 00:01 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("host", "0012_filter_ab_offset"), + ] + + operations = [ + migrations.AddField( + model_name="sedfittingresult", + name="mass_surviving_ratio", + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/app/host/migrations/0014_taskregister_user_warning.py b/app/host/migrations/0014_taskregister_user_warning.py new file mode 100644 index 00000000..436e9ec5 --- /dev/null +++ b/app/host/migrations/0014_taskregister_user_warning.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.9 on 2023-11-19 22:14 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("host", "0013_sedfittingresult_mass_surviving_ratio"), + ] + + operations = [ + migrations.AddField( + model_name="taskregister", + name="user_warning", + field=models.BooleanField(blank=True, null=True), + ), + ] diff --git a/app/host/migrations/0015_alter_taskregister_user_warning.py b/app/host/migrations/0015_alter_taskregister_user_warning.py new file mode 100644 index 00000000..0ca92ab5 --- /dev/null +++ b/app/host/migrations/0015_alter_taskregister_user_warning.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.9 on 2023-11-19 23:10 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("host", "0014_taskregister_user_warning"), + ] + + operations = [ + migrations.AlterField( + model_name="taskregister", + name="user_warning", + field=models.BooleanField(default=False), + ), + ] diff --git a/app/host/migrations/0016_transient_added_by.py b/app/host/migrations/0016_transient_added_by.py new file mode 100644 index 00000000..1cc9ed7c --- /dev/null +++ b/app/host/migrations/0016_transient_added_by.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.9 on 2023-12-26 02:30 +import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("host", "0015_alter_taskregister_user_warning"), + ] + + operations = [ + migrations.AddField( + model_name="transient", + name="added_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/app/host/migrations/0017_alter_aperturephotometry_is_validated.py b/app/host/migrations/0017_alter_aperturephotometry_is_validated.py new file mode 100644 index 00000000..bd7bd4b7 --- /dev/null +++ b/app/host/migrations/0017_alter_aperturephotometry_is_validated.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.9 on 2023-12-26 03:06 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("host", "0016_transient_added_by"), + ] + + operations = [ + migrations.AlterField( + model_name="aperturephotometry", + name="is_validated", + field=models.CharField(blank=True, max_length=40, null=True), + ), + ] diff --git a/app/host/migrations/0018_auto_20240205_0240.py b/app/host/migrations/0018_auto_20240205_0240.py new file mode 100644 index 00000000..6081612f --- /dev/null +++ b/app/host/migrations/0018_auto_20240205_0240.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.9 on 2024-02-05 02:40 +import host.models +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("host", "0017_alter_aperturephotometry_is_validated"), + ] + + operations = [ + migrations.AddField( + model_name="sedfittingresult", + name="chains_file", + field=models.FileField( + blank=True, null=True, upload_to=host.models.npz_chains_file_path + ), + ), + migrations.AddField( + model_name="sedfittingresult", + name="model_file", + field=models.FileField( + blank=True, null=True, upload_to=host.models.npz_model_file_path + ), + ), + migrations.AddField( + model_name="sedfittingresult", + name="percentiles_file", + field=models.FileField( + blank=True, null=True, upload_to=host.models.npz_percentiles_file_path + ), + ), + ] diff --git a/app/host/models.py b/app/host/models.py index 0e153f81..7a945fff 100644 --- a/app/host/models.py +++ b/app/host/models.py @@ -2,10 +2,13 @@ This modules contains the django code used to create tables in the database backend. """ +import os + import pandas as pd from astropy import units as u from astropy.coordinates import SkyCoord from django.conf import settings +from django.contrib.auth.models import User from django.db import models from django_celery_beat.models import PeriodicTask from photutils.aperture import SkyEllipticalAperture @@ -115,6 +118,7 @@ class Transient(SkyObject): photometric_class = models.CharField(max_length=20, null=True, blank=True) milkyway_dust_reddening = models.FloatField(null=True, blank=True) processing_status = models.CharField(max_length=20, default="processing") + added_by = models.ForeignKey(User, null=True, blank=True, on_delete=models.CASCADE) @property def progress(self): @@ -130,7 +134,15 @@ def progress(self): def best_redshift(self): """get the best redshift for a transient""" if self.host is not None and self.host.redshift is not None: - z = self.host.redshift + if ( + self.redshift is not None + and abs(self.host.redshift - self.redshift) < 0.02 + ): + z = self.host.redshift + elif self.redshift is None: + z = self.host.redshift + else: + z = self.redshift elif self.redshift is not None: z = self.redshift elif self.host is not None and self.host.photometric_redshift is not None: @@ -197,6 +209,7 @@ class TaskRegister(models.Model): task = models.ForeignKey(Task, on_delete=models.CASCADE) status = models.ForeignKey(Status, on_delete=models.CASCADE) transient = models.ForeignKey(Transient, on_delete=models.CASCADE) + user_warning = models.BooleanField(default=False) last_modified = models.DateTimeField(blank=True, null=True) last_processing_time_seconds = models.FloatField(blank=True, null=True) @@ -248,6 +261,7 @@ class Filter(models.Model): wavelength_max_angstrom = models.FloatField() vega_zero_point_jansky = models.FloatField() magnitude_zero_point = models.FloatField(null=True, blank=True) + ab_offset = models.FloatField(null=True, blank=True) magnitude_zero_point_keyword = models.CharField(null=True, blank=True, max_length=8) image_pixel_units = models.CharField(max_length=50, null=True, blank=True) @@ -263,11 +277,11 @@ def transmission_curve(self): curve_name = f"{settings.TRANSMISSION_CURVES_ROOT}/{self.name}.txt" try: - transmission_curve = pd.read_csv( - curve_name, delim_whitespace=True, header=None + transmission_curve = pd.read_csv(curve_name, sep="\s+", header=None) + except Exception as e: + raise ValueError( + f"{self.name}: Problem loading filter transmission curve from {curve_name}" ) - except: - raise ValueError(f"{self.name}: Problem loading filter transmission curve") wavelength = transmission_curve[0].to_numpy() transmission = transmission_curve[1].to_numpy() @@ -275,6 +289,27 @@ def transmission_curve(self): kname=self.name, nick=self.name, data=(wavelength, transmission) ) + def correlation_model(self): + """ + Returns the model for correlated errors of the filter, if it exists + """ + corr_model_name = ( + f"{settings.TRANSMISSION_CURVES_ROOT}/{self.name}_corrmodel.txt" + ) + if not os.path.exists(corr_model_name): + return None, None + + try: + corr_model = pd.read_csv(corr_model_name, sep="\s+", header=None) + except: + raise ValueError( + f"{self.name}: Problem loading filter transmission curve from {curve_name}" + ) + + app_radius = corr_model[0].to_numpy() + error_adjust = corr_model[1].to_numpy() ** (1 / 2.0) + return app_radius, error_adjust + class Catalog(models.Model): """ @@ -313,6 +348,27 @@ def hdf5_file_path(instance): return f"{instance.transient.name}/{instance.transient.name}_{instance.aperture.type}.h5" +def npz_chains_file_path(instance): + """ + Constructs a file path for a npz file + """ + return f"{instance.transient.name}/{instance.transient.name}_{instance.aperture.type}_chain.npz" + + +def npz_percentiles_file_path(instance): + """ + Constructs a file path for a npz file + """ + return f"{instance.transient.name}/{instance.transient.name}_{instance.aperture.type}_perc.npz" + + +def npz_model_file_path(instance): + """ + Constructs a file path for a npz file + """ + return f"{instance.transient.name}/{instance.transient.name}_{instance.aperture.type}_modeldata.npz" + + class Cutout(models.Model): """ Model to represent a cutout image of a host galaxy @@ -383,7 +439,7 @@ class AperturePhotometry(models.Model): flux_error = models.FloatField(blank=True, null=True) magnitude = models.FloatField(blank=True, null=True) magnitude_error = models.FloatField(blank=True, null=True) - is_validated = models.BooleanField(blank=True, null=True) + is_validated = models.CharField(blank=True, null=True, max_length=40) @property def flux_rounded(self): @@ -408,6 +464,10 @@ class SEDFittingResult(models.Model): log_mass_50 = models.FloatField(null=True, blank=True) log_mass_84 = models.FloatField(null=True, blank=True) + # from Prospector, we need to save the ratio of + # surviving stellar mass to the formed mass + mass_surviving_ratio = models.FloatField(null=True, blank=True) + log_sfr_16 = models.FloatField(null=True, blank=True) log_sfr_50 = models.FloatField(null=True, blank=True) log_sfr_84 = models.FloatField(null=True, blank=True) @@ -426,6 +486,14 @@ class SEDFittingResult(models.Model): log_tau_50 = models.FloatField(null=True, blank=True) log_tau_84 = models.FloatField(null=True, blank=True) + chains_file = models.FileField( + upload_to=npz_chains_file_path, null=True, blank=True + ) + percentiles_file = models.FileField( + upload_to=npz_percentiles_file_path, null=True, blank=True + ) + model_file = models.FileField(upload_to=npz_model_file_path, null=True, blank=True) + class TaskRegisterSnapshot(models.Model): """ diff --git a/app/host/plotting_utils.py b/app/host/plotting_utils.py index bdc25866..69768c67 100644 --- a/app/host/plotting_utils.py +++ b/app/host/plotting_utils.py @@ -1,9 +1,13 @@ +import math +import os from math import pi +import extinction import numpy as np import pandas as pd import prospect.io.read_results as reader from astropy.coordinates import SkyCoord +from astropy.cosmology import WMAP9 as cosmo from astropy.io import fits from astropy.visualization import AsinhStretch from astropy.visualization import PercentileInterval @@ -15,12 +19,16 @@ from bokeh.models import Cross from bokeh.models import Ellipse from bokeh.models import Grid +from bokeh.models import HoverTool from bokeh.models import LabelSet +from bokeh.models import Legend from bokeh.models import LinearAxis from bokeh.models import LogColorMapper from bokeh.models import Plot +from bokeh.models import Range1d from bokeh.models import Scatter from bokeh.palettes import Category20 +from bokeh.plotting import ColumnDataSource from bokeh.plotting import figure from bokeh.plotting import show from bokeh.transform import cumsum @@ -28,7 +36,9 @@ from host.host_utils import survey_list from host.models import Filter from host.photometric_calibration import maggies_to_mJy -from host.prospector import build_obs # , build_model +from host.photometric_calibration import mJy_to_maggies +from host.prospector import build_model +from host.prospector import build_obs from .models import Aperture @@ -43,20 +53,20 @@ def scale_image(image_data): def plot_image(image_data, figure): - image_data = np.nan_to_num(image_data, nan=np.nanmin(image_data)) - # image_data = np.nan_to_num(image_data, nan=np.median(image_data)) #np.amin(image_data)) + # sometimes low image mins mess up the plotting + perc01 = np.nanpercentile(image_data, 1) + + image_data = np.nan_to_num(image_data, nan=perc01) image_data = image_data + abs(np.amin(image_data)) + 0.1 scaled_image = scale_image(image_data) figure.image(image=[scaled_image]) - # color_mapper = LogColorMapper(palette="Greys256") figure.image( image=[scaled_image], x=0, y=0, - dw=len(image_data), - dh=len(image_data), - # color_mapper=color_mapper, + dw=np.shape(image_data)[1], + dh=np.shape(image_data)[0], level="image", ) @@ -84,6 +94,7 @@ def plot_aperture(figure, aperture, wcs, plotting_kwargs=None): "angle": theta_rad, "fill_color": "#cab2d6", "fill_alpha": 0.1, + "line_width": 4, } plot_dict = {**plot_dict, **plotting_kwargs} @@ -121,23 +132,23 @@ def plot_cutout_image( ): title = cutout.filter if cutout is not None else "No cutout selected" - fig = figure( - title=f"{title}", - x_axis_label="", - y_axis_label="", - plot_width=700, - plot_height=700, - ) - - fig.axis.visible = False - fig.xgrid.visible = False - fig.ygrid.visible = False if cutout is not None: with fits.open(cutout.fits.name) as fits_file: image_data = fits_file[0].data wcs = WCS(fits_file[0].header) + fig = figure( + title=f"{title}", + x_axis_label="", + y_axis_label="", + plot_width=700, + plot_height=int(np.shape(image_data)[0] / np.shape(image_data)[1] * 700), + ) + fig.axis.visible = False + fig.xgrid.visible = False + fig.ygrid.visible = False + transient_kwargs = { "legend_label": f"{transient.name}", "size": 30, @@ -185,28 +196,53 @@ def plot_cutout_image( else: image_data = np.zeros((500, 500)) + fig = figure( + title=f"{title}", + x_axis_label="", + y_axis_label="", + plot_width=700, + plot_height=700, + ) + fig.axis.visible = False + fig.xgrid.visible = False + fig.ygrid.visible = False + plot_image(image_data, fig) script, div = components(fig) return {"bokeh_cutout_script": script, "bokeh_cutout_div": div} -def plot_sed(aperture_photometry=None, sed_results_file=None, type=""): +def plot_sed(transient=None, sed_results_file=None, type=""): """ Plot SED from aperture photometry. """ - if aperture_photometry.exists(): - - flux = [measurement.flux for measurement in aperture_photometry] - flux_error = [measurement.flux_error for measurement in aperture_photometry] - wavelength = [ - measurement.filter.wavelength_eff_angstrom - for measurement in aperture_photometry - ] - else: - flux, flux_error, wavelength = [], [], [] - - flux_error = [0.0 if error is None else error for error in flux_error] + try: + obs = build_obs(transient, type, use_mag_offset=False) + except ValueError: + obs = {"filters": [], "maggies": [], "maggies_unc": []} + except AssertionError: + obs = {"filters": [], "maggies": [], "maggies_unc": []} + + def maggies_to_asinh(x): + """asinh magnitudes""" + a = 2.50 * np.log10(np.e) + mu = 35.0 + return -a * math.asinh((x / 2.0) * np.exp(mu / a)) + mu + + def asinh_to_maggies(x): + mu = 35.0 + a = 2.50 * np.log10(np.e) + return np.array([2 * math.sinh((mu - x1) / a) * np.exp(-mu / a) for x1 in x]) + + flux, flux_error, wavelength, filters, mag, mag_error = [], [], [], [], [], [] + for fl, f, fe in zip(obs["filters"], obs["maggies"], obs["maggies_unc"]): + wavelength += [fl.wave_effective] + flux += [maggies_to_mJy(f)] + flux_error += [maggies_to_mJy(fe)] + filters += [fl.name] + mag += [-2.5 * np.log10(f)] + mag_error += [1.086 * fe / f] fig = figure( title="", @@ -218,16 +254,98 @@ def plot_sed(aperture_photometry=None, sed_results_file=None, type=""): x_axis_label="Wavelength [Angstrom]", y_axis_label="Flux", ) + if len(flux): + fig.y_range = Range1d(-0.05 * np.max(flux), 1.5 * np.max(flux)) + fig.x_range = Range1d(np.min(wavelength) * 0.5, np.max(wavelength) * 1.5) + + source = ColumnDataSource( + data=dict( + x=wavelength, + y=flux, + flux_error=flux_error, + filters=filters, + mag=mag, + mag_error=mag_error, + ) + ) - fig = plot_errorbar(fig, wavelength, flux, yerr=flux_error) + fig, p = plot_errorbar( + fig, + wavelength, + flux, + yerr=flux_error, + point_kwargs={"size": 10, "legend_label": "data"}, + error_kwargs={"width": 2}, + source=source, + ) - if sed_results_file is not None: - print(sed_results_file) - result, obs, model = reader.results_from(sed_results_file, dangerous=False) + # mouse-over for data + TOOLTIPS = [ + ("flux (mJy)", "$y"), + ("flux error (mJy)", "@flux_error"), + ("wavelength", "$x"), + ("band", "@filters"), + ("mag (AB)", "@mag"), + ("mag error (AB)", "@mag_error"), + ] + hover = HoverTool(renderers=[p], tooltips=TOOLTIPS, toggleable=False) + fig.add_tools(hover) + + # second check on SED file + # long-term shouldn't be necessary, just a result of debugging + if sed_results_file is not None and os.path.exists( + sed_results_file.replace(".h5", "_modeldata.npz") + ): + result, obs, _ = reader.results_from(sed_results_file, dangerous=False) + model_data = np.load( + sed_results_file.replace(".h5", "_modeldata.npz"), allow_pickle=True + ) - best = result["bestfit"] - a = result["obs"]["redshift"] + 1 - fig.line(a * best["restframe_wavelengths"], maggies_to_mJy(best["spectrum"])) + # best = result["bestfit"] + if transient.best_redshift < 0.015: + a = result["obs"]["redshift"] - 0.015 + 1 + mag_off = ( + cosmo.distmod(result["obs"]["redshift"]).value + - cosmo.distmod(result["obs"]["redshift"] - 0.015).value + ) + print(f"mag off: {mag_off}") + fig.line( + a * model_data["rest_wavelength"], + maggies_to_mJy(model_data["spec"]) * 10 ** (0.4 * mag_off), + legend_label="average model", + ) + fig.line( + a * model_data["rest_wavelength"], + maggies_to_mJy(model_data["spec_16"]) * 10 ** (0.4 * mag_off), + line_dash="dashed", + legend_label="68% CI", + ) + fig.line( + a * model_data["rest_wavelength"], + maggies_to_mJy(model_data["spec_84"]) * 10 ** (0.4 * mag_off), + line_dash="dashed", + ) + else: + mag_off = 0 + a = result["obs"]["redshift"] + 1 + fig.line( + a * model_data["rest_wavelength"], + maggies_to_mJy(model_data["spec"]), + legend_label="average model", + ) + fig.line( + a * model_data["rest_wavelength"], + maggies_to_mJy(model_data["spec_16"]), + line_dash="dashed", + legend_label="68% CI", + ) + fig.line( + a * model_data["rest_wavelength"], + maggies_to_mJy(model_data["spec_84"]), + line_dash="dashed", + ) + + # pre-SBI++ version: fig.line(a * best["restframe_wavelengths"], maggies_to_mJy(best["spectrum"])) if obs["filters"] is not None: try: pwave = [ @@ -236,27 +354,41 @@ def plot_sed(aperture_photometry=None, sed_results_file=None, type=""): ] except: pwave = [f.wave_effective for f in obs["filters"]] - fig.circle(pwave, maggies_to_mJy(best["photometry"])) - - # xaxis = LinearAxis() - # figure.add_layout(xaxis, 'below') - # yaxis = LinearAxis() - # fig.add_layout(yaxis, 'left') + if transient.best_redshift < 0.015: + fig.circle( + pwave, + maggies_to_mJy(model_data["phot"]) * 10 ** (0.4 * mag_off), + size=10, + ) + else: + fig.circle(pwave, maggies_to_mJy(model_data["phot"]), size=10) - # fig.add_layout(Grid(dimension=0, ticker=xaxis.ticker)) - # fig.add_layout(Grid(dimension=1, ticker=yaxis.ticker)) + fig.legend.location = "top_left" script, div = components(fig) return {f"bokeh_sed_{type}_script": script, f"bokeh_sed_{type}_div": div} def plot_errorbar( - figure, x, y, xerr=None, yerr=None, color="red", point_kwargs={}, error_kwargs={} + figure, + x, + y, + xerr=None, + yerr=None, + color="red", + point_kwargs={}, + error_kwargs={}, + source=None, ): """ Plot data points with error bars on a bokeh plot """ - figure.circle(x, y, color=color, **point_kwargs) + + # to do the mouse-over + if source is not None: + p = figure.circle("x", "y", color=color, source=source, **point_kwargs) + else: + p = figure.circle(x, y, color=color, source=source, **point_kwargs) if xerr: x_err_x = [] @@ -273,7 +405,7 @@ def plot_errorbar( y_err_x.append((px, px)) y_err_y.append((py - err, py + err)) figure.multi_line(y_err_x, y_err_y, color=color, **error_kwargs) - return figure + return figure, p def plot_pie_chart(data_dict): @@ -307,13 +439,6 @@ def plot_pie_chart(data_dict): source=data, ) - # data["value"] = data['value'].astype(str) - # data["value"] = data["value"].str.pad(35, side="left") - # source = ColumnDataSource(data) - # labels = LabelSet(x=0, y=1, text='value', - # angle=cumsum('angle', include_zero=True), source=source, render_mode='canvas') - # p.add_layout(labels) - p.axis.axis_label = None p.axis.visible = False p.grid.grid_line_color = None diff --git a/app/host/postprocess_prosp.py b/app/host/postprocess_prosp.py new file mode 100644 index 00000000..61bea886 --- /dev/null +++ b/app/host/postprocess_prosp.py @@ -0,0 +1,368 @@ +"""Transform from prospector outputs to physical parameters. +Bingjie Wang 7/13/23 + +##### +example: +mod_fsps = build_model() # the same as the function I wrote in train_sbi.py +sps = build_sps() # the same as in train_sbi.py +run_all(fname='test.h5', mod_fsps=mod_fsps, sps=sps, percents=[15.9,50,84.1]) + +To use this script on SBI posteriors, replace that block that reads in a Prospector .h5 file in run_all(). res['theta_index'] should be kept the same, and res['chain'] should be replaced with the SBI posteriors. + +If you train the SBI on the stellar mass, instead the total mass formed (the default from prospector), then there is no need to run the block that does this transformation in run_all(). +""" +import os +import random +import sys + +import numpy as np +import prospect.io.read_results as reader +from astropy.cosmology import WMAP9 as cosmo +from astropy.io import fits +from astropy.table import Table +from dynesty.utils import resample_equal +from prospect.models.transforms import logsfr_ratios_to_masses +from prospect.plotting.corner import quantile + + +def theta_index(prior="p-alpha"): + """Corresponding to the Prospector-alpha model""" + # index = {'zred': slice(0, 1, None), 'logmass': slice(1, 2, None), 'logzsol': slice(2, 3, None), 'logsfr_ratios': slice(3, 9, None), + # 'dust2': slice(9, 10, None), 'dust_index': slice(10, 11, None), 'dust1_fraction': slice(11, 12, None), + # 'log_fagn': slice(12, 13, None), 'log_agn_tau': slice(13, 14, None), 'gas_logz': slice(14, 15, None), + # 'duste_qpah': slice(15, 16, None), 'duste_umin': slice(16, 17, None), 'log_duste_gamma': slice(17, 18, None)} + index = { + "zred": slice(0, 1, None), + "logmass": slice(1, 2, None), + "logzsol": slice(2, 3, None), + "logsfr_ratios": slice(3, 9, None), + "dust2": slice(9, 10, None), + "dust_index": slice(10, 11, None), + "dust1_fraction": slice(11, 12, None), + "log_fagn": slice(12, 13, None), + "log_agn_tau": slice(13, 14, None), + "gas_logz": slice(14, 15, None), + "duste_qpah": slice(15, 16, None), + "duste_umin": slice(16, 17, None), + "log_duste_gamma": slice(17, 18, None), + } + + return index + + +def getPercentiles( + chain, quantity="zred", theta_index=None, percents=[15.9, 50.0, 84.1] +): + """get the 16/50/84th percentile for a scalar output + that does not need transform functions + (e.g., mass, dust, etc). + """ + try: + npix = chain[theta_index[quantity]].shape[0] + except ValueError: + print('"' + quantity + '" does not exist in the output.') + return + + p = np.percentile(chain[:, theta_index[quantity]], q=percents) + return p.T + + +def z_to_agebins(zred=None, agebins=None, nbins_sfh=7, amin=7.1295, **extras): + """new agebins defined in Wang+2023:uncover_sps_catalog""" + tuniv = cosmo.age(zred).value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + + +def stepInterp(ab, val, ts): + """ab: agebins vector + val: the original value (sfr, etc) that we want to interpolate + ts: new values we want to interpolate to""" + newval = np.zeros_like(ts) + np.nan + for i in range(0, len(ab)): + newval[(ts >= ab[i, 0]) & (ts < ab[i, 1])] = val[i] + newval[-1] = 0 + return newval + + +def get_mwa(agebins, sfr): + ages = 10**agebins + dt = np.abs(ages[:, 1] - ages[:, 0]) + return np.sum(np.mean(ages, axis=1) * sfr * dt) / np.sum(sfr * dt) / 1e9 # in Gyr + + +def getSFH( + chain, + nagebins=7, + sfrtimes=[10, 30, 100], + tbins=100, + theta_index=None, + rtn_chains=False, + percents=[15.9, 50, 84.1], + zred=None, +): + """get the 16/50/84th percentile of the SFH for each pixel. + + Parameters + ___________ + + chain : chain object as defined in pirate.io + nagebins : number of agebins + sfrtimes : timescales that we want the output SFR averaged over (in Myr) + tbins : how many log-scaled timebins to interpolate results onto + + Returns + __________ + + age_interp : list of ages -- lookback time in Gyr + sfh : 3 x npix x len(age_interp) array that gives the 16/50/84% interval of the SFH at each lookback time + mwa : 3 x npix array-- 16/50/84th percentile mass-weighted age + sfr : 3 x npix x n_sfrtimes -- 16/50/84% SFR over each timescale + """ + + # run the transforms to get the SFH at each step in the chain for every pixel: + # (pixel, agebin, chain step) + nsamples = chain.shape[0] + allsfhs = np.zeros((nsamples, nagebins)) + allagebins = np.zeros((nsamples, nagebins, 2)) + allMWA = np.zeros((nsamples)) + + # make the SFR for each draw in each pixel + for iteration in range(nsamples): + # get agebins and time spacing + allagebins[iteration, :, :] = z_to_agebins( + zred=zred + ) # chain[:,theta_index['zred']][iteration][0]) + dt = 10 ** allagebins[iteration, :, 1] - 10 ** allagebins[iteration, :, 0] + + # get mass per bin + masses = logsfr_ratios_to_masses( + logsfr_ratios=chain[:, theta_index["logsfr_ratios"]][iteration, :], + agebins=allagebins[iteration, :, :], + logmass=chain[:, theta_index["logmass"]][iteration][0], + ) + + # convert to sfr + allsfhs[iteration, :] = masses / dt + + # go ahead and get the mass-weighted age too + allMWA[iteration] = get_mwa(allagebins[iteration, :, :], allsfhs[iteration, :]) + + # interpolate everything onto the same time grid + # define the grid as going from lookback time = 0 to the oldest time in all the samples + # with 1000 log-spaced samples in between (might want to update this later!) + allagebins_ago = 10**allagebins / 1e9 + age_interp = ( + np.logspace(1, np.log10(np.max(allagebins_ago * 1e9)), tbins) / 1e9 + ) # lookback time in Gyr + allsfhs_interp = np.zeros((nsamples, len(age_interp))) + allsfrs = np.zeros((nsamples, len(sfrtimes))) + dt = (age_interp - np.insert(age_interp, 0, 0)[:-1]) * 1e9 # in *years* to calc sfr + for iteration in range(nsamples): + allsfhs_interp[iteration, :] = stepInterp( + allagebins_ago[iteration, :], allsfhs[iteration, :], age_interp + ) + + # also get SFR averaged over all the timescales we want + for i, time in enumerate(sfrtimes): + allsfrs[iteration, i] = np.mean( + allsfhs_interp[iteration, (age_interp <= time * 1e-9)] + ) + + if rtn_chains: + return (age_interp, allsfhs_interp, allMWA, allsfrs) + else: + # sfr and MWA percentiles + sfh = np.percentile(allsfhs_interp, percents, axis=0) + mwa = np.percentile(allMWA, percents) + sfr = np.percentile(allsfrs, percents, axis=0) + return (age_interp, sfh.T, mwa, sfr.T) + + +def get_all_outputs_and_chains( + res=None, + keys=None, + run_params=None, + percents=[15.9, 50, 84.1], + nsamp=1000, + zred=None, +): + """get all the outputs; + nsamp: number of posterior samples drawn + """ + + # load the output file and get the unweighted chain + chain = res["chain"] + theta_index = res["theta_index"] + + # get the basic quantities + percentiles = {} + for key in keys: + percentiles[key] = getPercentiles(chain, key, theta_index) + + ### + train_pars = np.array( + [ + 0.11929193, + 10.39623445, + -1.32188065, + 0.18913014, + 0.16209094, + 0.06441716, + -0.19440235, + 0.11935996, + 0.37152376, + 0.34494525, + -0.42501956, + 0.51024255, + -2.06271797, + 1.37400889, + -1.21739342, + 3.32965967, + 1.28220919, + -1.79931691, + ] + ) + age_interp, allsfhs_interp, allMWA, allsfrs = getSFH( + chain, theta_index=theta_index, rtn_chains=True, zred=zred + ) + # import pdb; pdb.set_trace() + # sfr and MWA percentiles + # rtn_chains is defaulted to False: so need to transpose sfh and sfr + allsfhs_interp[np.isnan(allsfhs_interp)] = 0 + sfh = np.percentile(allsfhs_interp, percents, axis=0) + mwa = np.percentile(allMWA, percents) + sfr = np.percentile(allsfrs, percents, axis=0) + + # each of these keys is a (xpix x ypix x nparam x 16/50/84%) map + percentiles["age_interp"] = age_interp + percentiles["sfh"] = sfh.T + percentiles["mwa"] = mwa + percentiles["sfr"] = sfr.T + + # saved chains are subsampled, so that we can plot stellar mass on the corner plot + chain_len = res["chain"].shape[0] + if nsamp > chain_len: + sub_idx = random.sample(range(res["chain"].shape[0]), chain_len) + else: + sub_idx = random.sample(range(res["chain"].shape[0]), nsamp) + + chain = res["chain"][sub_idx] + chains = { + "age_interp": age_interp, + "sfh": allsfhs_interp, + "mwa": allMWA[sub_idx], + "sfr": allsfrs[sub_idx, :], + } + + for _k in keys: + chains[_k] = np.concatenate(chain[:, theta_index[_k]]) + + return percentiles, chains, sub_idx + + +def run_all( + fname, + unw_fname, + perc_fname, + zred, + prior="p-alpha", + mod_fsps=None, + sps=None, + percents=[15.9, 50, 84.1], + use_weights=True, + obs=None, + **extra +): + + # XXX read in prospector outputs + if obs is None: + res, obs, _ = reader.results_from(fname, dangerous=False) + else: + res, _, _ = reader.results_from(fname, dangerous=False) + + res["theta_index"] = theta_index(prior) + # If sampling using dynesty, we resample the chains so that each has an equal weight. + if use_weights: + res["chain"] = resample_equal(res["chain"], res["weights"]) # unweighted_chain + + """ + I think the chains output by SBI are the same as the Prospector ones, unless you change the paramters constituting the training set. + If so, then you could also use this same script to postprocess the SBI posteriors -- just replace the res['chain'] with the SBI posteriors. + """ + + # zred_idx = 0 + mass_idx = 1 + # scalar outputs that do not need transform functions + keys = [ + "logzsol", + "dust2", + "dust_index", + "dust1_fraction", + "log_fagn", + "log_agn_tau", + "gas_logz", + "duste_qpah", + "duste_umin", + "log_duste_gamma", + ] + percentiles, chains, sub_idx = get_all_outputs_and_chains(res, keys=keys, zred=zred) + + # ---------- total mass formed -> stellar mass + stellarmass = [] + ssfr = [] + modphots_all = [] + modspecs_all = [] + + for i, _subidx in enumerate(sub_idx): + modspec, modmags, sm = mod_fsps.predict( + res["chain"][int(_subidx)], sps=sps, obs=obs + ) + modphots_all.append(modmags) # model photometry + modspecs_all.append(modspec) # model spectrum + _mass = res["chain"][int(_subidx)][mass_idx] + + stellarmass.append( + _mass + ) ## mass is already converted to surviving mass in training + + ssfr.append( + chains["sfr"][i] / 10 ** stellarmass[-1] + ) # sfr chains are already sub-sampled + stellarmass = np.array(stellarmass) + ssfr = np.array(ssfr) + modphots_all = np.array(modphots_all) + + percentiles["stellar_mass"] = np.percentile(stellarmass, percents) + percentiles["ssfr"] = np.percentile(ssfr, percents, axis=0).T + percentiles["modphot"] = np.percentile(modphots_all, percents, axis=0).T + percentiles["modspec"] = np.percentile(modspecs_all, percents, axis=0).T + + # XXX save percentiles to files + # perc_fname = fname.replace('mcmc', 'perc') + # perc_fname = perc_fname.replace('.h5', '.npz') + np.savez(perc_fname, percentiles=percentiles, theta_lbs=res["theta_labels"]) + + # XXX save chains to files + # unw_fname = fname.replace('mcmc', 'chain') + # unw_fname = unw_fname.replace('.h5', '.npz') + chains["stellar_mass"] = stellarmass + chains["ssfr"] = ssfr + np.savez(unw_fname, chains=chains) + + # load as + # fnpz = np.load(perc_fname, allow_pickle=True) + # perc = fnpz['percentiles'][()] diff --git a/app/host/prospector.py b/app/host/prospector.py index 5a69d518..1050d041 100644 --- a/app/host/prospector.py +++ b/app/host/prospector.py @@ -1,16 +1,31 @@ # Utils and wrappers for the prospector SED fitting code +import copy +import json import os +import time import extinction +import h5py import numpy as np import prospect.io.read_results as reader +from astropy.cosmology import WMAP9 as cosmo from django.conf import settings +from django.db.models import Q +from host import postprocess_prosp as pp +from host.SBI.run_sbi_blast import fit_sbi_pp from prospect.fitting import fit_model as fit_model_prospect from prospect.fitting import lnprobfn from prospect.io import write_results as writer +from prospect.io.write_results import write_h5_header +from prospect.io.write_results import write_obs_to_h5 +from prospect.models import priors from prospect.models import SpecModel +from prospect.models.sedmodel import PolySpecModel from prospect.models.templates import TemplateLibrary +from prospect.models.transforms import logsfr_ratios_to_sfrs +from prospect.models.transforms import zred_to_agebins from prospect.sources import CSPSpecBasis +from prospect.sources import FastStepBasis from prospect.utils.obsutils import fix_obs from scipy.special import gamma from scipy.special import gammainc @@ -21,6 +36,28 @@ from .models import hdf5_file_path from .photometric_calibration import mJy_to_maggies ##jansky_to_maggies +# add redshift scaling to agebins, such that +# t_max = t_univ +def zred_to_agebins(zred=None, **extras): + amin = 7.1295 + nbins_sfh = 7 + tuniv = cosmo.age(zred)[0].value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + def get_CI(chain): chainlen = len(chain) @@ -44,7 +81,7 @@ def psi_from_sfh(mass, tage, tau): ) -def build_obs(transient, aperture_type): +def build_obs(transient, aperture_type, use_mag_offset=True): """ This functions is required by prospector and should return @@ -54,8 +91,8 @@ def build_obs(transient, aperture_type): """ photometry = AperturePhotometry.objects.filter( - transient=transient, aperture__type__exact=aperture_type, is_validated=True - ) + transient=transient, aperture__type__exact=aperture_type + ).filter(Q(is_validated="true") | Q(is_validated="contamination warning")) if not photometry.exists(): raise ValueError(f"No host photometry of type {aperture_type}") @@ -64,10 +101,16 @@ def build_obs(transient, aperture_type): raise ValueError("No host galaxy match") z = transient.best_redshift + if z is not None and z < 0.015 and use_mag_offset: + mag_offset = cosmo.distmod(z + 0.015).value - cosmo.distmod(z).value + z += 0.015 + else: + mag_offset = 0 filters, flux_maggies, flux_maggies_error = [], [], [] for filter in Filter.objects.all(): + # if 'PanSTARRS' in filter.name or 'DES' in filter.name: continue try: datapoint = photometry.get(filter=filter) except AperturePhotometry.DoesNotExist: @@ -76,6 +119,9 @@ def build_obs(transient, aperture_type): except AperturePhotometry.MultipleObjectsReturned: raise + if datapoint.flux is None: + continue + # correct for MW reddening if aperture_type == "global": mwebv = transient.host.milkyway_dust_reddening @@ -95,11 +141,29 @@ def build_obs(transient, aperture_type): ext_corr = extinction.fitzpatrick99(np.array([wave_eff]), mwebv * 3.1, r_v=3.1)[ 0 ] - flux_mwcorr = datapoint.flux * 10 ** (0.4 * ext_corr) + flux_mwcorr = ( + datapoint.flux * 10 ** (-0.4 * filter.ab_offset) * 10 ** (0.4 * ext_corr) + ) + # 1% error floor + fluxerr_mwcorr = np.sqrt( + ( + datapoint.flux_error + * 10 ** (-0.4 * filter.ab_offset) + * 10 ** (0.4 * ext_corr) + ) + ** 2.0 + + (0.01 * flux_mwcorr) ** 2.0 + ) + + # TEST - are low-S/N observations messing up prospector? + if flux_mwcorr / fluxerr_mwcorr < 3: + continue filters.append(filter.transmission_curve()) - flux_maggies.append(mJy_to_maggies(flux_mwcorr)) - flux_maggies_error.append(mJy_to_maggies(datapoint.flux_error)) + flux_maggies.append(mJy_to_maggies(flux_mwcorr * 10 ** (-0.4 * mag_offset))) + flux_maggies_error.append( + mJy_to_maggies(fluxerr_mwcorr * 10 ** (-0.4 * mag_offset)) + ) obs_data = dict( wavelength=None, @@ -114,35 +178,580 @@ def build_obs(transient, aperture_type): return fix_obs(obs_data) +def build_model_nonparam(obs=None, **extras): + """prospector-alpha""" + fit_order = [ + "zred", + "logmass", + "logzsol", + "logsfr_ratios", + "dust2", + "dust_index", + "dust1_fraction", + "log_fagn", + "log_agn_tau", + "gas_logz", + "duste_qpah", + "duste_umin", + "log_duste_gamma", + ] + + # ------------- + # MODEL_PARAMS + model_params = {} + + # --- BASIC PARAMETERS --- + model_params["zred"] = { + "N": 1, + "isfree": True, + "init": 0.5, + "prior": priors.FastUniform(a=0, b=0.2), + } + + model_params["logmass"] = { + "N": 1, + "isfree": True, + "init": 8.0, + "units": "Msun", + "prior": priors.FastUniform(a=7.0, b=12.5), + } + + model_params["logzsol"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"$\log (Z/Z_\odot)$", + "prior": priors.FastUniform(a=-1.98, b=0.19), + } + + model_params["imf_type"] = { + "N": 1, + "isfree": False, + "init": 1, # 1 = chabrier + "units": None, + "prior": None, + } + model_params["add_igm_absorption"] = {"N": 1, "isfree": False, "init": True} + model_params["add_agb_dust_model"] = {"N": 1, "isfree": False, "init": True} + model_params["pmetals"] = {"N": 1, "isfree": False, "init": -99} + + # --- SFH --- + nbins_sfh = 7 + model_params["sfh"] = {"N": 1, "isfree": False, "init": 3} + model_params["logsfr_ratios"] = { + "N": 6, + "isfree": True, + "init": 0.0, + "prior": priors.FastTruncatedEvenStudentTFreeDeg2( + hw=np.ones(6) * 5.0, sig=np.ones(6) * 0.3 + ), + } + + # add redshift scaling to agebins, such that + # t_max = t_univ + def zred_to_agebins(zred=None, **extras): + amin = 7.1295 + nbins_sfh = 7 + tuniv = cosmo.age(zred)[0].value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + + def logsfr_ratios_to_masses( + logmass=None, logsfr_ratios=None, agebins=None, **extras + ): + """This converts from an array of log_10(SFR_j / SFR_{j+1}) and a value of + log10(\Sum_i M_i) to values of M_i. j=0 is the most recent bin in lookback + time. + """ + nbins = agebins.shape[0] + sratios = 10 ** np.clip(logsfr_ratios, -100, 100) + dt = 10 ** agebins[:, 1] - 10 ** agebins[:, 0] + coeffs = np.array( + [ + (1.0 / np.prod(sratios[:i])) + * (np.prod(dt[1 : i + 1]) / np.prod(dt[:i])) + for i in range(nbins) + ] + ) + m1 = (10**logmass) / coeffs.sum() + + return m1 * coeffs + + model_params["mass"] = { + "N": 7, + "isfree": False, + "init": 1e6, + "units": r"M$_\odot$", + "depends_on": logsfr_ratios_to_masses, + } + + model_params["agebins"] = { + "N": 7, + "isfree": False, + "init": zred_to_agebins(np.atleast_1d(0.5)), + "prior": None, + "depends_on": zred_to_agebins, + } + + # --- Dust Absorption --- + model_params["dust_type"] = { + "N": 1, + "isfree": False, + "init": 4, + "units": "FSPS index", + } + model_params["dust1_fraction"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.0, b=2.0, mu=1.0, sig=0.3), + } + + model_params["dust2"] = { + "N": 1, + "isfree": True, + "init": 0.0, + "units": "", + "prior": priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0), + } + + def to_dust1(dust1_fraction=None, dust1=None, dust2=None, **extras): + return dust1_fraction * dust2 + + model_params["dust1"] = { + "N": 1, + "isfree": False, + "depends_on": to_dust1, + "init": 0.0, + "units": "optical depth towards young stars", + "prior": None, + } + model_params["dust_index"] = { + "N": 1, + "isfree": True, + "init": 0.7, + "units": "", + "prior": priors.FastUniform(a=-1.0, b=0.2), + } + + # --- Nebular Emission --- + model_params["add_neb_emission"] = {"N": 1, "isfree": False, "init": True} + model_params["add_neb_continuum"] = {"N": 1, "isfree": False, "init": True} + model_params["gas_logz"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"log Z/Z_\odot", + "prior": priors.FastUniform(a=-2.0, b=0.5), + } + model_params["gas_logu"] = { + "N": 1, + "isfree": False, + "init": -1.0, + "units": r"Q_H/N_H", + "prior": priors.FastUniform(a=-4, b=-1), + } + + # --- AGN dust --- + model_params["add_agn_dust"] = {"N": 1, "isfree": False, "init": True} + + model_params["log_fagn"] = { + "N": 1, + "isfree": True, + "init": -7.0e-5, + "prior": priors.FastUniform(a=-5.0, b=-4.9), + } + + def to_fagn(log_fagn=None, **extras): + return 10**log_fagn + + model_params["fagn"] = {"N": 1, "isfree": False, "init": 0, "depends_on": to_fagn} + + model_params["log_agn_tau"] = { + "N": 1, + "isfree": True, + "init": np.log10(20.0), + "prior": priors.FastUniform(a=np.log10(15.0), b=np.log10(15.1)), + } + + def to_agn_tau(log_agn_tau=None, **extras): + return 10**log_agn_tau + + model_params["agn_tau"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_agn_tau, + } + + # --- Dust Emission --- + model_params["duste_qpah"] = { + "N": 1, + "isfree": True, + "init": 2.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=2.0, sig=2.0), + } + + model_params["duste_umin"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=1.0, sig=10.0), + } + + model_params["log_duste_gamma"] = { + "N": 1, + "isfree": True, + "init": -2.0, + "prior": priors.FastTruncatedNormal(a=-2.1, b=-1.9, mu=-2.0, sig=1.0), + } + + def to_duste_gamma(log_duste_gamma=None, **extras): + return 10**log_duste_gamma + + model_params["duste_gamma"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_duste_gamma, + } + + # ---- Units ---- + model_params["peraa"] = {"N": 1, "isfree": False, "init": False} + + model_params["mass_units"] = {"N": 1, "isfree": False, "init": "mformed"} + + tparams = {} + for i in fit_order: + tparams[i] = model_params[i] + for i in list(model_params.keys()): + if i not in fit_order: + tparams[i] = model_params[i] + model_params = tparams + + return PolySpecModel(model_params) + + +def build_model_nonparam_bkp(obs=None, **extras): + """prospector-alpha""" + fit_order = [ + "zred", + "logmass", + "logzsol", + "logsfr_ratios", + "dust2", + "dust_index", + "dust1_fraction", + "log_fagn", + "log_agn_tau", + "gas_logz", + "duste_qpah", + "duste_umin", + "log_duste_gamma", + ] + + # ------------- + # MODEL_PARAMS + model_params = {} + + # --- BASIC PARAMETERS --- + model_params["zred"] = { + "N": 1, + "isfree": False, + "init": obs["redshift"], + # "prior": priors.FastUniform(a=0, b=0.2), + } + + model_params["logmass"] = { + "N": 1, + "isfree": True, + "init": 8.0, + "units": "Msun", + "prior": priors.FastUniform(a=7.0, b=12.5), + } + + model_params["logzsol"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"$\log (Z/Z_\odot)$", + "prior": priors.FastUniform(a=-1.98, b=0.19), + } + + model_params["imf_type"] = { + "N": 1, + "isfree": False, + "init": 1, # 1 = chabrier + "units": None, + "prior": None, + } + model_params["add_igm_absorption"] = {"N": 1, "isfree": False, "init": True} + model_params["add_agb_dust_model"] = {"N": 1, "isfree": False, "init": True} + model_params["pmetals"] = {"N": 1, "isfree": False, "init": -99} + + # --- SFH --- + nbins_sfh = 7 + model_params["sfh"] = {"N": 1, "isfree": False, "init": 3} + model_params["logsfr_ratios"] = { + "N": 6, + "isfree": True, + "init": 0.0, + "prior": priors.FastTruncatedEvenStudentTFreeDeg2( + hw=np.ones(6) * 5.0, sig=np.ones(6) * 0.3 + ), + } + + # add redshift scaling to agebins, such that + # t_max = t_univ + def zred_to_agebins(zred=None, **extras): + amin = 7.1295 + nbins_sfh = 7 + tuniv = cosmo.age(zred)[0].value * 1e9 + tbinmax = tuniv * 0.9 + if zred <= 3.0: + agelims = ( + [0.0, 7.47712] + + np.linspace(8.0, np.log10(tbinmax), nbins_sfh - 2).tolist() + + [np.log10(tuniv)] + ) + else: + agelims = np.linspace(amin, np.log10(tbinmax), nbins_sfh).tolist() + [ + np.log10(tuniv) + ] + agelims[0] = 0 + + agebins = np.array([agelims[:-1], agelims[1:]]) + return agebins.T + + def logsfr_ratios_to_masses( + logmass=None, logsfr_ratios=None, agebins=None, **extras + ): + """This converts from an array of log_10(SFR_j / SFR_{j+1}) and a value of + log10(\Sum_i M_i) to values of M_i. j=0 is the most recent bin in lookback + time. + """ + nbins = agebins.shape[0] + sratios = 10 ** np.clip(logsfr_ratios, -100, 100) + dt = 10 ** agebins[:, 1] - 10 ** agebins[:, 0] + coeffs = np.array( + [ + (1.0 / np.prod(sratios[:i])) + * (np.prod(dt[1 : i + 1]) / np.prod(dt[:i])) + for i in range(nbins) + ] + ) + m1 = (10**logmass) / coeffs.sum() + + return m1 * coeffs + + model_params["mass"] = { + "N": 7, + "isfree": False, + "init": 1e6, + "units": r"M$_\odot$", + "depends_on": logsfr_ratios_to_masses, + } + + model_params["agebins"] = { + "N": 7, + "isfree": False, + "init": zred_to_agebins(np.atleast_1d(0.5)), + "prior": None, + "depends_on": zred_to_agebins, + } + + # --- Dust Absorption --- + model_params["dust_type"] = { + "N": 1, + "isfree": False, + "init": 4, + "units": "FSPS index", + } + model_params["dust1_fraction"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.0, b=2.0, mu=1.0, sig=0.3), + } + + model_params["dust2"] = { + "N": 1, + "isfree": True, + "init": 0.0, + "units": "", + "prior": priors.FastTruncatedNormal(a=0.0, b=4.0, mu=0.3, sig=1.0), + } + + def to_dust1(dust1_fraction=None, dust1=None, dust2=None, **extras): + return dust1_fraction * dust2 + + model_params["dust1"] = { + "N": 1, + "isfree": False, + "depends_on": to_dust1, + "init": 0.0, + "units": "optical depth towards young stars", + "prior": None, + } + model_params["dust_index"] = { + "N": 1, + "isfree": True, + "init": 0.7, + "units": "", + "prior": priors.FastUniform(a=-1.0, b=0.4), + } + + # --- Nebular Emission --- + model_params["add_neb_emission"] = {"N": 1, "isfree": False, "init": True} + model_params["add_neb_continuum"] = {"N": 1, "isfree": False, "init": True} + model_params["gas_logz"] = { + "N": 1, + "isfree": True, + "init": -0.5, + "units": r"log Z/Z_\odot", + "prior": priors.FastUniform(a=-2.0, b=0.5), + } + model_params["gas_logu"] = { + "N": 1, + "isfree": False, + "init": -1.0, + "units": r"Q_H/N_H", + "prior": priors.FastUniform(a=-4, b=-1), + } + + # --- AGN dust --- + model_params["add_agn_dust"] = {"N": 1, "isfree": False, "init": True} + + model_params["log_fagn"] = { + "N": 1, + "isfree": True, + "init": -7.0e-5, + "prior": priors.FastUniform(a=-5.0, b=-4.9), + } + + def to_fagn(log_fagn=None, **extras): + return 10**log_fagn + + model_params["fagn"] = {"N": 1, "isfree": False, "init": 0, "depends_on": to_fagn} + + model_params["log_agn_tau"] = { + "N": 1, + "isfree": True, + "init": np.log10(20.0), + "prior": priors.FastUniform(a=np.log10(15.0), b=np.log10(15.1)), + } + + def to_agn_tau(log_agn_tau=None, **extras): + return 10**log_agn_tau + + model_params["agn_tau"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_agn_tau, + } + + # --- Dust Emission --- + model_params["duste_qpah"] = { + "N": 1, + "isfree": True, + "init": 2.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=2.0, sig=2.0), + } + + model_params["duste_umin"] = { + "N": 1, + "isfree": True, + "init": 1.0, + "prior": priors.FastTruncatedNormal(a=0.9, b=1.1, mu=1.0, sig=10.0), + } + + model_params["log_duste_gamma"] = { + "N": 1, + "isfree": True, + "init": -2.0, + "prior": priors.FastTruncatedNormal(a=-2.1, b=-1.9, mu=-2.0, sig=1.0), + } + + def to_duste_gamma(log_duste_gamma=None, **extras): + return 10**log_duste_gamma + + model_params["duste_gamma"] = { + "N": 1, + "isfree": False, + "init": 0, + "depends_on": to_duste_gamma, + } + + # ---- Units ---- + model_params["peraa"] = {"N": 1, "isfree": False, "init": False} + + model_params["mass_units"] = {"N": 1, "isfree": False, "init": "mformed"} + + tparams = {} + for i in fit_order: + tparams[i] = model_params[i] + for i in list(model_params.keys()): + if i not in fit_order: + tparams[i] = model_params[i] + model_params = tparams + + return PolySpecModel(model_params) + + def build_model(observations): """ Construct all model components """ - model_params = TemplateLibrary["parametric_sfh"] - model_params.update(TemplateLibrary["nebular"]) - model_params["zred"]["init"] = observations["redshift"] - model = SpecModel(model_params) - sps = CSPSpecBasis(zcontinuous=1) + # model_params = TemplateLibrary["parametric_sfh"] + # model_params.update(TemplateLibrary["nebular"]) + # model_params["zred"]["init"] = observations["redshift"] + # model = SpecModel(model_params) + model = build_model_nonparam(observations) + + # new SPS model + # sps = CSPSpecBasis(zcontinuous=1) + sps = FastStepBasis(zcontinuous=2, compute_vega_mags=False) noise_model = (None, None) return {"model": model, "sps": sps, "noise_model": noise_model} -def fit_model(observations, model_components, fitting_kwargs): +def fit_model( + observations, model_components, fitting_kwargs, sbipp=False, fit_type="global" +): """Fit the model""" - output = fit_model_prospect( - observations, - model_components["model"], - model_components["sps"], - optimize=False, - dynesty=True, - lnprobfn=lnprobfn, - noise=model_components["noise_model"], - **fitting_kwargs, - ) - return output + if sbipp: + output, errflag = fit_sbi_pp(observations, fit_type=fit_type) + else: + output = fit_model_prospect( + observations, + model_components["model"], + model_components["sps"], + optimize=False, + dynesty=True, + lnprobfn=lnprobfn, + noise=model_components["noise_model"], + **fitting_kwargs, + ) + errflag = 0 + + return output, errflag def prospector_result_to_blast( @@ -152,68 +761,154 @@ def prospector_result_to_blast( model_components, observations, sed_output_root=settings.SED_OUTPUT_ROOT, + parametric_sfh=False, + sbipp=False, ): # write the results - hdf5_file = ( + hdf5_file_name = ( f"{sed_output_root}/{transient.name}/{transient.name}_{aperture.type}.h5" ) + hdf5_file = hdf5_file_name if not os.path.exists(f"{sed_output_root}/{transient.name}"): os.makedirs(f"{sed_output_root}/{transient.name}/") - - writer.write_hdf5( - hdf5_file, - {}, - model_components["model"], - observations, - prospector_output["sampling"][0], - None, - sps=model_components["sps"], - tsample=prospector_output["sampling"][1], - toptimize=0.0, - ) + if os.path.exists(hdf5_file): + # prospector won't overwrite, which causes problems + os.remove(hdf5_file) + + if sbipp: + + hf = h5py.File(hdf5_file_name, "w") + + sdat = hf.create_group("sampling") + sdat.create_dataset("chain", data=prospector_output["sampling"][0]["samples"]) + sdat.attrs["theta_labels"] = json.dumps( + list(model_components["model"].theta_labels()) + ) + + # High level parameter and version info + write_h5_header(hf, {}, model_components["model"]) + + # ---------------------- + # Observational data + write_obs_to_h5(hf, observations) + hf.flush() + else: + writer.write_hdf5( + hdf5_file, + {}, + model_components["model"], + observations, + prospector_output["sampling"][0], + None, + sps=model_components["sps"], + tsample=prospector_output["sampling"][1], + toptimize=0.0, + ) # load up the hdf5 file to get the results resultpars, obs, _ = reader.results_from(hdf5_file, dangerous=False) - # logmass, age, tau - logmass = np.log10( - resultpars["chain"][ - ..., np.where(np.array(resultpars["theta_labels"]) == "mass")[0][0] + # if sbipp: + # theta_max = resultpars["chain"][100] #np.mean(resultpars["chain"], axis=0) # + np.std(resultpars["chain"], axis=0) + # else: + # imax = np.argmax(resultpars["lnprobability"]) + # csz = resultpars["chain"].shape + # if resultpars["chain"].ndim > 2: + # emcee + # i, j = np.unravel_index(imax, resultpars["lnprobability"].shape) + # theta_max = resultpars["chain"][i, j, :].copy() + # else: + # dynesty + # theta_max = resultpars["chain"][imax, :].copy() + + model_init = copy.deepcopy(model_components["model"]) + tstart = time.time() + ### take the mean of 50 random samples to get the "best fit" model + # best_phot_store = np.array([]) + for i in range(50): + theta = resultpars["chain"][ + np.random.choice(np.arange(np.shape(resultpars["chain"])[0])), : ] - ) - logmass16, logmass50, logmass84 = get_CI(logmass) - age = resultpars["chain"][ - ..., np.where(np.array(resultpars["theta_labels"]) == "tage")[0][0] - ] - age16, age50, age84 = get_CI(age) - tau = resultpars["chain"][ - ..., np.where(np.array(resultpars["theta_labels"]) == "tau")[0][0] - ] - tau16, tau50, tau84 = get_CI(tau) - - # sfr, ssfr - sfr = psi_from_sfh( - resultpars["chain"][ - ..., np.where(np.array(resultpars["theta_labels"]) == "mass")[0][0] - ], - resultpars["chain"][ - ..., np.where(np.array(resultpars["theta_labels"]) == "tage")[0][0] - ], - resultpars["chain"][ + theta[0] = observations["redshift"] + if i == 0: + best_spec, best_phot, mfrac = model_components["model"].predict( + theta, obs=observations, sps=model_components["sps"] + ) + + theta[1] -= np.log10(mfrac) + ### we need our mass to be conv to total \ + ### formed mass for the model to be accurate + best_spec, best_phot, mfrac = model_init.predict( + theta, obs=observations, sps=model_components["sps"] + ) + best_phot_store = best_phot[:] + best_spec_store = best_spec[:] + else: + best_spec_single, best_phot_single, mfrac_single = model_components[ + "model" + ].predict(theta, obs=observations, sps=model_components["sps"]) + theta[1] -= np.log10(mfrac) + ### we need our mass to be conv to total \ + ### formed mass for the model to be accurate + best_spec_single, best_phot_single, mfrac_single = model_init.predict( + theta, obs=observations, sps=model_components["sps"] + ) + + # iteratively update the mean + best_spec = (best_spec * i + best_spec_single) / (i + 1) + best_phot = (best_phot * i + best_phot_single) / (i + 1) + mfrac = (mfrac * i + mfrac_single) / (i + 1) + best_phot_store = np.vstack([best_phot_store, best_phot_single]) + best_spec_store = np.vstack([best_spec_store, best_spec_single]) + best_phot = np.median(best_phot_store, axis=0) + best_spec = np.median(best_spec_store, axis=0) + phot_16, phot_84 = np.percentile(best_phot_store, [16, 84], axis=0) + spec_16, spec_84 = np.percentile(best_spec_store, [16, 84], axis=0) + tfin = time.time() + print(f"sampling chains to get best-fit model took {tfin-tstart:.0f} seconds") + + if not parametric_sfh: + use_weights = not sbipp + + pp.run_all( + hdf5_file_name, + hdf5_file_name.replace(".h5", "_chain.npz"), + hdf5_file_name.replace(".h5", "_perc.npz"), + model_components["model"].init_config["zred"]["init"], + prior="p-alpha", + mod_fsps=model_components["model"], + sps=model_components["sps"], + percents=[15.9, 50, 84.1], + use_weights=use_weights, + sbipp=sbipp, + obs=observations, + ) + + percentiles = np.load( + hdf5_file_name.replace(".h5", "_perc.npz"), allow_pickle=True + ) + perc = np.atleast_1d(percentiles["percentiles"])[0] + + logmass16, logmass50, logmass84 = perc["stellar_mass"] + age16, age50, age84 = perc["mwa"] + logsfr16, logsfr50, logsfr84 = np.log10(perc["sfr"][0]) + logssfr16, logssfr50, logssfr84 = np.log10(perc["ssfr"][0]) + + if parametric_sfh: + tau = resultpars["chain"][ ..., np.where(np.array(resultpars["theta_labels"]) == "tau")[0][0] - ], - ) - logsfr = np.log10(sfr) - logssfr = np.log10(sfr) - logmass - logsfr16, logsfr50, logsfr84 = get_CI(logsfr) - logssfr16, logssfr50, logssfr84 = get_CI(logssfr) + ] + tau16, tau50, tau84 = get_CI(tau) prosp_results = { "transient": transient, "aperture": aperture, "posterior": hdf5_file, + "chains_file": hdf5_file_name.replace(".h5", "_chain.npz"), + "percentiles_file": hdf5_file_name.replace(".h5", "_perc.npz"), + "model_file": hdf5_file_name.replace(".h5", "_modeldata.npz"), "log_mass_16": logmass16, "log_mass_50": logmass50, "log_mass_84": logmass84, @@ -226,9 +921,22 @@ def prospector_result_to_blast( "log_age_16": age16, "log_age_50": age50, "log_age_84": age84, - "log_tau_16": tau16, - "log_tau_50": tau50, - "log_tau_84": tau84, + "mass_surviving_ratio": mfrac, } + if parametric_sfh: + prosp_results["log_tau_16"] = (tau16,) + prosp_results["log_tau_50"] = (tau50,) + prosp_results["log_tau_84"] = (tau84,) + + np.savez( + hdf5_file_name.replace(".h5", "_modeldata.npz"), + rest_wavelength=model_components["sps"].wavelengths, + spec=best_spec, + phot=best_phot, + spec_16=spec_16, + spec_84=spec_84, + phot_16=phot_16, + phot_84=phot_84, + ) return prosp_results diff --git a/app/host/slurm/run_batch.py b/app/host/slurm/run_batch.py new file mode 100644 index 00000000..378b39d4 --- /dev/null +++ b/app/host/slurm/run_batch.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +import argparse +import glob +import os +import time + +import astropy.table as at + + +def main( + batchfile, + batchdir, + blastdir="/n/holystore01/LABS/berger_lab/Lab/djones01/blast/app", +): + data = at.Table.read(batchfile) + + if not os.path.exists(batchdir): + os.makedirs(batchdir) + + for d in data: + sedfiles = glob.glob(f"../../../data/sed_output/{d['transient_name']}/*global*") + if len(sedfiles): + os.system(f"rm {sedfiles[0]}") + # continue + slurmfile = f"{batchdir}/slurm_{d['transient_name']}.sh" + + with open("slurm_template.sh", "r") as fin, open(slurmfile, "w") as fout: + for line in fin: + line = ( + line.replace("\n", "") + .replace("", str(d["transient_name"])) + .replace("", str(d["transient_ra"])) + .replace("", str(d["transient_dec"])) + .replace("", batchdir) + .replace("", blastdir) + ) + if "transient_redshift" in d.keys(): + line = line.replace( + "", str(d["transient_redshift"]) + ) + else: + line = line.replace( + "export BLAST_TRANSIENT_REDSHIFT=", "" + ) + print(line, file=fout) + os.system(f"sbatch {slurmfile}") + time.sleep(60 * 30) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="batch submission for slurm systems") + parser.add_argument("batchfile", help="batch CSV with transient name,ra,dec") + parser.add_argument("batchdir", help="output directory for logs, job scripts") + args = vars(parser.parse_args()) + + main(args["batchfile"], args["batchdir"]) diff --git a/app/host/slurm/run_single_transient.py b/app/host/slurm/run_single_transient.py new file mode 100644 index 00000000..a7ab9003 --- /dev/null +++ b/app/host/slurm/run_single_transient.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# D. Jones - 4/5/23 +# django cron to be called by slurm +# should run a single transient all +# the way through. data saved to sqlite +# and we can figure out how to export it +# later +import datetime +import os + +import host.transient_tasks +from django_cron import CronJobBase +from django_cron import Schedule +from host import tasks +from host.models import Status +from host.models import Task +from host.models import TaskRegister +from host.models import Transient + +transient_name = os.environ["BLAST_TRANSIENT_NAME"] +transient_ra = os.environ["BLAST_TRANSIENT_RA"] +transient_dec = os.environ["BLAST_TRANSIENT_DEC"] +try: + transient_redshift = os.environ["BLAST_TRANSIENT_REDSHIFT"] +except Exception as e: + transient_redshift = None + +tasks_in_order = [ + "Transient MWEBV", + "Host match", + "Host MWEBV", + "Cutout download", + "Host information", + "Transient information", + "Global aperture construction", + "Global aperture photometry", + "Validate global photometry", + "Local aperture photometry", + "Validate local photometry", + "Global host SED inference", + "Local host SED inference", +] +tasks_classes_in_order = [ + host.transient_tasks.MWEBV_Transient(), + host.transient_tasks.Ghost(), + host.transient_tasks.MWEBV_Host(), + host.transient_tasks.ImageDownload(), + host.transient_tasks.HostInformation(), + host.transient_tasks.TransientInformation(), + host.transient_tasks.GlobalApertureConstruction(), + host.transient_tasks.GlobalAperturePhotometry(), + host.transient_tasks.ValidateGlobalPhotometry(), + host.transient_tasks.LocalAperturePhotometry(), + host.transient_tasks.ValidateLocalPhotometry(), + host.transient_tasks.GlobalHostSEDFitting(), + host.transient_tasks.LocalHostSEDFitting(), +] + +tasks_sed = [ + "Global host SED inference", +] +tasks_sed_classes = [ + host.transient_tasks.GlobalHostSEDFitting(), +] + + +class run_single(CronJobBase): + + RUN_EVERY_MINS = 3 + + schedule = Schedule(run_every_mins=RUN_EVERY_MINS) + code = "host.slurm.run_single_transient.run_single" + + def do(self): + + self.run() + + def run(self): + + transients = Transient.objects.filter(name=transient_name) + if not len(transients): + transient = Transient.objects.create( + name=transient_name, + ra_deg=transient_ra, + dec_deg=transient_dec, + tns_id=1, + redshift=transient_redshift, + ) + else: + transient = transients[0] + + for to, tc in zip(tasks_in_order, tasks_classes_in_order): + for t in tasks.periodic_tasks: + if t.task_name == to: + task = Task.objects.get(name__exact=t.task_name) + task_register = TaskRegister.objects.all() + task_register = task_register.filter(transient=transient, task=task) + + if not len(task_register): + task_register = TaskRegister.objects.create( + transient=transient, + task=task, + status=Status.objects.get(message="not processed"), + last_modified=datetime.datetime.now(), + last_processing_time_seconds=0, + ) + else: + task_register = task_register[0] + + # if task_register.status.message != 'processed': + task_register.status = Status.objects.get(message="not processed") + try: + status = t.run_process(task_register) + except Exception as e: + print(e) + # import pdb; pdb.set_trace() + # status = tc._run_process(transient) + # task_register.status = Status.objects.get(message=status) + # task_register.save() + raise e + + break + + +class run_single_sed(CronJobBase): + + RUN_EVERY_MINS = 3 + + schedule = Schedule(run_every_mins=RUN_EVERY_MINS) + code = "host.slurm.run_single_transient.run_single" + + def do(self): + + self.run() + + def run(self): + + transients = Transient.objects.filter(name=transient_name) + if not len(transients): + transient = Transient.objects.create( + name=transient_name, + ra_deg=transient_ra, + dec_deg=transient_dec, + tns_id=1, + redshift=transient_redshift, + ) + else: + transient = transients[0] + + for to, tc in zip(tasks_sed, tasks_sed_classes): + for t in tasks.periodic_tasks: + if t.task_name == to: + task = Task.objects.get(name__exact=t.task_name) + task_register = TaskRegister.objects.all() + task_register = task_register.filter(transient=transient, task=task) + + if not len(task_register): + task_register = TaskRegister.objects.create( + transient=transient, + task=task, + status=Status.objects.get(message="not processed"), + last_modified=datetime.datetime.now(), + last_processing_time_seconds=0, + ) + else: + task_register = task_register[0] + + # if task_register.status.message != 'processed': + task_register.status = Status.objects.get(message="not processed") + try: + status = t.run_process(task_register, save=False) + except Exception as e: + print(e) + # import pdb; pdb.set_trace() + # status = tc._run_process(transient) + # task_register.status = Status.objects.get(message=status) + # task_register.save() + raise e + + break diff --git a/app/host/slurm/slurm_template.sh b/app/host/slurm/slurm_template.sh new file mode 100644 index 00000000..dd1994b3 --- /dev/null +++ b/app/host/slurm/slurm_template.sh @@ -0,0 +1,16 @@ +#!/bin/bash -l +#SBATCH --job-name=blast_transient_ +#SBATCH --time=34:00:00 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --output=/blast_transient_.log +#SBATCH --mem=12GB +#SBATCH -p shared + +export BLAST_TRANSIENT_NAME= +export BLAST_TRANSIENT_RA= +export BLAST_TRANSIENT_DEC= + +conda activate blast +cd +python manage.py runcrons host.slurm.run_single_transient.run_single --force diff --git a/app/host/system_tasks.py b/app/host/system_tasks.py index ad34e531..23eaaaa7 100644 --- a/app/host/system_tasks.py +++ b/app/host/system_tasks.py @@ -3,10 +3,12 @@ import shutil from django.conf import settings +from django.db.models import Q from django.utils import timezone from .base_tasks import initialise_all_tasks_status from .base_tasks import SystemTaskRunner +from .models import Status from .models import TaskRegister from .models import TaskRegisterSnapshot from .models import Transient @@ -19,28 +21,56 @@ class TNSDataIngestion(SystemTaskRunner): - def run_process(self, interval_minutes=100): + def run_process(self, interval_minutes=200): + print("TNS STARTED") now = timezone.now() time_delta = datetime.timedelta(minutes=interval_minutes) tns_credentials = get_tns_credentials() recent_transients = get_transients_from_tns( now - time_delta, tns_credentials=tns_credentials ) + print("TNS DONE") saved_transients = Transient.objects.all() - + count = 0 for transient in recent_transients: try: - saved_transients.get(name__exact=transient.name) + saved_transient = saved_transients.get(name__exact=transient.name) + + # if there was *not* a redshift before and there *is* one now + # then it would be safest to reprocess everything + if not saved_transient.redshift and transient.redshift: + tasks = TaskRegister.objects.filter(transient=saved_transient) + for t in tasks: + t.status = Status.objects.get(message="not processed") + t.save() + + ### update info + new_transient_dict = transient.__dict__ + if "host_id" in new_transient_dict.keys(): + del new_transient_dict["host_id"] + del new_transient_dict["_state"] + del new_transient_dict["id"] + saved_transients.filter(name__exact=transient.name).update( + **new_transient_dict + ) + except Transient.DoesNotExist: transient.save() + count += 1 + print(f"Added {count} new transients") + print("TNS UPLOADED") @property def task_name(self): return "TNS data ingestion" + @property + def task_frequency_seconds(self): + return 240 + @property def task_initially_enabled(self): - return False + return True class InitializeTransientTasks(SystemTaskRunner): @@ -161,7 +191,12 @@ def run_process(self): transients = Transient.objects.all() for transient in transients: - tasks = TaskRegister.objects.filter(transient__name__exact=transient.name) + tasks = TaskRegister.objects.filter( + Q(transient__name__exact=transient.name) & ~Q(task__name=self.task_name) + ) + processing_task_qs = TaskRegister.objects.filter( + transient__name__exact=transient.name, task__name=self.task_name + ) total_tasks = len(tasks) completed_tasks = len( @@ -180,6 +215,15 @@ def run_process(self): elif blocked > 0: progress = "blocked" + # save task + if len(processing_task_qs) == 1: + processing_task = processing_task_qs[0] + processing_task.status = Status.objects.get( + message=progress if progress != "completed" else "processed" + ) + processing_task.save() + + # save transient progress transient.processing_status = progress transient.save() diff --git a/app/host/tables.py b/app/host/tables.py new file mode 100644 index 00000000..54120140 --- /dev/null +++ b/app/host/tables.py @@ -0,0 +1,72 @@ +import django_tables2 as tables + +from .models import Transient + + +class TransientTable(tables.Table): + + name = tables.TemplateColumn( + "{{ record.name }}", + verbose_name="Name", + orderable=True, + order_by="name", + ) + + prefix = tables.Column( + accessor="tns_prefix", verbose_name="Prefix", orderable=True, order_by="prefix" + ) + + disc_date = tables.Column( + accessor="public_timestamp", + verbose_name="Discovery Date", + orderable=True, + order_by="ra_deg", + ) + + ra_string = tables.Column( + accessor="ra", verbose_name="Right Ascension", orderable=True, order_by="ra_deg" + ) + + dec_string = tables.Column( + accessor="dec", verbose_name="Declination", orderable=True, order_by="dec_deg" + ) + + progress = tables.TemplateColumn( + """ {% if record.progress == 0 %} + Waiting + {% elif record.progress == 100 %} +
+
{{record.progress}}%
+
+ {% else %} +
+
{{record.progress}}%
+
+ {% endif %} +""", + verbose_name="Progress", + orderable=False, + order_by="progress", + ) + + class Meta: + model = Transient + fields = () + + template_name = "django_tables2/bootstrap.html" + attrs = { + "th": { + "_ordering": { + "orderable": "sortable", # Instead of `orderable` + "ascending": "ascend", # Instead of `asc` + "descending": "descend", # Instead of `desc` + } + }, + "class": "table table-bordered table-hover", + "id": "k2_transient_tbl", + "columnDefs": [ + {"type": "title-numeric", "targets": 1}, + {"type": "title-numeric", "targets": 2}, + ], + "order": [[3, "desc"]], + } diff --git a/app/host/tasks.py b/app/host/tasks.py index f1462138..e9623023 100644 --- a/app/host/tasks.py +++ b/app/host/tasks.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import time + from celery import shared_task from .system_tasks import DeleteGHOSTFiles @@ -24,6 +26,7 @@ from .transient_tasks import ValidateLocalPhotometry periodic_tasks = [ + TNSDataIngestion(), MWEBV_Transient(), Ghost(), MWEBV_Host(), @@ -37,7 +40,6 @@ HostInformation(), GlobalHostSEDFitting(), LocalHostSEDFitting(), - TNSDataIngestion(), InitializeTransientTasks(), IngestMissedTNSTransients(), DeleteGHOSTFiles(), @@ -47,4 +49,13 @@ for task in periodic_tasks: func_name = task.task_name.replace(" ", "_").lower() - exec(f"@shared_task\ndef {func_name}(): {type(task).__name__}().run_process()") + exec( + f""" +@shared_task(time_limit=3800,soft_time_limit=3600) +def {func_name}(): + + {type(task).__name__}().run_process() + + return +""" + ) diff --git a/app/host/templates/host/base.html b/app/host/templates/host/base.html index bea88641..3db30bdc 100644 --- a/app/host/templates/host/base.html +++ b/app/host/templates/host/base.html @@ -34,7 +34,9 @@