From 4206c9e930c6743e13e3d3a3720448e3692617ee Mon Sep 17 00:00:00 2001 From: David Jones Date: Mon, 22 Jun 2026 16:50:55 -1000 Subject: [PATCH 1/4] user-specified changes to transient, host, and aperture properties --- app/host/forms.py | 10 + ...omment_transient_update_fields_and_more.py | 23 ++ app/host/models.py | 4 +- app/host/templates/host/add_transient.html | 61 ++++- app/host/views.py | 232 +++++++++++++++++- 5 files changed, 327 insertions(+), 3 deletions(-) create mode 100644 app/host/migrations/0051_transient_update_comment_transient_update_fields_and_more.py diff --git a/app/host/forms.py b/app/host/forms.py index 9d10600b..2d372382 100644 --- a/app/host/forms.py +++ b/app/host/forms.py @@ -49,4 +49,14 @@ class TransientUploadForm(forms.Form): label='Transient definition table', required=False, ) + update_info = forms.CharField( + widget=forms.Textarea(attrs={ + 'style': 'min-width: 30rem;', + 'cols': 100, + 'placeholder': 'Identifier, RA, Dec, Redshift, Classification, HostName, HostRA, HostDec, HostRedshift, Global_aperture_a, Global_aperture_b, Global_aperture_theta, Comment' + }), + label='Transient update table', + required=False, + ) + file = forms.FileField(required=False) diff --git a/app/host/migrations/0051_transient_update_comment_transient_update_fields_and_more.py b/app/host/migrations/0051_transient_update_comment_transient_update_fields_and_more.py new file mode 100644 index 00000000..7ae87683 --- /dev/null +++ b/app/host/migrations/0051_transient_update_comment_transient_update_fields_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.14 on 2026-06-23 00:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('host', '0050_alter_host_name_alias'), + ] + + operations = [ + migrations.AddField( + model_name='transient', + name='update_comment', + field=models.CharField(blank=True, max_length=500, null=True), + ), + migrations.AddField( + model_name='transient', + name='update_fields', + field=models.CharField(blank=True, max_length=500, null=True), + ), + ] diff --git a/app/host/models.py b/app/host/models.py index 43599689..f0e800bc 100644 --- a/app/host/models.py +++ b/app/host/models.py @@ -162,7 +162,9 @@ def validate_name(name): added_by = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) progress = models.IntegerField(default=0) software_version = models.CharField(max_length=50, blank=True, null=True) - + update_comment = models.CharField(max_length=500, blank=True, null=True) + update_fields = models.CharField(max_length=500, blank=True, null=True) + @property def best_redshift(self): """get the best redshift for a transient""" diff --git a/app/host/templates/host/add_transient.html b/app/host/templates/host/add_transient.html index c07d72f1..ef89bad2 100644 --- a/app/host/templates/host/add_transient.html +++ b/app/host/templates/host/add_transient.html @@ -57,7 +57,7 @@

Add Transients

- There are three ways to add transients to the Blast database and trigger their data processing. + There are four ways to add or update transients to the Blast database and trigger their data processing. Click a method below to enter information:

@@ -71,6 +71,10 @@

Add Transients

  Define transients by specifying properties. + + +   Update properties of existing transients. +   Import transient from an exported archive file. @@ -111,6 +115,18 @@

Add Transients

{% endif %} + {% if updated_transient_names|length %} +
+

The following transients were updated and re-triggered:

+ +
+ {% endif %} + + {% if defined_transient_names|length %}

The following transients were successfully added to the Blast database:

@@ -170,6 +186,28 @@

Add Transients

{{ form.full_info | as_crispy_field }}
+
+

Update transients by supplying a comma-separated value (CSV) table, + where each row defines a transient with the following columns of information:
+ Identifier, RA, Dec, Redshift, Classification, HostName, HostRA, HostDec, HostRedshift, Global_aperture_a, Global_aperture_b, Global_aperture_theta, Comment +

+

A transient will not be added if the name already exists in the database or if the coordinates are within one arcsecond of an existing transient.

+

+
+ {{ form.update_info | as_crispy_field }} +
+
+
@@ -184,9 +222,11 @@

Add Transients

function showImportForm() { document.getElementById("import-form").style.display = "block"; document.getElementById("define-form").style.display = "none"; + document.getElementById("update-form").style.display = "none"; document.getElementById("import-archive-form").style.display = "none"; document.getElementById("import-button").classList.add("active"); document.getElementById("define-button").classList.remove("active"); + document.getElementById("update-button").classList.remove("active"); document.getElementById("upload-archive-button").classList.remove("active"); document.getElementById("submit-btn").style.display = "block"; document.getElementById("submit-btn").innerHTML = "Submit"; @@ -196,9 +236,11 @@

Add Transients

function showCreateForm() { document.getElementById("import-form").style.display = "none"; document.getElementById("define-form").style.display = "block"; + document.getElementById("update-form").style.display = "none"; document.getElementById("import-archive-form").style.display = "none"; document.getElementById("import-button").classList.remove("active"); document.getElementById("define-button").classList.add("active"); + document.getElementById("update-button").classList.remove("active"); document.getElementById("upload-archive-button").classList.remove("active"); document.getElementById("submit-btn").style.display = "block"; document.getElementById("submit-btn").innerHTML = "Submit"; @@ -208,9 +250,11 @@

Add Transients

function showImportArchiveForm() { document.getElementById("import-form").style.display = "none"; document.getElementById("define-form").style.display = "none"; + document.getElementById("update-form").style.display = "none"; document.getElementById("import-archive-form").style.display = "block"; document.getElementById("import-button").classList.remove("active"); document.getElementById("define-button").classList.remove("active"); + document.getElementById("update-button").classList.remove("active"); document.getElementById("upload-archive-button").classList.add("active"); document.getElementById("submit-btn").style.display = "block"; document.getElementById("submit-btn").innerHTML = "Upload"; @@ -218,5 +262,20 @@

Add Transients

document.getElementById("id_full_info").value = ""; document.getElementById("add-form").enctype = "multipart/form-data"; } + function showUpdateForm() { + document.getElementById("import-form").style.display = "none"; + document.getElementById("define-form").style.display = "none"; + document.getElementById("update-form").style.display = "block"; + document.getElementById("import-archive-form").style.display = "none"; + document.getElementById("import-button").classList.remove("active"); + document.getElementById("define-button").classList.remove("active"); + document.getElementById("update-button").classList.add("active"); + document.getElementById("upload-archive-button").classList.remove("active"); + document.getElementById("submit-btn").style.display = "block"; + document.getElementById("submit-btn").innerHTML = "Submit"; + document.getElementById("id_tns_names").value = ""; + document.getElementById("add-form").removeAttribute("enctype"); + } + {% endblock %} diff --git a/app/host/views.py b/app/host/views.py index b052132e..6b42adf8 100644 --- a/app/host/views.py +++ b/app/host/views.py @@ -3,6 +3,7 @@ import csv import io import base64 +import numpy as np from django.contrib.auth.decorators import login_required, permission_required from django.db.models import Q from django.http import HttpResponse, JsonResponse @@ -28,12 +29,14 @@ from host.models import Task from host.models import Status from host.models import Transient +from host.models import Host from host.plotting_utils import plot_bar_chart from host.plotting_utils import plot_cutout_image from host.plotting_utils import render_sed_plot from host.plotting_utils import plot_sed from host.tables import TransientTable from host.tasks import import_transient_list +from host.tasks import retrigger_transient from host.object_store import ObjectStore from silk.profiling.profiler import silk_profile from django.template.loader import render_to_string @@ -42,6 +45,9 @@ from host.decorators import log_usage_metric from host.host_utils import ARCSEC_DEC_IN_DEG from host.host_utils import ARCSEC_RA_IN_DEG +from host.host_utils import select_cutout_aperture +from host.transient_tasks import MWEBV_Host +from host.transient_tasks import HostInformation from host.log import get_logger logger = get_logger(__name__) @@ -198,12 +204,16 @@ def identify_existing_transients(transients=[]): imported_transient_names = [] file_imported_transient_names = [] existing_transient_names = [] - + transients_not_found = [] + transients_to_update = [] + if request.method == "POST": form = TransientUploadForm(request.POST, request.FILES) if form.is_valid(): tns_names = form.cleaned_data["tns_names"] full_info = form.cleaned_data["full_info"] + update_info = form.cleaned_data["update_info"] + # Import transient dataset from an uploaded archive file if "file" in request.FILES: logger.debug('Importing transient from uploaded archive file...') @@ -292,6 +302,224 @@ def identify_existing_transients(transients=[]): errors.append(err_msg) # Trigger processing of new transients import_transient_list.delay(defined_transient_names) + # update transient or host properties + elif update_info: + reader = csv.DictReader(io.StringIO(update_info), fieldnames=[ + 'name', + 'ra_deg', + 'dec_deg', + 'redshift', + 'specclass', + 'host_name', + 'host_ra_deg', + 'host_dec_deg', + 'host_redshift', + 'aperture_semi_major_axis_arcsec', + 'aperture_semi_minor_axis_arcsec', + 'aperture_orientation_deg', + 'update_comment' + ]) + for transient in reader: + try: + try: + existing_transient = Transient.objects.get(name=transient['name']) + except: + transients_not_found += [transient['name']] + if transient['specclass'].lower().strip() != "none": + existing_transient.spectroscopic_class = transient['specclass'].strip() + if transient['redshift'].lower().strip() != "none": + existing_transient.redshift = float(transient['redshift'].strip()) + if transient['ra_deg'].lower().strip() != "none": + existing_transient.ra_deg = float(transient['ra_deg'].strip()) + if transient['dec_deg'].lower().strip() != "none": + existing_transient.dec_deg = float(transient['dec_deg'].strip()) + if transient['update_comment'].lower().strip() != "none": + existing_transient.update_comment = transient['update_comment'].strip() + fields_updated = [] + for k in transient.keys(): + if k == 'update_comment' or k == 'name': continue + if transient[k].lower().strip() != "none": + fields_updated += [k] + existing_transient.update_fields = ','.join(fields_updated) + + if 'redshift' in fields_updated: + update_transient_redshift = True + else: + update_transient_redshift = False + + if 'host_redshift' in fields_updated: + update_host_redshift = True + else: + update_host_redshift = False + + + if 'ra_deg' in fields_updated or 'dec_deg' in fields_updated: + update_transient_loc = True + else: + update_transient_loc = False + + # host parameters + if transient['host_name'].lower().strip() != "none" or \ + transient['host_ra_deg'].lower().strip() != "none" or \ + transient['host_dec_deg'].lower().strip() != "none" or \ + transient['host_redshift'].lower().strip() != "none": + update_host = True + host = Host.objects.filter(transient__name=transient['name']) + if not len(host): + # if any host info is being updated, we might need to create + # a host object + host = Host.objects.create( + ra_deg=float(transient['host_ra_deg'].lower().strip()), + dec_deg=float(transient['host_dec_deg'].lower().strip()), + transient=existing_transient + ) + else: + host = host[0] + if transient['host_name'].lower().strip() != "none": + host.name = transient['host_name'].strip() + if transient['host_ra_deg'].lower().strip() != "none": + host.ra_deg = float(transient['host_ra_deg'].strip()) + if transient['host_dec_deg'].lower().strip() != "none": + host.dec_deg = float(transient['host_dec_deg'].strip()) + if transient['host_redshift'].lower().strip() != "none": + host.redshift = float(transient['host_redshift'].strip()) + else: + update_host = False + + # aperture parameters + if transient['aperture_semi_major_axis_arcsec'].lower().strip() != "none" or \ + transient['aperture_semi_minor_axis_arcsec'].lower().strip() != "none" or \ + transient['aperture_orientation_deg'].lower().strip() != "none": + update_aperture = True + + if not update_host: + host = Host.objects.get(transient__name=transient['name']) + + cutouts = Cutout.objects.filter(transient=existing_transient).filter(~Q(fits="")) + aperture_cutout = select_cutout_aperture(cutouts, choice=0)[0] + + # we need to delete any other global apertures associated with this transient + # if we want to update the parameters. + other_apertures = Aperture.objects.filter( + transient__name=transient['name'], + type='global' + ).exclude( + cutout__filter=aperture_cutout.filter + ) + # we also need to delete any global photometry associated with those apertures + for o in other_apertures: + AperturePhotometry.objects.filter(aperture=o).delete() + other_apertures.delete() + + + aperture = Aperture.objects.filter( + transient__name=transient['name'], + cutout__filter=aperture_cutout.filter, + type='global' + ) + if not len(aperture): + aperture = Aperture.objects.create( + name=f"{existing_transient.name}_{aperture_cutout.filter.name}_global", + cutout=aperture_cutout, + transient=existing_transient, + ra_deg=host.ra_deg, + dec_deg=host.dec_deg, + orientation_deg=float(transient['aperture_orientation_deg'].lower().strip()), + semi_major_axis_arcsec=float(transient['aperture_semi_major_axis_arcsec'].lower().strip()), + semi_minor_axis_arcsec=float(transient['aperture_semi_minor_axis_arcsec'].lower().strip()), + type='global' + ) + else: + aperture = aperture[0] + if update_host: + aperture.ra_deg = host.ra_deg + aperture.dec_deg = host.dec_deg + if transient['aperture_orientation_deg'].lower().strip() != "none": + aperture.orientation_deg = float(transient['aperture_orientation_deg'].strip()) + if transient['aperture_semi_major_axis_arcsec'].lower().strip() != "none": + aperture.semi_major_axis_arcsec = float(transient['aperture_semi_major_axis_arcsec'].strip()) + if transient['aperture_semi_minor_axis_arcsec'].lower().strip() != "none": + aperture.semi_minor_axis_arcsec = float(transient['aperture_semi_minor_axis_arcsec'].strip()) + + else: + update_aperture = False + + # if we made it all the way to the end, we can save + existing_transient.save() + if update_host: host.save() + if update_aperture: aperture.save() + + # reset the relevant task statuses to "not processed" so that they can be re-triggered. + # this is a pretty annoying set of logic, the goal is not to reprocess any jobs that + # would override the customized data + tasknames_to_redo = [] + tasknames_to_skip = [] + if update_transient_loc: + tasknames_to_redo += [ + 'Transient MWEBV','Host match','Host information', + 'Host MWEBV','Global aperture construction', + 'Global aperture photometry','Validate global photometry', + 'Global host SED inference','Local aperture photometry', + 'Validate local photometry','Local host SED inference' + ] + if update_transient_redshift or update_host_redshift: + tasknames_to_redo += [ + 'Global host SED inference','Local aperture photometry', + 'Validate local photometry','Local host SED inference' + ] + + if update_host: + tasknames_to_skip += [ + 'Cutout download','Transient MWEBV','Host match', + 'Host information','Host MWEBV' + ] + tasknames_to_redo += [ + 'Global aperture construction','Global aperture photometry', + 'Validate global photometry','Global host SED inference' + ] + mwebv_task_register = TaskRegister.objects.get(transient=existing_transient,task__name='Host MWEBV') + mwebv_status = MWEBV_Host(existing_transient.name)._run_process(existing_transient) + mwebv_task_register.status = Status.objects.get(message=mwebv_status) + mwebv_task_register.save() + host_task_register = TaskRegister.objects.get(transient=existing_transient,task__name='Host information') + host_status = HostInformation(existing_transient.name)._run_process(existing_transient) + host_task_register.status = Status.objects.get(message=host_status) + host_task_register.save() + + if update_aperture: + tasknames_to_skip += ['Global aperture construction'] + tasknames_to_redo += [ + 'Global aperture photometry', + 'Validate global photometry', + 'Global host SED inference' + ] + # now we make sure that tasks_to_redo doesn't include anything in tasks to skip + for i,tnr in enumerate(tasknames_to_redo): + if tnr in tasknames_to_skip: + tasknames_to_redo.pop(i) + tasknames_to_redo = np.unique(tasknames_to_redo) + tasknames_to_skip = np.unique(tasknames_to_skip) + + tasks_to_redo = TaskRegister.objects.filter(transient=existing_transient,task__name__in=tasknames_to_redo) + tasks_to_skip = TaskRegister.objects.filter(transient=existing_transient,task__name__in=tasknames_to_skip) + for tr in tasks_to_redo: + tr.status = Status.objects.get(message='not processed') + tr.save() + for tr in tasks_to_skip: + tr.status = Status.objects.get(message='processed') + tr.save() + transients_to_update += [existing_transient.name] + + except Exception as err: + err_msg = f'''Error parsing line "{transient}": {err}''' + logger.error(err_msg) + errors.append(err_msg) + continue + + # Retrigger updated transients + for tu in transients_to_update: + retrigger_transient(transient_name=tu) + else: form = TransientUploadForm() @@ -303,6 +531,8 @@ def identify_existing_transients(transients=[]): "imported_transient_names": imported_transient_names, "file_imported_transient_names": file_imported_transient_names, "existing_transient_names": existing_transient_names, + "not_existing_transient_names": transients_not_found, + "updated_transient_names":transients_to_update, } return render(request, "add_transient.html", context) From 21657d682fe9bb0062287ea3ea4ac298de2fb72d Mon Sep 17 00:00:00 2001 From: David Jones Date: Fri, 26 Jun 2026 16:59:16 -1000 Subject: [PATCH 2/4] changelog update and initial framework for unit test --- CHANGELOG.md | 8 ++++++++ app/host/tests/test_views.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb93af13..6251d851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ Types of changes: - `Fixed`: for any bug fixes. - `Security`: in case of vulnerabilities. +## [x.y.z] + +### Added + +- Added a new form to update properties of existing transients +- Form prompts retriggering of updated transients from the relevant stages +- Added a new migration to record comments on any user-updated properties + ## [1.12.0] ### Added diff --git a/app/host/tests/test_views.py b/app/host/tests/test_views.py index f391076a..6353dd33 100644 --- a/app/host/tests/test_views.py +++ b/app/host/tests/test_views.py @@ -18,7 +18,40 @@ def test_transient_page(self): response = self.client.get("/transients/2022testtwo/") self.assertEqual(response.status_code, 200) +class ModifyTransientTest(TestCase): + def setUp(self): + import base64 + username = 'testuser' + b64username = base64.urlsafe_b64encode(username.encode('utf-8')).decode('utf-8').strip('=') + self.credentials = {"username": b64username, "password": "secret"} + User.objects.create_user(**self.credentials, is_superuser=True) + self.client.login(**self.credentials) + with open('''/data/transient_datasets/2026dix.tar.gz''', 'rb') as dataset_fileobj: + import_transient_info(dataset_fileobj) + def test_add_tansients_by_definition(self): + # TODO: This test is fragile due to the explicit HTML string search. + response = self.client.post("/add/", data={ + 'update_info': dedent(''' + 2026dix,None,None,0.05,SN Ia,None,None,None,None,None,None,None,z and class test + 2026dgt,None,None,None,None,None,132.3561625,29.5105694,None,5,5,0,host pos test + ''')}) + self.assertEqual(response.status_code, 200) + # Check that transients were updated + self.assertContains( + response, + text=(""" + +""") + # Check that 2026dix z and class were updated + + # Check the 2026dgt host coord and aperture were updated + + # check that 2026dix tasks set to unprocessed + + # check that 2026dgt tasks set to unprocessed + + class AddTransientTest(TestCase): def setUp(self): import base64 @@ -85,3 +118,4 @@ def test_add_tansients_by_definition(self): # print(f'''Response: [{response.status_code}]\n{response.content}''') # print(f'''Response: [{response.status_code}]\n{response.content.decode('utf-8')}''') + From e318cbc1115ddc37cf97dd668217c35e031becd8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 28 Jun 2026 07:45:20 +0000 Subject: [PATCH 3/4] working unit test for modifying transients. Fixed bug where missing transients in the database were not handled properly --- app/host/templates/host/add_transient.html | 12 +++- app/host/tests/test_views.py | 65 ++++++++++++++++++++-- app/host/views.py | 1 + 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/app/host/templates/host/add_transient.html b/app/host/templates/host/add_transient.html index ef89bad2..69232e6c 100644 --- a/app/host/templates/host/add_transient.html +++ b/app/host/templates/host/add_transient.html @@ -126,6 +126,16 @@

Add Transients

{% endif %} + {% if not_existing_transient_names|length %} +
+

The following transients were not found in the database:

+ +
+ {% endif %} {% if defined_transient_names|length %}
@@ -197,7 +207,7 @@

Add Transients

  • If Blast has not successfully identified a host galaxy, you must provide HostRA and HostDec if you wish to input aperture parameters..
  • Global_aperture_a and Global_aperture_b must be in arcseconds.
  • Global_aperture_theta must be in degrees. For a right-handed world coordinate system, theta increases counterclockwise from North (PA=0).
    • -
  • The comment field is a brief explanation (500-character max) for why the transient data were updated/
    • +
  • The comment field is a brief explanation (500-character max) for why the transient data were updated.
  • Use "None" to indicate data that should not be updated.
  • For a given transient, the "Cutout download" and "Transient MWEBV" tasks must have completed successfully for these updates to be recorded.
    • diff --git a/app/host/tests/test_views.py b/app/host/tests/test_views.py index 6353dd33..bc3a994e 100644 --- a/app/host/tests/test_views.py +++ b/app/host/tests/test_views.py @@ -1,7 +1,11 @@ from django.test import TestCase from textwrap import dedent from django.contrib.auth.models import User +from django.db.models import Q from ..host_utils import import_transient_info +from host.models import Transient +from host.models import Aperture +from host.models import TaskRegister class ViewTest(TestCase): @@ -28,28 +32,81 @@ def setUp(self): self.client.login(**self.credentials) with open('''/data/transient_datasets/2026dix.tar.gz''', 'rb') as dataset_fileobj: import_transient_info(dataset_fileobj) + with open('''/data/transient_datasets/2026dgt.tar.gz''', 'rb') as dataset_fileobj: + import_transient_info(dataset_fileobj) def test_add_tansients_by_definition(self): # TODO: This test is fragile due to the explicit HTML string search. response = self.client.post("/add/", data={ - 'update_info': dedent(''' + 'update_info': dedent(""" 2026dix,None,None,0.05,SN Ia,None,None,None,None,None,None,None,z and class test 2026dgt,None,None,None,None,None,132.3561625,29.5105694,None,5,5,0,host pos test - ''')}) + """)}) self.assertEqual(response.status_code, 200) # Check that transients were updated self.assertContains( response, - text=(""" + text=("""

    The following transients were updated and re-triggered:

    \n