From 0644edfedfa2a51a41e027090bd311c57c2e979b Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 16 Sep 2026 16:45:07 -0700 Subject: [PATCH 1/7] Add test that explicitly tests nulls_distinct=False --- tom_dataproducts/tests/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 0959f5066..71c55a4b2 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -653,6 +653,27 @@ def test_create_reduced_datum_duplicate(self): bandpass="r" ) + def test_create_reduced_datum_duplicate_none(self): + """Test that we cannot add a duplicate ReducedDataum, even when a field is None. + exposure_time is null in both cases, but we still expect a ValidationError. + This tests nulls_distinct=False behaves correctly.""" + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=1.0, + bandpass="r", + exposure_time=None, + ) + + with self.assertRaises(ValidationError): + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=1.0, + bandpass="r", + exposure_time=None, + ) + @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, From 9b2e3086634adaf002c858bb43c3dd2f342351c0 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 23 Sep 2026 12:47:22 -0700 Subject: [PATCH 2/7] Revert "Silence warning about unique constraints and null" This reverts commit e8647bc0508c90924d15140f692eb4d11f93dcb0. --- tom_base/settings.py | 4 ---- tom_setup/templates/tom_setup/settings.tmpl | 4 ---- 2 files changed, 8 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index ffe3b3f85..8e29250bd 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -320,10 +320,6 @@ 'PAGE_SIZE': 100 } -# Silence system checks that are not applicable to how the TOM Toolkit is designed. -# These are likely temporary and can be removed in downstream TOMs if desired. -SILENCED_SYSTEM_CHECKS = ['models.W047'] - try: from local_settings import * # noqa except ImportError: diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index 0c2f0d8c7..6b8f53838 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -332,10 +332,6 @@ PLOTLY_THEME = 'plotly_white' # Set this to False if you have a particularly large DB and paginated views are slow. SHOW_PAGINATION_INFO = True -# Silence system checks that are not applicable to how the TOM Toolkit is designed. -# These are likely temporary and can be removed in downstream TOMs if desired. -SILENCED_SYSTEM_CHECKS = ['models.W047'] - try: from local_settings import * # noqa except ImportError: From ba1efdf5bc9fc64431a9cdb7c7f67c2eb0abd0c6 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 23 Sep 2026 12:49:04 -0700 Subject: [PATCH 3/7] Add system check warning to default_settings --- tom_common/default_settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tom_common/default_settings.py b/tom_common/default_settings.py index 525ff5b0f..6d71e32a7 100644 --- a/tom_common/default_settings.py +++ b/tom_common/default_settings.py @@ -59,3 +59,7 @@ # Backwards typo compatibility TOMTOOKIT_INSTALLED_APPS = TOMTOOLKIT_INSTALLED_APPS TOMTOOKIT_MIDDLEWARE = TOMTOOLKIT_MIDDLEWARE + +# Silence system checks that are not applicable to how the TOM Toolkit is designed. +# These are likely temporary and can be removed in downstream TOMs if desired. +SILENCED_SYSTEM_CHECKS = ['models.W047'] From 6ba1123204fb5063d64c38764ffa2a56b1d408d1 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 23 Sep 2026 13:54:53 -0700 Subject: [PATCH 4/7] Add test for bulk creation with nulld_distinct=False --- tom_dataproducts/tests/tests.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 71c55a4b2..c1e4b29cc 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -1,3 +1,4 @@ +from django.db import IntegrityError from tom_targets.base_models import get_target_model_app_label from http import HTTPStatus import os @@ -665,6 +666,8 @@ def test_create_reduced_datum_duplicate_none(self): exposure_time=None, ) + # This raises a ValidationError because by using create the uniqueness constraint + # is enforced in code, and never reaches the database level constraint. with self.assertRaises(ValidationError): PhotometryReducedDatum.objects.create( target=self.target, @@ -674,6 +677,25 @@ def test_create_reduced_datum_duplicate_none(self): exposure_time=None, ) + def test_create_reduced_datum_duplicate_none_bulk(self): + """Test that we cannot add duplicate ReducedDatums, even when a field is None. + Tests that this applies to bulk operations. + """ + datums = [ + PhotometryReducedDatum( + target=self.target, + timestamp=self.timestamp, + brightness=1.0, + bandpass="r", + exposure_time=None, + ) + for _ in range(3) + ] + # This raises an IntegrityError because Python level validation does not occur in + # bulk operations, here we rely on the database level constraint to prevent duplicates + with self.assertRaises(IntegrityError): + PhotometryReducedDatum.objects.bulk_create(datums) + @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, From cee4f073137df484778bd0aa6dfe014f165d9604 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 23 Sep 2026 14:11:12 -0700 Subject: [PATCH 5/7] Be explicit about the null field being set --- tom_dataproducts/tests/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index c1e4b29cc..5c206fad9 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -663,7 +663,7 @@ def test_create_reduced_datum_duplicate_none(self): timestamp=self.timestamp, brightness=1.0, bandpass="r", - exposure_time=None, + limit=None, ) # This raises a ValidationError because by using create the uniqueness constraint @@ -674,7 +674,7 @@ def test_create_reduced_datum_duplicate_none(self): timestamp=self.timestamp, brightness=1.0, bandpass="r", - exposure_time=None, + limit=None, ) def test_create_reduced_datum_duplicate_none_bulk(self): @@ -687,7 +687,7 @@ def test_create_reduced_datum_duplicate_none_bulk(self): timestamp=self.timestamp, brightness=1.0, bandpass="r", - exposure_time=None, + limit=None, ) for _ in range(3) ] From cfcc6f3b97d5b5d1a1efb3653c3bd04225bdf3d5 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 23 Sep 2026 15:40:56 -0700 Subject: [PATCH 6/7] Remove nulls_district=False, use multiple constraits for PhotometryReducedDatum --- ...reduceddatum_unique_astrometry_and_more.py | 42 ++++++++++ tom_dataproducts/models.py | 11 +-- tom_dataproducts/tests/tests.py | 81 +++++++++++++++++-- 3 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 tom_dataproducts/migrations/0020_remove_astrometryreduceddatum_unique_astrometry_and_more.py diff --git a/tom_dataproducts/migrations/0020_remove_astrometryreduceddatum_unique_astrometry_and_more.py b/tom_dataproducts/migrations/0020_remove_astrometryreduceddatum_unique_astrometry_and_more.py new file mode 100644 index 000000000..8d64896e7 --- /dev/null +++ b/tom_dataproducts/migrations/0020_remove_astrometryreduceddatum_unique_astrometry_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 5.2.13 on 2026-09-23 22:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_dataproducts', '0019_alter_astrometryreduceddatum_options_and_more'), + ('tom_targets', '0031_basetarget_shared_by_basetarget_shared_from'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='astrometryreduceddatum', + name='unique_astrometry', + ), + migrations.RemoveConstraint( + model_name='photometryreduceddatum', + name='unique_photometry', + ), + migrations.RemoveConstraint( + model_name='spectroscopyreduceddatum', + name='unique_spectroscopy', + ), + migrations.AddConstraint( + model_name='astrometryreduceddatum', + constraint=models.UniqueConstraint(fields=('target', 'timestamp', 'telescope', 'instrument', 'ra', 'dec', 'reduction_version'), name='unique_astrometry'), + ), + migrations.AddConstraint( + model_name='photometryreduceddatum', + constraint=models.UniqueConstraint(fields=('target', 'bandpass', 'timestamp', 'brightness', 'instrument', 'reduction_version'), name='unique_photometry_brightness'), + ), + migrations.AddConstraint( + model_name='photometryreduceddatum', + constraint=models.UniqueConstraint(fields=('target', 'bandpass', 'timestamp', 'limit', 'instrument', 'reduction_version'), name='unique_photometry_limit'), + ), + migrations.AddConstraint( + model_name='spectroscopyreduceddatum', + constraint=models.UniqueConstraint(fields=('target', 'timestamp', 'telescope', 'instrument', 'flux', 'reduction_version'), name='unique_spectroscopy'), + ), + ] diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index ec79e054e..7d7b2521f 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -456,9 +456,12 @@ class PhotometryReducedDatum(ReducedDatumCommon): class Meta(ReducedDatumCommon.Meta): constraints = [ models.UniqueConstraint( - fields=["target", "bandpass", "timestamp", "limit", "brightness", "instrument", "reduction_version"], - name="unique_photometry", - nulls_distinct=False + fields=["target", "bandpass", "timestamp", "brightness", "instrument", "reduction_version"], + name="unique_photometry_brightness", + ), + models.UniqueConstraint( + fields=["target", "bandpass", "timestamp", "limit", "instrument", "reduction_version"], + name="unique_photometry_limit", ) ] @@ -481,7 +484,6 @@ class Meta(ReducedDatumCommon.Meta): models.UniqueConstraint( fields=["target", "timestamp", "telescope", "instrument", "flux", "reduction_version"], name="unique_spectroscopy", - nulls_distinct=False ) ] @@ -503,7 +505,6 @@ class Meta(ReducedDatumCommon.Meta): models.UniqueConstraint( fields=["target", "timestamp", "telescope", "instrument", "ra", "dec", "reduction_version"], name="unique_astrometry", - nulls_distinct=False ) ] diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 5c206fad9..84a188380 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -636,6 +636,17 @@ def test_create_reduced_datum(self): self.assertEqual(2, ReducedDatum.objects.count()) + def test_create_reduced_datum_without_legacy_value(self): + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=None, + bandpass="r", + limit=2.0, + ) + + self.assertEqual(1, ReducedDatum.objects.count()) + def test_create_reduced_datum_duplicate(self): """Test that we cannot add a second PhotometryReducedDatum with the same target, timestamp, brightness, and bandpass""" @@ -654,10 +665,9 @@ def test_create_reduced_datum_duplicate(self): bandpass="r" ) - def test_create_reduced_datum_duplicate_none(self): - """Test that we cannot add a duplicate ReducedDataum, even when a field is None. - exposure_time is null in both cases, but we still expect a ValidationError. - This tests nulls_distinct=False behaves correctly.""" + def test_create_reduced_datum_duplicate_limit_none(self): + """Test that we cannot add a duplicate ReducedDataum, even when limit is None. + exposure_time is null in both cases, but we still expect a ValidationError.""" PhotometryReducedDatum.objects.create( target=self.target, timestamp=self.timestamp, @@ -677,10 +687,41 @@ def test_create_reduced_datum_duplicate_none(self): limit=None, ) - def test_create_reduced_datum_duplicate_none_bulk(self): - """Test that we cannot add duplicate ReducedDatums, even when a field is None. + def test_create_reduced_datum_duplicate_brightness_none(self): + """Test that we cannot add a duplicate ReducedDataum, even when brightness is None. + exposure_time is null in both cases, but we still expect a ValidationError.""" + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=None, + bandpass="r", + limit=2.0, + ) + + # This raises a ValidationError because by using create the uniqueness constraint + # is enforced in code, and never reaches the database level constraint. + with self.assertRaises(ValidationError): + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=None, + bandpass="r", + limit=2.0, + ) + + def test_create_reduced_datum_duplicate_none_limit_bulk(self): + """Test that we cannot add duplicate ReducedDatums, even when limit is None. Tests that this applies to bulk operations. """ + # Ensure we can create one first + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=1.0, + bandpass="r", + limit=None, + ) + self.assertEqual(1, PhotometryReducedDatum.objects.count()) datums = [ PhotometryReducedDatum( target=self.target, @@ -696,6 +737,34 @@ def test_create_reduced_datum_duplicate_none_bulk(self): with self.assertRaises(IntegrityError): PhotometryReducedDatum.objects.bulk_create(datums) + def test_create_reduced_datum_duplicate_none_brightness_bulk(self): + """Test that we cannot add duplicate ReducedDatums, even when brightness is None. + Tests that this applies to bulk operations. + """ + # Ensure we can create one first + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=None, + bandpass="r", + limit=2.0, + ) + self.assertEqual(1, PhotometryReducedDatum.objects.count()) + datums = [ + PhotometryReducedDatum( + target=self.target, + timestamp=self.timestamp, + brightness=None, + bandpass="r", + limit=2.0, + ) + for _ in range(3) + ] + # This raises an IntegrityError because Python level validation does not occur in + # bulk operations, here we rely on the database level constraint to prevent duplicates + with self.assertRaises(IntegrityError): + PhotometryReducedDatum.objects.bulk_create(datums) + @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, From b0235222c45cc703b3a8c2c6c682a0f29c42c3f5 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 23 Sep 2026 15:42:11 -0700 Subject: [PATCH 7/7] No longer need to silence warnings out constraints --- tom_common/default_settings.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tom_common/default_settings.py b/tom_common/default_settings.py index 6d71e32a7..525ff5b0f 100644 --- a/tom_common/default_settings.py +++ b/tom_common/default_settings.py @@ -59,7 +59,3 @@ # Backwards typo compatibility TOMTOOKIT_INSTALLED_APPS = TOMTOOLKIT_INSTALLED_APPS TOMTOOKIT_MIDDLEWARE = TOMTOOLKIT_MIDDLEWARE - -# Silence system checks that are not applicable to how the TOM Toolkit is designed. -# These are likely temporary and can be removed in downstream TOMs if desired. -SILENCED_SYSTEM_CHECKS = ['models.W047']