From df87397cf4d6e0fa89ac0f0e3cb575da57c5019d Mon Sep 17 00:00:00 2001 From: depryf Date: Mon, 31 Aug 2026 12:44:44 -0400 Subject: [PATCH] Honor the min/max DX dates from the options in the date of diagnosis rule The rule used to clear all its date constraints as soon as a tumor index was in the context (which is always the case when generating a patient), and rebuild the minimum from the patient's date of birth and the age group of the tumor's site. Since the date of birth is derived from the oldest age group of all the tumors of the patient, any tumor assigned to a younger age group could be diagnosed decades before the min DX date requested in the options; the ordering of the tumors was lost as well. The date range from the options is now the hard requirement and the other constraints (year of birth, DX date of the previous tumor, age group of the site) are only applied when they don't push the date out of that range. When the default min DX date (ten years ago) ends up after a requested max DX date, the ten years before the max date are used. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 4 + .../rule/tumor/DateOfDiagnosisRule.java | 63 ++++++++------- .../NaaccrFixedColumnsDataGeneratorTest.java | 24 +++--- .../rule/tumor/DateOfDiagnosisRuleTest.java | 80 ++++++++++++++++++- 4 files changed, 127 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae3b161..779f274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Synthetic data generator +**Version 2.7** + +- Fixed issue in date of diagnosis rule that would produce DX dates outside of the min/max DX dates provided in the options. + **Version 2.6** - Fixed issue in DOLC that would produce tumors with vital status 0 (dead) and DX date later than DOLC. diff --git a/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java b/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java index 2c40f14..1099a47 100644 --- a/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java +++ b/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java @@ -1,9 +1,7 @@ package com.imsweb.datagenerator.naaccr.rule.tumor; import java.time.LocalDate; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import com.imsweb.datagenerator.naaccr.NaaccrDataGeneratorOptions; import com.imsweb.datagenerator.naaccr.NaaccrDataGeneratorTumorRule; @@ -29,51 +27,56 @@ public DateOfDiagnosisRule() { @Override public void execute(Tumor tumor, Patient patient, NaaccrDataGeneratorOptions options, Map context) { - // latest possible date set only by options if defined - Set maxDxDates = new HashSet<>(); + // the date range requested in the options is a hard requirement; the other constraints (age group of the site, year of birth, dx date of the + // previous tumor) are only applied when they don't push the generated date outside of that range LocalDate maxDate = options == null ? LocalDate.now() : options.getMaxDxDate(); - maxDxDates.add(maxDate); + LocalDate minDate = options == null ? LocalDate.now().minusYears(10) : options.getMinDxDate(); + // the min dx date defaults to ten years ago when it's not provided in the options; that default can end up after a requested max dx date + if (minDate.isAfter(maxDate)) + minDate = maxDate.minusYears(10); - Set minDxDates = new HashSet<>(); - // never go before min date defined in options, or current date minus ten years if options not defined - minDxDates.add(options == null ? LocalDate.now().minusYears(10) : options.getMinDxDate()); // never go before the year of birth if (hasValue(patient, "dateOfBirthYear")) - minDxDates.add(LocalDate.of(Integer.parseInt(patient.getItemValue("dateOfBirthYear")) + 1, 1, 1)); + minDate = tightenMinDate(minDate, maxDate, LocalDate.of(Integer.parseInt(patient.getItemValue("dateOfBirthYear")) + 1, 1, 1)); + // never go before dx date of patient's most recent tumor (if this isn't the first one) if (!patient.getTumors().isEmpty()) { Tumor lastTumor = patient.getTumor(patient.getTumors().size() - 1); - minDxDates.add(LocalDate.of( + minDate = tightenMinDate(minDate, maxDate, LocalDate.of( Integer.parseInt(lastTumor.getItemValue("dateOfDiagnosisYear")), Integer.parseInt(lastTumor.getItemValue("dateOfDiagnosisMonth")), Integer.parseInt(lastTumor.getItemValue("dateOfDiagnosisDay")))); } - if (context.get(CONTEXT_FLAG_CURRENT_TUMOR_INDEX) != null) { - int birthYear = Integer.parseInt(patient.getItemValue("dateOfBirthYear")); - int birthMonth = Integer.parseInt(patient.getItemValue("dateOfBirthMonth")); - int birthDay = Integer.parseInt(patient.getItemValue("dateOfBirthDay")); - LocalDate dateOfBirth = LocalDate.of(birthYear, birthMonth, birthDay); - - // PROBLEM: This brakes 3 previous rules: - // 1. Minimum date must be within 10 years of today. - // 2. Options specify a minimum DX date. - // 3. This tumor must be diagnosed after the previous ones for this patient. - // From Fabian: Only #2 is required. Try to get all tumors to use this minimum. If that can't be done, at least one tumor must meet it. - minDxDates.clear(); - maxDxDates.clear(); - - int currentTumorIndex = (int)context.get(CONTEXT_FLAG_CURRENT_TUMOR_INDEX); - @SuppressWarnings("unchecked") - Map ageGroupMap = (Map)context.get(CONTEXT_FLAG_AGE_GROUP_MAP); - minDxDates.add(dateOfBirth.plusYears((ageGroupMap.get(currentTumorIndex) * 10))); - maxDxDates.add(maxDate); + // never diagnose the tumor before the patient reaches the age group that was picked for its site + @SuppressWarnings("unchecked") + Map ageGroupMap = (Map)context.get(CONTEXT_FLAG_AGE_GROUP_MAP); + Integer currentTumorIndex = (Integer)context.get(CONTEXT_FLAG_CURRENT_TUMOR_INDEX); + if (ageGroupMap != null && currentTumorIndex != null && hasValue(patient, "dateOfBirthYear", "dateOfBirthMonth", "dateOfBirthDay")) { + Integer ageGroup = ageGroupMap.get(currentTumorIndex); + // the age group is -1 for a site that has no age distribution, in which case it tells us nothing about the dx date + if (ageGroup != null && ageGroup > 0) { + LocalDate dateOfBirth = LocalDate.of( + Integer.parseInt(patient.getItemValue("dateOfBirthYear")), + Integer.parseInt(patient.getItemValue("dateOfBirthMonth")), + Integer.parseInt(patient.getItemValue("dateOfBirthDay"))); + minDate = tightenMinDate(minDate, maxDate, dateOfBirth.plusYears(ageGroup * 10L)); + } } - LocalDate randomDate = RandomUtils.getRandomDateBetween(minDxDates, maxDxDates); + LocalDate randomDate = RandomUtils.getRandomDateBetween(minDate, maxDate); setValue(tumor, "dateOfDiagnosisYear", Integer.toString(randomDate.getYear())); setValue(tumor, "dateOfDiagnosisMonth", Integer.toString(randomDate.getMonthValue())); setValue(tumor, "dateOfDiagnosisDay", Integer.toString(randomDate.getDayOfMonth())); } + + /** + * Returns the candidate date if it is a tighter minimum than the current one and still leaves a valid range, the current minimum otherwise. + */ + private static LocalDate tightenMinDate(LocalDate currentMinDate, LocalDate maxDate, LocalDate candidate) { + if (candidate.isAfter(currentMinDate) && !candidate.isAfter(maxDate)) + return candidate; + return currentMinDate; + } } diff --git a/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java b/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java index 66fd4c2..f17892b 100644 --- a/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java +++ b/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java @@ -144,22 +144,20 @@ public void testGeneratePatient() throws IOException { patient = generator.generatePatient(1, options); Assert.assertEquals("TEST", patient.getFirst().get("nameLast")); - // Test context - int numTumors = 2; + // Test context; the requested dx date range applies to every tumor, not just to one of them + int numTumors = 3; options = new NaaccrDataGeneratorOptions(); options.setMinDxYear(2000); options.setMaxDxYear(2005); - patient = generator.generatePatient(numTumors, options); - - LocalDate dateOfDx1 = LocalDate.of(Integer.parseInt(patient.get(0).get("dateOfDiagnosisYear")), Integer.parseInt(patient.get(0).get("dateOfDiagnosisMonth")), - Integer.parseInt(patient.get(0).get("dateOfDiagnosisDay"))); - LocalDate dateOfDx2 = LocalDate.of(Integer.parseInt(patient.get(1).get("dateOfDiagnosisYear")), Integer.parseInt(patient.get(1).get("dateOfDiagnosisMonth")), - Integer.parseInt(patient.get(1).get("dateOfDiagnosisDay"))); - - boolean dateInRange1 = dateOfDx1.isAfter(options.getMinDxDate().minusDays(1)) && dateOfDx1.isBefore(options.getMaxDxDate().plusDays(1)); - boolean dateInRange2 = dateOfDx2.isAfter(options.getMinDxDate().minusDays(1)) && dateOfDx2.isBefore(options.getMaxDxDate().plusDays(1)); - - Assert.assertTrue("Diagnosis Date outside options Minimum and Maximum.", dateInRange1 || dateInRange2); + for (int i = 0; i < 250; i++) { + patient = generator.generatePatient(numTumors, options); + for (Map rec : patient) { + LocalDate dateOfDx = LocalDate.of(Integer.parseInt(rec.get("dateOfDiagnosisYear")), Integer.parseInt(rec.get("dateOfDiagnosisMonth")), + Integer.parseInt(rec.get("dateOfDiagnosisDay"))); + Assert.assertFalse("Diagnosis Date before options Minimum: " + dateOfDx, dateOfDx.isBefore(options.getMinDxDate())); + Assert.assertFalse("Diagnosis Date after options Maximum: " + dateOfDx, dateOfDx.isAfter(options.getMaxDxDate())); + } + } // another test with an incidence generator generator = new NaaccrFixedColumnsDataGenerator(LayoutFactory.LAYOUT_ID_NAACCR_18_INCIDENCE); diff --git a/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java b/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java index 498697b..910c3a3 100644 --- a/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java +++ b/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java @@ -101,7 +101,85 @@ public void testExecute() { LocalDate startDate = dateOfBirth.plusYears(5 * 10); LocalDate endDate = options.getMaxDxDate(); - Assert.assertTrue(dateOfDx.toString(), dateOfDx.isAfter(startDate) && dateOfDx.isBefore(endDate)); + Assert.assertTrue(dateOfDx.toString(), !dateOfDx.isBefore(startDate) && !dateOfDx.isAfter(endDate)); } + + @Test + public void testExecuteRespectsRequestedDxDateRange() { + + NaaccrDataGeneratorOptions options = new NaaccrDataGeneratorOptions(); + options.setMinDxYear(2015); + options.setMaxDxYear(2020); + + // the birth date of a patient is based on the oldest age group of all its tumors (see BirthRule), so a tumor assigned to a + // younger age group used to be allowed to go decades before the requested min dx date + Map ageGroupMap = new HashMap<>(); + ageGroupMap.put(0, 7); + ageGroupMap.put(1, 2); + + Map context = new HashMap<>(); + context.put(CONTEXT_FLAG_AGE_GROUP_MAP, ageGroupMap); + context.put(CONTEXT_FLAG_MAX_AGE_GROUP, 7); + + for (int i = 0; i < 100; i++) { + Patient patient = new Patient(); + patient.addItem(new Item("dateOfBirthYear", "1945")); + patient.addItem(new Item("dateOfBirthMonth", "6")); + patient.addItem(new Item("dateOfBirthDay", "15")); + + LocalDate previousDxDate = null; + for (int tumorIdx = 0; tumorIdx < ageGroupMap.size(); tumorIdx++) { + context.put(CONTEXT_FLAG_CURRENT_TUMOR_INDEX, tumorIdx); + + Tumor tumor = new Tumor(); + _rule.execute(tumor, patient, options, context); + patient.addTumor(tumor); + + LocalDate dxDate = LocalDate.of( + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisYear")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisMonth")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisDay"))); + + Assert.assertFalse("Dx date before requested min dx date: " + dxDate, dxDate.isBefore(options.getMinDxDate())); + Assert.assertFalse("Dx date after requested max dx date: " + dxDate, dxDate.isAfter(options.getMaxDxDate())); + if (previousDxDate != null) + Assert.assertFalse("Dx date before previous tumor dx date: " + dxDate, dxDate.isBefore(previousDxDate)); + previousDxDate = dxDate; + } + } + } + + @Test + public void testExecuteWhenMinDxDateCannotBeReached() { + + // the min dx date defaults to ten years ago; combined with a max dx year in the past, that default range is inverted + NaaccrDataGeneratorOptions options = new NaaccrDataGeneratorOptions(); + options.setMaxDxYear(2005); + + Map ageGroupMap = new HashMap<>(); + ageGroupMap.put(0, 5); + + Map context = new HashMap<>(); + context.put(CONTEXT_FLAG_AGE_GROUP_MAP, ageGroupMap); + context.put(CONTEXT_FLAG_CURRENT_TUMOR_INDEX, 0); + context.put(CONTEXT_FLAG_MAX_AGE_GROUP, 5); + + Patient patient = new Patient(); + patient.addItem(new Item("dateOfBirthYear", "1940")); + patient.addItem(new Item("dateOfBirthMonth", "7")); + patient.addItem(new Item("dateOfBirthDay", "1")); + + Tumor tumor = new Tumor(); + _rule.execute(tumor, patient, options, context); + + LocalDate dxDate = LocalDate.of( + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisYear")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisMonth")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisDay"))); + + // the max dx date is the hard requirement, the dx date must fall in the ten years before it + Assert.assertFalse(dxDate.toString(), dxDate.isAfter(options.getMaxDxDate())); + Assert.assertFalse(dxDate.toString(), dxDate.isBefore(options.getMaxDxDate().minusYears(10))); + } }