diff --git a/.github/workflows/root-ci-config/buildconfig/global.txt b/.github/workflows/root-ci-config/buildconfig/global.txt index 0d83e5d28d4e6..e51fd0dea391b 100644 --- a/.github/workflows/root-ci-config/buildconfig/global.txt +++ b/.github/workflows/root-ci-config/buildconfig/global.txt @@ -80,7 +80,7 @@ sqlite=ON ssl=ON test_distrdf_dask=ON test_distrdf_pyspark=ON -test_roofit_hs3testsuite=ON +test_roofit_hs3testsuite=OFF test_tmva_sofie=ON testing=ON tmva-cpu=ON diff --git a/README/ReleaseNotes/v642/index.md b/README/ReleaseNotes/v642/index.md index 8eb3d7de6b291..c5131555ec1d6 100644 --- a/README/ReleaseNotes/v642/index.md +++ b/README/ReleaseNotes/v642/index.md @@ -175,6 +175,41 @@ Users are strongly encouraged to switch to the vectorized CPU backend if they ar If the vectorized backend does not work for a given use case, **please report it by opening an issue on the ROOT GitHub repository**. +### Removal of the legacy evaluation backend + +The `legacy` evaluation backend for likelihood and chi-square fits is removed. +It was superseded by the vectorized `cpu` backend, which is the default since +ROOT 6.32. After the removal of the constant term optimization (see above), the +legacy backend also had no performance-relevant feature left that would justify +its continued maintenance. + +Concretely, this means: + + * `RooFit::EvalBackend::Legacy()` and the corresponding enum value are + removed. Passing `RooFit::EvalBackend("legacy")` to `fitTo()`, + `createNLL()`, `chi2FitTo()` or `createChi2()` now throws an exception, and + so does the deprecated `RooFit::BatchMode("off")`. + * The implementation classes of the legacy test statistics are removed: + **RooNLLVar**, **RooChi2Var**, **RooAbsOptTestStatistic** and + **RooAbsTestStatistic**. Their headers were not part of the public + interface anymore since ROOT 6.32, but they were still installed for + backwards compatibility. + * The old multiprocessing mechanism of the legacy backend is removed as well, + consisting of the **RooRealMPFE** class and the underlying + **BidirMMapPipe**. The `RooFit::NumCPU()` command argument, which was + forking off one `RooRealMPFE` process per CPU, is now ignored in fits. For + parallelized fits, use the `RooFit::Parallelize()` argument that is based + on the new `RooFit::MultiProcess` framework (requires building ROOT with + `roofit_multiprocess=ON`). + * The `nll::name[pdf,data]` and `chi2::name[pdf,data]` expressions in the + `RooWorkspace::factory()` language are removed, since they instantiated the + removed classes directly. Use `RooAbsPdf::createNLL()` or + `RooAbsReal::createChi2()` instead. + * The `RooFit::TestStatistics::RooUnbinnedL` class now always evaluates with + the `RooFit::Evaluator` and its `evalBackend` constructor parameter + defaults to the `cpu` backend, like `RooFit::TestStatistics::NLLFactory`. + * The `roofit_legacy_eval_backend` CMake option is gone. + ### Default binning of RooFit variables changed to zero bins A freshly-constructed `RooRealVar` (or `RooErrorVar`) no longer has a default binning of 100 bins. diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 309e6cf0a0060..f83d21893229f 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -4,8 +4,6 @@ # For the licensing terms see $ROOTSYS/LICENSE. # For the list of contributors see $ROOTSYS/README/CREDITS. -set(roofit_legacy_eval_backend ON CACHE BOOL "" FORCE) - add_subdirectory(batchcompute) add_subdirectory(codegen) if (roofit_multiprocess) @@ -20,7 +18,7 @@ endif() add_subdirectory(roostats) add_subdirectory(histfactory) add_subdirectory(hs3) -if(roofit_legacy_eval_backend AND NOT MSVC) +if(NOT MSVC) add_subdirectory(xroofit) endif() diff --git a/roofit/histfactory/test/testHistFactory.cxx b/roofit/histfactory/test/testHistFactory.cxx index 594d7b73395b0..23a03e96d1c69 100644 --- a/roofit/histfactory/test/testHistFactory.cxx +++ b/roofit/histfactory/test/testHistFactory.cxx @@ -444,7 +444,7 @@ TEST_P(HFFixtureEval, Evaluation) const double systEps = 1e-6; const MakeModelMode makeModelMode = std::get<0>(GetParam()); - const bool useBatchMode = std::get<2>(GetParam()) != RooFit::EvalBackend::Legacy(); + const bool useBatchMode = true; RooHelpers::HijackMessageStream evalMessages(RooFit::INFO, RooFit::FastEvaluations); diff --git a/roofit/hs3/test/hs3testsuite_roofit_backend.py b/roofit/hs3/test/hs3testsuite_roofit_backend.py index 47fa56e77ded5..09d13c286c64d 100644 --- a/roofit/hs3/test/hs3testsuite_roofit_backend.py +++ b/roofit/hs3/test/hs3testsuite_roofit_backend.py @@ -54,8 +54,7 @@ def run_twice_delta_nll_scan(self, workspace, check: dict[str, Any]) -> list[flo with suppress_root_output(): nll = pdf.createNLL( data, - self.ROOT.RooFit.NumCPU(1), - self.ROOT.RooFit.EvalBackend("legacy"), + self.ROOT.RooFit.EvalBackend("cpu"), ) reference = float(nll.getVal()) values = [] diff --git a/roofit/roofit/test/testRooIntegralMorph.cxx b/roofit/roofit/test/testRooIntegralMorph.cxx index e2b45f2617aa7..c8a962e3990d2 100644 --- a/roofit/roofit/test/testRooIntegralMorph.cxx +++ b/roofit/roofit/test/testRooIntegralMorph.cxx @@ -251,8 +251,8 @@ TEST(RooIntegralMorph, AlphaCacheScan) /// Generating a toy dataset from the morph pdf and fitting it back must /// recover the true alpha, with the alpha cache enabled as in a realistic -/// fitting application, on both the legacy and cpu evaluation backends. This -/// covers the toy fit of the former stressRooFit test 705. +/// fitting application. This covers the toy fit of the former stressRooFit +/// test 705. TEST(RooIntegralMorph, GenerateAndFit) { GaussPolySetup s; @@ -270,7 +270,7 @@ TEST(RooIntegralMorph, GenerateAndFit) morph.setCacheAlpha(true); - for (std::string backend : {"legacy", "cpu"}) { + for (std::string backend : {"cpu"}) { alpha.setVal(0.5); alpha.setError(0.0); std::unique_ptr res{ diff --git a/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.cxx b/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.cxx index 1a9ccfac3d162..fc70fc419579a 100644 --- a/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.cxx +++ b/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.cxx @@ -420,35 +420,6 @@ void PDFTest::checkParameters() } } -void PDFTest::runBatchVsScalar(bool clonePDF) -{ - RooAbsPdf *pdfScalar = _pdf.get(); - RooAbsPdf *pdfBatch = _pdf.get(); - std::unique_ptr cleanupScalar; - std::unique_ptr cleanupBatch; - - if (clonePDF) { - pdfScalar = static_cast(_pdf->cloneTree("PDFForScalar")); - pdfBatch = static_cast(_pdf->cloneTree("PDFForScalar")); - - cleanupScalar.reset(pdfScalar); - cleanupBatch.reset(pdfBatch); - } - - resetParameters(); - auto resultScalar = runScalarFit(pdfScalar); - - resetParameters(); - auto resultBatch = runBatchFit(pdfBatch); - - resetParameters(); - - ASSERT_NE(resultScalar, nullptr); - ASSERT_NE(resultBatch, nullptr); - - EXPECT_TRUE(resultScalar->isIdentical(*resultBatch, _toleranceParameter, _toleranceCorrelation)); -} - std::unique_ptr PDFTest::runBatchFit(RooAbsPdf *pdf) { if (!_dataFit) @@ -493,50 +464,6 @@ std::unique_ptr PDFTest::runBatchFit(RooAbsPdf *pdf) return result; } -std::unique_ptr PDFTest::runScalarFit(RooAbsPdf *pdf) -{ - if (!_dataFit) - makeFitData(); - - kickParameters(); - makePlots(::testing::UnitTest::GetInstance()->current_test_info()->name() + std::string("_scalar_prefit")); - - std::unique_ptr pars{pdf->getParameters(*_dataFit)}; - pars->assign(_parameters); - - for (unsigned int index = 0; index < pars->size(); ++index) { - auto pdfParameter = static_cast((*pars)[index]); - auto origParameter = static_cast(_origParameters.find(*pdfParameter)); - if (!origParameter || origParameter->isConstant()) - continue; - - EXPECT_NE(pdfParameter->getVal(), origParameter->getVal()) - << "Parameter #" << index << "=" << pdfParameter->GetName() << " is identical after kicking."; - } - - if (HasFailure()) { - std::cout << "Pre-fit parameters:\n"; - _parameters.Print("V"); - std::cout << "Orig parameters:\n"; - _origParameters.Print("V"); - } - - MyTimer singleTimer("Fitting scalar mode " + _name); - std::unique_ptr result{pdf->fitTo(*_dataFit, RooFit::EvalBackend::Legacy(), RooFit::SumW2Error(false), - RooFit::PrintLevel(_printLevel), RooFit::Save(), - _multiProcess > 0 ? RooFit::NumCPU(_multiProcess) : RooCmdArg())}; - std::cout << singleTimer; - EXPECT_NE(result, nullptr); - if (!result) - return nullptr; - - EXPECT_EQ(result->status(), 0) << "[Scalar fit did not converge.]"; - - makePlots(::testing::UnitTest::GetInstance()->current_test_info()->name() + std::string("_scalar_postfit")); - - return result; -} - void PDFTestWeightedData::makeFitData() { PDFTest::makeFitData(); diff --git a/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.h b/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.h index 3914f9ba45d39..b2e4b83d74166 100644 --- a/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.h +++ b/roofit/roofit/test/vectorisedPDFs/VectorisedPDFTests.h @@ -51,12 +51,9 @@ class PDFTest : public ::testing::Test { void checkParameters(); - void runBatchVsScalar(bool clonePDF = false); std::unique_ptr runBatchFit(RooAbsPdf *pdf); - std::unique_ptr runScalarFit(RooAbsPdf *pdf); - std::unique_ptr _pdf; std::unique_ptr _dataUniform; std::unique_ptr _dataFit; @@ -166,32 +163,3 @@ class PDFTestWeightedData : public PDFTest { checkParameters(); \ } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - -/// Run a fit for batch and scalar code and compare results. -#define FIT_TEST_BATCH_VS_SCALAR(TEST_CLASS, TEST_NAME) \ - TEST_F(TEST_CLASS, TEST_NAME) { runBatchVsScalar(); } - -/// Run a fit for batch and scalar code and compare results. -/// Clone the PDFs before running the tests. This can run the test even if some internal state -/// is propagated / saved wrongly. -#define FIT_TEST_BATCH_VS_SCALAR_CLONE_PDF(TEST_CLASS, TEST_NAME) \ - TEST_F(TEST_CLASS, TEST_NAME) { runBatchVsScalar(true); } - -/// Run a fit in legacy mode and compare results to pre-fit values. -#define FIT_TEST_SCALAR(TEST_CLASS, TEST_NAME) \ - TEST_F(TEST_CLASS, TEST_NAME) \ - { \ - auto result = runScalarFit(_pdf.get()); \ - ASSERT_NE(result, nullptr); \ - checkParameters(); \ - } - -#else - -// Ignore legacy tests if legacy backend is not available -#define FIT_TEST_BATCH_VS_SCALAR(TEST_CLASS, TEST_NAME) -#define FIT_TEST_BATCH_VS_SCALAR_CLONE_PDF(TEST_CLASS, TEST_NAME) -#define FIT_TEST_SCALAR(TEST_CLASS, TEST_NAME) - -#endif diff --git a/roofit/roofit/test/vectorisedPDFs/testAddPdf.cxx b/roofit/roofit/test/vectorisedPDFs/testAddPdf.cxx index 0e2ebdf375090..5ac73e6383f1e 100644 --- a/roofit/roofit/test/vectorisedPDFs/testAddPdf.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testAddPdf.cxx @@ -67,10 +67,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestGaussPlusPoisson, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestGaussPlusPoisson, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGaussPlusPoisson, CompareFixedValuesNormLog) -FIT_TEST_SCALAR(TestGaussPlusPoisson, DISABLED_Scalar) // Save time FIT_TEST_BATCH(TestGaussPlusPoisson, DISABLED_Batch) // Save time -FIT_TEST_BATCH_VS_SCALAR(TestGaussPlusPoisson, CompareBatchScalar) - class TestGaussPlusGaussPlusExp : public PDFTest { protected: TestGaussPlusGaussPlusExp() : PDFTest("Gauss + Gauss + Exp") @@ -126,10 +123,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestGaussPlusGaussPlusExp, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestGaussPlusGaussPlusExp, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGaussPlusGaussPlusExp, CompareFixedValuesNormLog) -FIT_TEST_SCALAR(TestGaussPlusGaussPlusExp, DISABLED_Scalar) // Save time FIT_TEST_BATCH(TestGaussPlusGaussPlusExp, DISABLED_Batch) // Save time -FIT_TEST_BATCH_VS_SCALAR(TestGaussPlusGaussPlusExp, CompareBatchScalar) - #if !defined(_MSC_VER) // RooFit multiprocessing doesn't work on Windows class TestGaussPlusGaussPlusExp_MP : public TestGaussPlusGaussPlusExp { @@ -141,8 +135,5 @@ COMPARE_FIXED_VALUES_UNNORM(TestGaussPlusGaussPlusExp_MP, CompareFixedValuesUnno COMPARE_FIXED_VALUES_NORM(TestGaussPlusGaussPlusExp_MP, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGaussPlusGaussPlusExp_MP, CompareFixedValuesNormLog) -FIT_TEST_SCALAR(TestGaussPlusGaussPlusExp_MP, DISABLED_Scalar) // Save time FIT_TEST_BATCH(TestGaussPlusGaussPlusExp_MP, DISABLED_Batch) // Save time -FIT_TEST_BATCH_VS_SCALAR(TestGaussPlusGaussPlusExp_MP, CompareBatchScalar) - #endif // !defined(_MSC_VER) diff --git a/roofit/roofit/test/vectorisedPDFs/testArgusBG.cxx b/roofit/roofit/test/vectorisedPDFs/testArgusBG.cxx index c4b671b6d6db3..cc62d8dc4088d 100644 --- a/roofit/roofit/test/vectorisedPDFs/testArgusBG.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testArgusBG.cxx @@ -46,6 +46,4 @@ class TestArgus : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestArgus, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestArgus, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestArgus, CompareFixedNormLog) -FIT_TEST_SCALAR(TestArgus, RunScalar) FIT_TEST_BATCH(TestArgus, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestArgus, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testBernstein.cxx b/roofit/roofit/test/vectorisedPDFs/testBernstein.cxx index 70baa63ca96e5..f13c09030d0a7 100644 --- a/roofit/roofit/test/vectorisedPDFs/testBernstein.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testBernstein.cxx @@ -41,10 +41,7 @@ class TestBernstein2 : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestBernstein2, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestBernstein2, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestBernstein2, CompareFixedNormLog) -FIT_TEST_SCALAR(TestBernstein2, RunScalar) FIT_TEST_BATCH(TestBernstein2, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestBernstein2, CompareBatchScalar) - class TestBernstein5 : public PDFTest { protected: TestBernstein5() : PDFTest("Bernstein5") @@ -74,6 +71,4 @@ class TestBernstein5 : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestBernstein5, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestBernstein5, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestBernstein5, CompareFixedNormLog) -FIT_TEST_SCALAR(TestBernstein5, RunScalar) FIT_TEST_BATCH(TestBernstein5, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestBernstein5, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testBifurGauss.cxx b/roofit/roofit/test/vectorisedPDFs/testBifurGauss.cxx index fdbef9a1cc965..90e85d635d102 100644 --- a/roofit/roofit/test/vectorisedPDFs/testBifurGauss.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testBifurGauss.cxx @@ -45,6 +45,4 @@ class TestBifurGauss : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestBifurGauss, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestBifurGauss, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestBifurGauss, CompareFixedNormLog) -FIT_TEST_SCALAR(TestBifurGauss, RunScalar) FIT_TEST_BATCH(TestBifurGauss, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestBifurGauss, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testBreitWigner.cxx b/roofit/roofit/test/vectorisedPDFs/testBreitWigner.cxx index 2b0a01d18ebc8..e4e1ce6665b92 100644 --- a/roofit/roofit/test/vectorisedPDFs/testBreitWigner.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testBreitWigner.cxx @@ -42,6 +42,4 @@ class TestBreitWigner : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestBreitWigner, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestBreitWigner, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestBreitWigner, CompareFixedNormLog) -FIT_TEST_SCALAR(TestBreitWigner, RunScalar) FIT_TEST_BATCH(TestBreitWigner, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestBreitWigner, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testBukin.cxx b/roofit/roofit/test/vectorisedPDFs/testBukin.cxx index d4beced6deef7..77ba8b654bb70 100644 --- a/roofit/roofit/test/vectorisedPDFs/testBukin.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testBukin.cxx @@ -56,6 +56,4 @@ class TestBukin : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestBukin, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestBukin, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestBukin, CompareFixedNormLog) -FIT_TEST_SCALAR(TestBukin, RunScalar) FIT_TEST_BATCH(TestBukin, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestBukin, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testCBShape.cxx b/roofit/roofit/test/vectorisedPDFs/testCBShape.cxx index 95c86465b2049..4af280309277b 100644 --- a/roofit/roofit/test/vectorisedPDFs/testCBShape.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testCBShape.cxx @@ -48,6 +48,4 @@ class TestCBShape : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestCBShape, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestCBShape, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestCBShape, CompareFixedNormLog) -FIT_TEST_SCALAR(TestCBShape, RunScalar) FIT_TEST_BATCH(TestCBShape, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestCBShape, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testChebychev.cxx b/roofit/roofit/test/vectorisedPDFs/testChebychev.cxx index e872c087f0492..c3e428b6bfe17 100644 --- a/roofit/roofit/test/vectorisedPDFs/testChebychev.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testChebychev.cxx @@ -40,10 +40,7 @@ class TestChebychev2 : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestChebychev2, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestChebychev2, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestChebychev2, CompareFixedNormLog) -FIT_TEST_SCALAR(TestChebychev2, DISABLED_RunScalar) FIT_TEST_BATCH(TestChebychev2, DISABLED_RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestChebychev2, CompareBatchScalar) - class TestChebychev5 : public PDFTest { protected: TestChebychev5() : PDFTest("Chebychev5") @@ -76,6 +73,4 @@ class TestChebychev5 : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestChebychev5, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestChebychev5, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestChebychev5, CompareFixedNormLog) -FIT_TEST_SCALAR(TestChebychev5, RunScalar) FIT_TEST_BATCH(TestChebychev5, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestChebychev5, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testChiSquarePdf.cxx b/roofit/roofit/test/vectorisedPDFs/testChiSquarePdf.cxx index 0f1b3c9dbc13b..46ca4e3e14d17 100644 --- a/roofit/roofit/test/vectorisedPDFs/testChiSquarePdf.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testChiSquarePdf.cxx @@ -40,6 +40,4 @@ class TestChiSquarePdfinX : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestChiSquarePdfinX, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestChiSquarePdfinX, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestChiSquarePdfinX, CompareFixedNormLog) -FIT_TEST_SCALAR(TestChiSquarePdfinX, RunScalar) FIT_TEST_BATCH(TestChiSquarePdfinX, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestChiSquarePdfinX, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testCompatMode.cxx b/roofit/roofit/test/vectorisedPDFs/testCompatMode.cxx index ebff0331c9278..544688c066a7e 100644 --- a/roofit/roofit/test/vectorisedPDFs/testCompatMode.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testCompatMode.cxx @@ -51,10 +51,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestRooPolynomial, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestRooPolynomial, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestRooPolynomial, CompareFixedNormLog) -FIT_TEST_SCALAR(TestRooPolynomial, RunScalar) FIT_TEST_BATCH(TestRooPolynomial, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestRooPolynomial, CompareBatchScalar) - class RooNonVecGaussian : public RooAbsPdf { public: RooNonVecGaussian() {}; @@ -191,10 +188,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestNonVecGauss, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestNonVecGauss, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestNonVecGauss, CompareFixedNormLog) -FIT_TEST_SCALAR(TestNonVecGauss, RunScalar) FIT_TEST_BATCH(TestNonVecGauss, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestNonVecGauss, CompareBatchScalar) - class TestNonVecGaussWeighted : public PDFTestWeightedData { protected: TestNonVecGaussWeighted() : PDFTestWeightedData("GaussNoBatchesWithWeights", 50000) @@ -220,11 +214,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestNonVecGaussWeighted, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestNonVecGaussWeighted, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestNonVecGaussWeighted, CompareFixedNormLog) -FIT_TEST_SCALAR(TestNonVecGaussWeighted, - DISABLED_RunScalar) // Would need SumW2 error matrix correction, but no done in macro FIT_TEST_BATCH(TestNonVecGaussWeighted, DISABLED_RunBatch) // As above -FIT_TEST_BATCH_VS_SCALAR(TestNonVecGaussWeighted, CompareBatchScalar) - class TestNonVecGaussInMeanAndX : public PDFTest { protected: TestNonVecGaussInMeanAndX() : PDFTest("GaussNoBatches(x, mean)") @@ -249,6 +239,4 @@ COMPARE_FIXED_VALUES_UNNORM(TestNonVecGaussInMeanAndX, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestNonVecGaussInMeanAndX, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestNonVecGaussInMeanAndX, CompareFixedNormLog) -FIT_TEST_SCALAR(TestNonVecGaussInMeanAndX, RunScalar) FIT_TEST_BATCH(TestNonVecGaussInMeanAndX, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestNonVecGaussInMeanAndX, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testDstD0BG.cxx b/roofit/roofit/test/vectorisedPDFs/testDstD0BG.cxx index bdb1ce1dff38b..f4485fbcbf6a4 100644 --- a/roofit/roofit/test/vectorisedPDFs/testDstD0BG.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testDstD0BG.cxx @@ -41,6 +41,4 @@ class TestDstD0BG : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestDstD0BG, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestDstD0BG, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestDstD0BG, CompareFixedNormLog) -FIT_TEST_SCALAR(TestDstD0BG, RunScalar) FIT_TEST_BATCH(TestDstD0BG, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestDstD0BG, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testExponential.cxx b/roofit/roofit/test/vectorisedPDFs/testExponential.cxx index 87438e884e9fb..9333bddb700ba 100644 --- a/roofit/roofit/test/vectorisedPDFs/testExponential.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testExponential.cxx @@ -46,6 +46,4 @@ class TestExponential : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestExponential, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestExponential, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestExponential, CompareFixedValuesNormLog) -FIT_TEST_SCALAR(TestExponential, RunScalar) FIT_TEST_BATCH(TestExponential, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestExponential, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testGamma.cxx b/roofit/roofit/test/vectorisedPDFs/testGamma.cxx index e289edbeb3045..2ae5104ce16de 100644 --- a/roofit/roofit/test/vectorisedPDFs/testGamma.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testGamma.cxx @@ -47,6 +47,4 @@ class TestGamma : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestGamma, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestGamma, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGamma, CompareFixedNormLog) -FIT_TEST_SCALAR(TestGamma, RunScalar) FIT_TEST_BATCH(TestGamma, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestGamma, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testGauss.cxx b/roofit/roofit/test/vectorisedPDFs/testGauss.cxx index 069b9353087b5..8d12ed702f3e5 100644 --- a/roofit/roofit/test/vectorisedPDFs/testGauss.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testGauss.cxx @@ -48,10 +48,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestGauss, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestGauss, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGauss, CompareFixedNormLog) -FIT_TEST_SCALAR(TestGauss, RunScalar) FIT_TEST_BATCH(TestGauss, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestGauss, CompareBatchScalar) - #if !defined(_MSC_VER) // TODO: make TestGaussWeighted work on Windows class TestGaussWeighted : public PDFTestWeightedData { @@ -77,8 +74,6 @@ class TestGaussWeighted : public PDFTestWeightedData { FIT_TEST_BATCH(TestGaussWeighted, DISABLED_RunBatch) // Would need SumW2 or asymptotic error correction, but that's not in test macro. -FIT_TEST_BATCH_VS_SCALAR(TestGaussWeighted, CompareBatchScalar) - #endif // !defined(_MSC_VER) class TestGaussInMeanAndX : public PDFTest { @@ -107,8 +102,6 @@ COMPARE_FIXED_VALUES_NORM(TestGaussInMeanAndX, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGaussInMeanAndX, CompareFixedNormLog) FIT_TEST_BATCH(TestGaussInMeanAndX, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestGaussInMeanAndX, CompareBatchScalar) - class TestGaussWithFormulaParameters : public PDFTest { protected: TestGaussWithFormulaParameters() : PDFTest("Gauss(x, mean)") @@ -141,6 +134,4 @@ COMPARE_FIXED_VALUES_UNNORM(TestGaussWithFormulaParameters, FixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestGaussWithFormulaParameters, FixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestGaussWithFormulaParameters, FixedValuesNormLog) -FIT_TEST_SCALAR(TestGaussWithFormulaParameters, RunScalar) FIT_TEST_BATCH(TestGaussWithFormulaParameters, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestGaussWithFormulaParameters, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testGaussBinned.cxx b/roofit/roofit/test/vectorisedPDFs/testGaussBinned.cxx index 4840edd7c412d..ead935477ce44 100644 --- a/roofit/roofit/test/vectorisedPDFs/testGaussBinned.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testGaussBinned.cxx @@ -135,11 +135,6 @@ TEST_P(GaussBinnedFit, DISABLED_BatchFitFineBins) EXPECT_NEAR(s.getVal(), 4., s.getError()); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -INSTANTIATE_TEST_SUITE_P(RunFits, GaussBinnedFit, - testing::Values(RooFit::EvalBackend::Value::Legacy, RooFit::EvalBackend::Value::Cpu)); -#else INSTANTIATE_TEST_SUITE_P(RunFits, GaussBinnedFit, testing::Values(RooFit::EvalBackend::Value::Cpu)); -#endif // TODO Test a batch fit that uses categories once categories can be passed through the batch interface. diff --git a/roofit/roofit/test/vectorisedPDFs/testJohnson.cxx b/roofit/roofit/test/vectorisedPDFs/testJohnson.cxx index 1c33b1b99ee5f..075c19e7640ba 100644 --- a/roofit/roofit/test/vectorisedPDFs/testJohnson.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testJohnson.cxx @@ -55,10 +55,7 @@ COMPARE_FIXED_VALUES_UNNORM(TestJohnson, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestJohnson, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestJohnson, CompareFixedNormLog) -FIT_TEST_SCALAR(TestJohnson, FitScalar) FIT_TEST_BATCH(TestJohnson, FitBatch) -FIT_TEST_BATCH_VS_SCALAR(TestJohnson, FitBatchVsScalar) - class TestJohnsonInMassAndMu : public PDFTest { protected: TestJohnsonInMassAndMu() : PDFTest("Johnson in mass and mu") @@ -93,10 +90,7 @@ COMPARE_FIXED_VALUES_NORM(TestJohnsonInMassAndMu, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestJohnsonInMassAndMu, CompareFixedNormLog) // Is it clear that the fits can infer the value of lambda when generating in mu? -FIT_TEST_SCALAR(TestJohnsonInMassAndMu, DISABLED_FitScalar) FIT_TEST_BATCH(TestJohnsonInMassAndMu, DISABLED_FitBatch) -FIT_TEST_BATCH_VS_SCALAR(TestJohnsonInMassAndMu, CompareBatchScalar) - class TestJohnsonWithFormulaParameters : public PDFTest { protected: TestJohnsonWithFormulaParameters() : PDFTest("Johnson with formula") @@ -132,6 +126,4 @@ COMPARE_FIXED_VALUES_UNNORM(TestJohnsonWithFormulaParameters, CompareFixedUnnorm COMPARE_FIXED_VALUES_NORM(TestJohnsonWithFormulaParameters, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestJohnsonWithFormulaParameters, CompareFixedNormLog) -FIT_TEST_SCALAR(TestJohnsonWithFormulaParameters, RunScalar) FIT_TEST_BATCH(TestJohnsonWithFormulaParameters, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestJohnsonWithFormulaParameters, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testLandau.cxx b/roofit/roofit/test/vectorisedPDFs/testLandau.cxx index 72949f18d98cb..aea3fe85e5f9f 100644 --- a/roofit/roofit/test/vectorisedPDFs/testLandau.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testLandau.cxx @@ -45,10 +45,7 @@ class TestLandauEvil : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestLandauEvil, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestLandauEvil, CompareFixedValuesNorm) // No testing of logs because landau can return 0. -FIT_TEST_SCALAR(TestLandauEvil, DISABLED_RunScalar) // numerical integral presumably inaccurate FIT_TEST_BATCH(TestLandauEvil, DISABLED_RunBatch) // numerical integral presumably inaccurate -FIT_TEST_BATCH_VS_SCALAR(TestLandauEvil, CompareBatchScalar) - #endif // !defined(_MSC_VER) class TestLandau : public PDFTest { @@ -77,6 +74,4 @@ class TestLandau : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestLandau, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestLandau, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestLandau, CompareFixedNormLog) -FIT_TEST_SCALAR(TestLandau, DISABLED_RunScalar) // numerical integral presumably inaccurate FIT_TEST_BATCH(TestLandau, DISABLED_RunBatch) // numerical integral presumably inaccurate -FIT_TEST_BATCH_VS_SCALAR(TestLandau, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testLognormal.cxx b/roofit/roofit/test/vectorisedPDFs/testLognormal.cxx index 742066795f275..09386312d0aeb 100644 --- a/roofit/roofit/test/vectorisedPDFs/testLognormal.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testLognormal.cxx @@ -42,10 +42,7 @@ class TestLognormal : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestLognormal, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestLognormal, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestLognormal, CompareFixedNormLog) -FIT_TEST_SCALAR(TestLognormal, RunScalar) FIT_TEST_BATCH(TestLognormal, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestLognormal, CompareBatchScalar) - class TestLognormalInMeanAndX : public PDFTest { protected: TestLognormalInMeanAndX() : PDFTest("Lognormal(x, mean)") @@ -71,6 +68,4 @@ class TestLognormalInMeanAndX : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestLognormalInMeanAndX, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestLognormalInMeanAndX, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestLognormalInMeanAndX, CompareFixedNormLog) -FIT_TEST_SCALAR(TestLognormalInMeanAndX, RunScalar) FIT_TEST_BATCH(TestLognormalInMeanAndX, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestLognormalInMeanAndX, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testNestedPDFs.cxx b/roofit/roofit/test/vectorisedPDFs/testNestedPDFs.cxx index b4ddef035e963..4bda800c0b66b 100644 --- a/roofit/roofit/test/vectorisedPDFs/testNestedPDFs.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testNestedPDFs.cxx @@ -78,6 +78,4 @@ COMPARE_FIXED_VALUES_UNNORM(TestNestedPDFs, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestNestedPDFs, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestNestedPDFs, CompareFixedNormLog) -FIT_TEST_SCALAR(TestNestedPDFs, RunScalar) FIT_TEST_BATCH(TestNestedPDFs, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestNestedPDFs, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testNovosibirsk.cxx b/roofit/roofit/test/vectorisedPDFs/testNovosibirsk.cxx index 27fbe06d17483..b2ae170dc6c32 100644 --- a/roofit/roofit/test/vectorisedPDFs/testNovosibirsk.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testNovosibirsk.cxx @@ -45,6 +45,4 @@ class TestNovosibirsk : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestNovosibirsk, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestNovosibirsk, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestNovosibirsk, CompareFixedNormLog) -FIT_TEST_SCALAR(TestNovosibirsk, RunScalar) FIT_TEST_BATCH(TestNovosibirsk, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestNovosibirsk, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testPoisson.cxx b/roofit/roofit/test/vectorisedPDFs/testPoisson.cxx index 938069ae3e8c5..bdeee6f2040c1 100644 --- a/roofit/roofit/test/vectorisedPDFs/testPoisson.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testPoisson.cxx @@ -33,8 +33,6 @@ class TestPoisson : public PDFTest { }; FIT_TEST_BATCH(TestPoisson, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestPoisson, CompareBatchScalar) - class TestPoissonOddMean : public PDFTest { protected: TestPoissonOddMean() : PDFTest("PoissonOddMean") @@ -50,8 +48,6 @@ class TestPoissonOddMean : public PDFTest { }; FIT_TEST_BATCH(TestPoissonOddMean, DISABLED_RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestPoissonOddMean, CompareBatchScalar) - class TestPoissonOddMeanNoRounding : public PDFTest { protected: TestPoissonOddMeanNoRounding() : PDFTest("PoissonOddMeanNoRounding") @@ -75,6 +71,4 @@ COMPARE_FIXED_VALUES_NORM(TestPoissonOddMeanNoRounding, CompareFixedValuesNorm); COMPARE_FIXED_VALUES_NORM_LOG(TestPoissonOddMeanNoRounding, CompareFixedValuesNormLog); // Fit tests have a small bias. Unclear why. -FIT_TEST_SCALAR(TestPoissonOddMeanNoRounding, DISABLED_RunScalar) FIT_TEST_BATCH(TestPoissonOddMeanNoRounding, DISABLED_RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestPoissonOddMeanNoRounding, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testPolynomial.cxx b/roofit/roofit/test/vectorisedPDFs/testPolynomial.cxx index ad18ad9499c98..deecdcd0e78b6 100644 --- a/roofit/roofit/test/vectorisedPDFs/testPolynomial.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testPolynomial.cxx @@ -39,10 +39,7 @@ class TestPolynomial2 : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestPolynomial2, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestPolynomial2, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestPolynomial2, CompareFixedNormLog) -FIT_TEST_SCALAR(TestPolynomial2, DISABLED_RunScalar) // Save time FIT_TEST_BATCH(TestPolynomial2, DISABLED_RunBatch) // Save time -FIT_TEST_BATCH_VS_SCALAR(TestPolynomial2, CompareBatchScalar) - class TestPolynomial5 : public PDFTest { protected: TestPolynomial5() : PDFTest("Polynomial5") @@ -78,6 +75,4 @@ class TestPolynomial5 : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestPolynomial5, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestPolynomial5, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestPolynomial5, CompareFixedNormLog) -FIT_TEST_SCALAR(TestPolynomial5, DISABLED_RunScalar) // Save time FIT_TEST_BATCH(TestPolynomial5, DISABLED_RunBatch) // Save time -FIT_TEST_BATCH_VS_SCALAR(TestPolynomial5, CompareBatchScalar) diff --git a/roofit/roofit/test/vectorisedPDFs/testProductPdf.cxx b/roofit/roofit/test/vectorisedPDFs/testProductPdf.cxx index 8e7c03e433bc7..dd25a34f28d35 100644 --- a/roofit/roofit/test/vectorisedPDFs/testProductPdf.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testProductPdf.cxx @@ -55,8 +55,5 @@ COMPARE_FIXED_VALUES_UNNORM(TestProdPdf, CompareFixedValuesUnnorm) COMPARE_FIXED_VALUES_NORM(TestProdPdf, CompareFixedValuesNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestProdPdf, CompareFixedValuesNormLog) -FIT_TEST_SCALAR(TestProdPdf, FitScalar) FIT_TEST_BATCH(TestProdPdf, FitBatch) -FIT_TEST_BATCH_VS_SCALAR(TestProdPdf, FitBatchScalar) -FIT_TEST_BATCH_VS_SCALAR_CLONE_PDF(TestProdPdf, FitBatchScalarWithCloning) diff --git a/roofit/roofit/test/vectorisedPDFs/testVoigtian.cxx b/roofit/roofit/test/vectorisedPDFs/testVoigtian.cxx index ca9335ca46b96..2757909185b56 100644 --- a/roofit/roofit/test/vectorisedPDFs/testVoigtian.cxx +++ b/roofit/roofit/test/vectorisedPDFs/testVoigtian.cxx @@ -45,10 +45,7 @@ class TestVoigtian : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestVoigtian, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestVoigtian, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestVoigtian, CompareFixedNormLog) -FIT_TEST_SCALAR(TestVoigtian, RunScalar) FIT_TEST_BATCH(TestVoigtian, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestVoigtian, CompareBatchScalar) - class TestVoigtianInXandMean : public PDFTest { protected: TestVoigtianInXandMean() : PDFTest("Voigtian(x,m)") @@ -73,6 +70,4 @@ class TestVoigtianInXandMean : public PDFTest { COMPARE_FIXED_VALUES_UNNORM(TestVoigtianInXandMean, CompareFixedUnnorm) COMPARE_FIXED_VALUES_NORM(TestVoigtianInXandMean, CompareFixedNorm) COMPARE_FIXED_VALUES_NORM_LOG(TestVoigtianInXandMean, CompareFixedNormLog) -FIT_TEST_SCALAR(TestVoigtianInXandMean, RunScalar) FIT_TEST_BATCH(TestVoigtianInXandMean, RunBatch) -FIT_TEST_BATCH_VS_SCALAR(TestVoigtianInXandMean, CompareBatchScalar) diff --git a/roofit/roofitcore/CMakeLists.txt b/roofit/roofitcore/CMakeLists.txt index d8143917c8073..1302771498f1e 100644 --- a/roofit/roofitcore/CMakeLists.txt +++ b/roofit/roofitcore/CMakeLists.txt @@ -27,25 +27,6 @@ if(roofit_multiprocess) list(APPEND EXTRA_DEPENDENCIES Minuit2) endif() -if(roofit_legacy_eval_backend) - set(LegacyEvalBackendSources - src/BidirMMapPipe.cxx - src/RooAbsOptTestStatistic.cxx - src/RooAbsTestStatistic.cxx - src/RooChi2Var.cxx - src/RooNLLVar.cxx - src/RooRealMPFE.cxx - ) - set(LegacyEvalBackendHeaders - src/BidirMMapPipe.h - src/RooAbsOptTestStatistic.h - src/RooAbsTestStatistic.h - src/RooChi2Var.h - src/RooNLLVar.h - src/RooRealMPFE.h - ) -endif() - set (EXTRA_DICT_OPTS) if (runtime_cxxmodules AND WIN32) set (EXTRA_DICT_OPTS NO_CXXMODULE) @@ -464,7 +445,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitCore src/TestStatistics/RooUnbinnedL.cxx src/TestStatistics/buildLikelihood.cxx src/TestStatistics/SharedOffset.cxx - ${LegacyEvalBackendSources} ${RooFitMPTestStatisticsSources} DICTIONARY_OPTIONS "-writeEmptyRootPCM" @@ -488,10 +468,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitCore # The following definitions are PUBLIC so they can also be used in ROOT-internal tests -if(roofit_legacy_eval_backend) - target_compile_definitions(RooFitCore PUBLIC ROOFIT_LEGACY_EVAL_BACKEND) -endif() - if(roofit_multiprocess) target_compile_definitions(RooFitCore PUBLIC ROOFIT_MULTIPROCESS) endif() @@ -552,7 +528,6 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.23.0") src/RooRombergIntegrator.h src/RooBinIntegrator.h ${RooFitMPTestStatisticsHeaders} - ${LegacyEvalBackendHeaders} src/RooFit/BatchModeDataHelpers.h src/RooMinimizerFcn.h src/RooAbsNumGenerator.h diff --git a/roofit/roofitcore/inc/RooAbsArg.h b/roofit/roofitcore/inc/RooAbsArg.h index b53f27a8185b6..2deb0f91733d6 100644 --- a/roofit/roofitcore/inc/RooAbsArg.h +++ b/roofit/roofitcore/inc/RooAbsArg.h @@ -575,7 +575,6 @@ class RooAbsArg : public TNamed, public RooPrintable { friend class RooTreeDataStore; friend class RooVectorDataStore; friend class RooDataSet; - friend class RooRealMPFE; virtual void syncCache(const RooArgSet *nset = nullptr) = 0; virtual void copyCache(const RooAbsArg *source, bool valueOnly = false, bool setValDirty = true) = 0; diff --git a/roofit/roofitcore/inc/RooAbsData.h b/roofit/roofitcore/inc/RooAbsData.h index 78d4868c0d45a..355e051771938 100644 --- a/roofit/roofitcore/inc/RooAbsData.h +++ b/roofit/roofitcore/inc/RooAbsData.h @@ -331,7 +331,6 @@ class RooAbsData : public TNamed, public RooPrintable { // Constant term optimizer interface - friend class RooAbsOptTestStatistic ; // for access into copied dataset: friend class RooFit::TestStatistics::RooAbsL; diff --git a/roofit/roofitcore/inc/RooAbsPdf.h b/roofit/roofitcore/inc/RooAbsPdf.h index 18d154764ef2d..806918ac05b5e 100644 --- a/roofit/roofitcore/inc/RooAbsPdf.h +++ b/roofit/roofitcore/inc/RooAbsPdf.h @@ -340,7 +340,6 @@ class RooAbsPdf : public RooAbsReal { mutable RooFit::UniqueId::Value_t _normSetId = RooFit::UniqueId::nullval; /// paramTracker_; Section lastSection_ = {0, 0}; // used for cache together with the parameter tracker mutable ROOT::Math::KahanSum cachedResult_{0.}; diff --git a/roofit/roofitcore/inc/RooFit/TestStatistics/buildLikelihood.h b/roofit/roofitcore/inc/RooFit/TestStatistics/buildLikelihood.h index 19d6aa4d93f63..848229b7d1227 100644 --- a/roofit/roofitcore/inc/RooFit/TestStatistics/buildLikelihood.h +++ b/roofit/roofitcore/inc/RooFit/TestStatistics/buildLikelihood.h @@ -48,7 +48,7 @@ class NLLFactory { RooArgSet _externalConstraints; RooArgSet _globalObservables; std::string _globalObservablesTag; - RooFit::EvalBackend _evalBackend = RooFit::EvalBackend::Legacy(); + RooFit::EvalBackend _evalBackend = RooFit::EvalBackend::Cpu(); }; /// Delegating function to build a likelihood without additional arguments. diff --git a/roofit/roofitcore/inc/RooFormulaVar.h b/roofit/roofitcore/inc/RooFormulaVar.h index 047c6455bca04..8e471566dc474 100644 --- a/roofit/roofitcore/inc/RooFormulaVar.h +++ b/roofit/roofitcore/inc/RooFormulaVar.h @@ -69,8 +69,6 @@ class RooFormulaVar : public RooAbsReal { /// Dump the formula to stdout. void dumpFormula(); - double defaultErrorLevel() const override ; - void setBinning(const RooAbsRealLValue &obs, const RooAbsBinning &binning, bool checkFlatness = true); const RooAbsBinning *getBinning(const RooAbsRealLValue &obs) const; bool removeBinning(const RooAbsRealLValue &obs); diff --git a/roofit/roofitcore/inc/RooGlobalFunc.h b/roofit/roofitcore/inc/RooGlobalFunc.h index 2ad697c57b88a..76416693e1624 100644 --- a/roofit/roofitcore/inc/RooGlobalFunc.h +++ b/roofit/roofitcore/inc/RooGlobalFunc.h @@ -233,7 +233,7 @@ RooCmdArg EventRange(Int_t nStart, Int_t nStop) ; * \defgroup Fitting Arguments for fitting * @{ */ -// RooChi2Var::ctor / RooNLLVar arguments +// createChi2() / createNLL() arguments RooCmdArg Extended(bool flag=true) ; RooCmdArg DataError(Int_t) ; RooCmdArg DataError(std::string const&) ; @@ -257,13 +257,12 @@ RooCmdArg Optimize(Int_t flag = 2); class EvalBackend : public RooCmdArg { public: - enum class Value { Legacy, Cpu, Cuda, Codegen, CodegenNoGrad }; + enum class Value { Cpu, Cuda, Codegen, CodegenNoGrad }; EvalBackend(Value value); EvalBackend(std::string const &name); - static EvalBackend Legacy(); static EvalBackend Cpu(); static EvalBackend Cuda(); static EvalBackend Codegen(); diff --git a/roofit/roofitcore/src/BidirMMapPipe.cxx b/roofit/roofitcore/src/BidirMMapPipe.cxx deleted file mode 100644 index 4ad63fcc02d4c..0000000000000 --- a/roofit/roofitcore/src/BidirMMapPipe.cxx +++ /dev/null @@ -1,2008 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/** @file BidirMMapPipe.cxx - * - * implementation of BidirMMapPipe, a class which forks off a child process - * and serves as communications channel between parent and child - * - * @author Manuel Schiller - * @date 2013-07-07 - */ - -#ifndef _WIN32 - -#include "BidirMMapPipe.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#define BEGIN_NAMESPACE_ROOFIT namespace RooFit { -#define END_NAMESPACE_ROOFIT } - -BEGIN_NAMESPACE_ROOFIT - -/// namespace for implementation details of BidirMMapPipe -namespace BidirMMapPipe_impl { - /** @brief exception to throw if low-level OS calls go wrong - * - * @author Manuel Schiller - * @date 2013-07-07 - */ - class BidirMMapPipeException : public std::exception - { - private: - enum { - s_sz = 256 ///< length of buffer - }; - char m_buf[s_sz]; ///< buffer containing the error message - - /// for the POSIX version of strerror_r - static int dostrerror_r(int err, char* buf, std::size_t sz, - int (*f)(int, char*, std::size_t)) - { return f(err, buf, sz); } - /// for the GNU version of strerror_r - static int dostrerror_r(int, char*, std::size_t, - char* (*f)(int, char*, std::size_t)); - public: - /// constructor taking error code, hint on operation (msg) - BidirMMapPipeException(const char* msg, int err); - /// return a destcription of what went wrong - const char* what() const noexcept override { return m_buf; } - }; - - BidirMMapPipeException::BidirMMapPipeException(const char* msg, int err) - { - std::size_t msgsz = std::strlen(msg); - if (msgsz) { - msgsz = std::min(msgsz, std::size_t(s_sz)); - std::copy(msg, msg + msgsz, m_buf); - if (msgsz < s_sz) { m_buf[msgsz] = ':'; ++msgsz; } - if (msgsz < s_sz) { m_buf[msgsz] = ' '; ++msgsz; } - } - if (msgsz < s_sz) { - // UGLY: GNU and POSIX cannot agree on prototype and behaviour, so - // have to sort it out with overloads - dostrerror_r(err, &m_buf[msgsz], s_sz - msgsz, ::strerror_r); - } - m_buf[s_sz - 1] = 0; // enforce zero-termination - } - - int BidirMMapPipeException::dostrerror_r(int err, char* buf, - std::size_t sz, char* (*f)(int, char*, std::size_t)) - { - buf[0] = 0; - char *tmp = f(err, buf, sz); - if (tmp && tmp != buf) { - std::strncpy(buf, tmp, sz); - buf[sz - 1] = 0; - if (std::strlen(tmp) > sz - 1) return ERANGE; - } - return 0; - } - - /** @brief class representing the header structure in an mmapped page - * - * @author Manuel Schiller - * @date 2013-07-07 - * - * contains a field to put pages into a linked list, a field for the size - * of the data being transmitted, and a field for the position until which - * the data has been read - */ - class Page - { - private: - // use as small a data type as possible to maximise payload area - // of pages - short m_next = 0; ///< next page in list (in pagesizes) - unsigned short m_size = 0; ///< size of payload (in bytes) - unsigned short m_pos = 0; ///< index of next byte in payload area - public: - /// constructor - Page() - { - // check that short is big enough - must be done at runtime - // because the page size is not known until runtime - assert(std::numeric_limits::max() >= - PageChunk::pagesize()); - } - /// copy construction forbidden - Page(const Page &) = delete; - /// assignment forbidden - Page &operator=(const Page &) = delete; - /// set pointer to next page - void setNext(const Page* p); - /// return pointer to next page - Page* next() const; - /// return reference to size field - unsigned short& size() { return m_size; } - /// return size (of payload data) - unsigned size() const { return m_size; } - /// return reference to position field - unsigned short& pos() { return m_pos; } - /// return position - unsigned pos() const { return m_pos; } - /// return pointer to first byte in payload data area of page - inline unsigned char* begin() const - { return reinterpret_cast(const_cast(this)) - + sizeof(Page); } - /// return pointer to first byte in payload data area of page - inline unsigned char* end() const - { return reinterpret_cast(const_cast(this)) - + PageChunk::pagesize(); } - /// return the capacity of the page - static unsigned capacity() - { return PageChunk::pagesize() - sizeof(Page); } - /// true if page empty - bool empty() const { return !m_size; } - /// true if page partially filled - bool filled() const { return !empty(); } - /// free space left (to be written to) - unsigned free() const { return capacity() - m_size; } - /// bytes remaining to be read - unsigned remaining() const { return m_size - m_pos; } - /// true if page completely full - bool full() const { return !free(); } - }; - - void Page::setNext(const Page* p) - { - if (!p) { - m_next = 0; - } else { - const char* p1 = reinterpret_cast(this); - const char* p2 = reinterpret_cast(p); - std::ptrdiff_t tmp = p2 - p1; - // difference must be divisible by page size - assert(!(tmp % PageChunk::pagesize())); - tmp /= static_cast(PageChunk::pagesize()); - m_next = tmp; - // no truncation when saving in a short - assert(m_next == tmp); - // final check: next() must return p - assert(next() == p); - } - } - - Page* Page::next() const - { - if (!m_next) return nullptr; - char* ptmp = reinterpret_cast(const_cast(this)); - ptmp += std::ptrdiff_t(m_next) * PageChunk::pagesize(); - return reinterpret_cast(ptmp); - } - - /** @brief class representing a page pool - * - * @author Manuel Schiller - * @date 2013-07-24 - * - * pool of mmapped pages (on systems which support it, on all others, the - * functionality is emulated with dynamically allocated memory) - * - * in most operating systems there is a limit to how many mappings any one - * process is allowed to request; for this reason, we mmap a relatively - * large amount up front, and then carve off little pieces as we need them - * - * Moreover, some systems have too large a physical page size in their MMU - * for the code to handle (we want offsets and lengths to fit into 16 - * bits), so we carve such big physical pages into smaller logical Pages - * if needed. The largest logical page size is currently 16 KiB. - */ - class PagePool { - private: - /// convenience typedef - typedef BidirMMapPipeException Exception; - - enum { - minsz = 7, ///< minimum chunk size (just below 1 << minsz bytes) - maxsz = 20, ///< maximum chunk size (just below 1 << maxsz bytes) - szincr = 1 ///< size class increment (sz = 1 << (minsz + k * szincr)) - }; - /// a chunk of memory in the pool - typedef BidirMMapPipe_impl::PageChunk Chunk; - /// list of chunks - typedef std::list ChunkList; - - friend class BidirMMapPipe_impl::PageChunk; - public: - /// convenience typedef - typedef PageChunk::MMapVariety MMapVariety; - /// constructor - PagePool(unsigned nPagesPerGroup); - /// destructor - ~PagePool(); - /// pop a free element out of the pool - Pages pop(); - - /// return (logical) page size of the system - static unsigned pagesize() { return PageChunk::pagesize(); } - /// return variety of mmap supported on the system - static MMapVariety mmapVariety() - { return PageChunk::mmapVariety(); } - - /// return number of pages per group (ie. as returned by pop()) - unsigned nPagesPerGroup() const { return m_nPgPerGrp; } - - /// zap the pool (unmap all but Pages p) - void zap(Pages& p); - - private: - /// list of chunks used by the pool - ChunkList m_chunks; - /// list of chunks used by the pool which are not full - ChunkList m_freelist; - /// chunk size map (histogram of chunk sizes) - unsigned m_szmap[(maxsz - minsz) / szincr]; - /// current chunk size - int m_cursz = minsz; - /// page group size - unsigned m_nPgPerGrp; - - /// adjust _cursz to current largest block - void updateCurSz(int sz, int incr); - /// find size of next chunk to allocate (in a hopefully smart way) - int nextChunkSz() const; - /// release a chunk - void putOnFreeList(Chunk* chunk); - /// release a chunk - void release(Chunk* chunk); - }; - - Pages::Pages(PageChunk* parent, Page* pages, unsigned npg) : - m_pimpl(new impl) - { - assert(npg < 256); - m_pimpl->m_parent = parent; - m_pimpl->m_pages = pages; - m_pimpl->m_refcnt = 1; - m_pimpl->m_npages = npg; - /// initialise pages - for (unsigned i = 0; i < m_pimpl->m_npages; ++i) new(page(i)) Page(); - } - - unsigned PageChunk::s_physpgsz = PageChunk::getPageSize(); - unsigned PageChunk::s_pagesize = std::min(PageChunk::s_physpgsz, 16384u); - PageChunk::MMapVariety PageChunk::s_mmapworks = PageChunk::Unknown; - - Pages::~Pages() - { - if (m_pimpl && !--(m_pimpl->m_refcnt)) { - if (m_pimpl->m_parent) m_pimpl->m_parent->push(*this); - delete m_pimpl; - } - } - - Pages::Pages(const Pages& other) : - m_pimpl(other.m_pimpl) - { ++(m_pimpl->m_refcnt); } - - Pages& Pages::operator=(const Pages& other) - { - if (&other == this) return *this; - if (!--(m_pimpl->m_refcnt)) { - if (m_pimpl->m_parent) m_pimpl->m_parent->push(*this); - delete m_pimpl; - } - m_pimpl = other.m_pimpl; - ++(m_pimpl->m_refcnt); - return *this; - } - - unsigned Pages::pagesize() { return PageChunk::pagesize(); } - - Page* Pages::page(unsigned pgno) const - { - assert(pgno < m_pimpl->m_npages); - unsigned char* pptr = - reinterpret_cast(m_pimpl->m_pages); - pptr += pgno * pagesize(); - return reinterpret_cast(pptr); - } - - unsigned Pages::pageno(Page* p) const - { - const unsigned char* pptr = - reinterpret_cast(p); - const unsigned char* bptr = - reinterpret_cast(m_pimpl->m_pages); - assert(0 == ((pptr - bptr) % pagesize())); - const unsigned nr = (pptr - bptr) / pagesize(); - assert(nr < m_pimpl->m_npages); - return nr; - } - - unsigned PageChunk::getPageSize() - { - // find out page size of system - long pgsz = sysconf(_SC_PAGESIZE); - if (-1 == pgsz) throw Exception("sysconf", errno); - if (pgsz > 512 && pgsz > long(sizeof(Page))) - return pgsz; - - // in case of failure or implausible value, use a safe default: 4k - // page size, and do not try to mmap - s_mmapworks = Copy; - return 1 << 12; - } - - PageChunk::PageChunk(PagePool* parent, - unsigned length, unsigned nPgPerGroup) : - m_begin(dommap(length)), - m_end(reinterpret_cast( - reinterpret_cast(m_begin) + length)), - m_parent(parent), m_nPgPerGrp(nPgPerGroup), m_nUsedGrp(0) - { - // ok, push groups of pages onto freelist here - unsigned char* p = reinterpret_cast(m_begin); - unsigned char* pend = reinterpret_cast(m_end); - while (p < pend) { - m_freelist.push_back(reinterpret_cast(p)); - p += nPgPerGroup * PagePool::pagesize(); - } - } - - PageChunk::~PageChunk() - { - if (m_parent) assert(empty()); - if (m_begin) domunmap(m_begin, len()); - } - - bool PageChunk::contains(const Pages& p) const - { return p.m_pimpl->m_parent == this; } - - Pages PageChunk::pop() - { - assert(!m_freelist.empty()); - void* p = m_freelist.front(); - m_freelist.pop_front(); - ++m_nUsedGrp; - return Pages(this, reinterpret_cast(p), m_nPgPerGrp); - } - - void PageChunk::push(const Pages& p) - { - assert(contains(p)); - bool wasempty = m_freelist.empty(); - m_freelist.push_front(reinterpret_cast(p[0u])); - --m_nUsedGrp; - if (m_parent) { - // notify parent if we need to be put on the free list again - if (wasempty) m_parent->putOnFreeList(this); - // notify parent if we're empty - if (empty()) return m_parent->release(this); - } - } - - void* PageChunk::dommap(unsigned len) - { - assert(len && 0 == (len % s_physpgsz)); - // ok, the idea here is to try the different methods of mmapping, and - // choose the first one that works. we have four flavours: - // 1 - anonymous mmap (best) - // 2 - mmap of /dev/zero (about as good as anonymous mmap, but a tiny - // bit more tedious to set up, since you need to open/close a - // device file) - // 3 - mmap of a temporary file (very tedious to set up - need to - // create a temporary file, delete it, make the underlying storage - // large enough, then mmap the fd and close it) - // 4 - if all those fail, we malloc the buffers, and copy the data - // through the OS (then we're no better than normal pipes) - static bool msgprinted = false; - if (Anonymous == s_mmapworks || Unknown == s_mmapworks) { -#if defined(MAP_ANONYMOUS) -#undef MYANONFLAG -#define MYANONFLAG MAP_ANONYMOUS -#elif defined(MAP_ANON) -#undef MYANONFLAG -#define MYANONFLAG MAP_ANON -#else -#undef MYANONFLAG -#endif -#ifdef MYANONFLAG - void* retVal = ::mmap(nullptr, len, PROT_READ | PROT_WRITE, - MYANONFLAG | MAP_SHARED, -1, 0); - if (MAP_FAILED == retVal) { - if (Anonymous == s_mmapworks) throw Exception("mmap", errno); - } else { - assert(Unknown == s_mmapworks || Anonymous == s_mmapworks); - s_mmapworks = Anonymous; - if (BidirMMapPipe::debugflag() && !msgprinted) { - std::cerr << " INFO: In " << __func__ << " (" << - __FILE__ << ", line " << __LINE__ << - "): anonymous mmapping works, excellent!" << - std::endl; - msgprinted = true; - } - return retVal; - } -#endif -#undef MYANONFLAG - } - if (DevZero == s_mmapworks || Unknown == s_mmapworks) { - // ok, no anonymous mappings supported directly, so try to map - // /dev/zero which has much the same effect on many systems - int fd = ::open("/dev/zero", O_RDWR); - if (-1 == fd) - throw Exception("open /dev/zero", errno); - void* retVal = ::mmap(nullptr, len, - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (MAP_FAILED == retVal) { - int errsv = errno; - ::close(fd); - if (DevZero == s_mmapworks) throw Exception("mmap", errsv); - } else { - assert(Unknown == s_mmapworks || DevZero == s_mmapworks); - s_mmapworks = DevZero; - } - if (-1 == ::close(fd)) - throw Exception("close /dev/zero", errno); - if (BidirMMapPipe::debugflag() && !msgprinted) { - std::cerr << " INFO: In " << __func__ << " (" << __FILE__ << - ", line " << __LINE__ << "): mmapping /dev/zero works, " - "very good!" << std::endl; - msgprinted = true; - } - return retVal; - } - if (FileBacked == s_mmapworks || Unknown == s_mmapworks) { - std::string tmpPath = gSystem->TempDirectory(); - std::string name = tmpPath + "/roofit_BidirMMapPipe-XXXXXX"; - int fd; - // open temp file - if (-1 == (fd = ::mkstemp(const_cast(name.c_str())))) throw Exception("mkstemp", errno); - // remove it, but keep fd open - if (-1 == ::unlink(name.c_str())) { - int errsv = errno; - ::close(fd); - throw Exception("unlink", errsv); - } - // make it the right size: lseek - if (-1 == ::lseek(fd, len - 1, SEEK_SET)) { - int errsv = errno; - ::close(fd); - throw Exception("lseek", errsv); - } - // make it the right size: write a byte - if (1 != ::write(fd, name.c_str(), 1)) { - int errsv = errno; - ::close(fd); - throw Exception("write", errsv); - } - // do mmap - void* retVal = ::mmap(nullptr, len, - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (MAP_FAILED == retVal) { - int errsv = errno; - ::close(fd); - if (FileBacked == s_mmapworks) throw Exception("mmap", errsv); - } else { - assert(Unknown == s_mmapworks || FileBacked == s_mmapworks); - s_mmapworks = FileBacked; - } - if (-1 == ::close(fd)) { - int errsv = errno; - ::munmap(retVal, len); - throw Exception("close", errsv); - } - if (BidirMMapPipe::debugflag() && !msgprinted) { - std::cerr << " INFO: In " << __func__ << " (" << __FILE__ << - ", line " << __LINE__ << "): mmapping temporary files " - "works, good!" << std::endl; - msgprinted = true; - } - return retVal; - } - if (Copy == s_mmapworks || Unknown == s_mmapworks) { - // fallback solution: mmap does not work on this OS (or does not - // work for what we want to use it), so use a normal buffer of - // memory instead, and collect data in that buffer - this needs an - // additional write/read to/from the pipe(s), but there you go... - if (BidirMMapPipe::debugflag() && !msgprinted) { - std::cerr << "WARNING: In " << __func__ << " (" << __FILE__ << - ", line " << __LINE__ << "): anonymous mmapping of " - "shared buffers failed, falling back to read/write on " - " pipes!" << std::endl; - msgprinted = true; - } - s_mmapworks = Copy; - void* retVal = std::malloc(len); - if (!retVal) throw Exception("malloc", errno); - return retVal; - } - // should never get here - assert(false); - return nullptr; - } - - void PageChunk::domunmap(void* addr, unsigned len) - { - assert(len && 0 == (len % s_physpgsz)); - if (addr) { - assert(Unknown != s_mmapworks); - if (Copy != s_mmapworks) { - if (-1 == ::munmap(addr, len)) - throw Exception("munmap", errno); - } else { - std::free(addr); - } - } - } - - void PageChunk::zap(Pages& p) - { - // try to mprotect the other bits of the pool with no access... - // we'd really like a version of mremap here that can unmap all the - // other pages in the chunk, but that does not exist, so we protect - // the other pages in this chunk such that they may neither be read, - // written nor executed, only the pages we're interested in for - // communications stay readable and writable - // - // if an OS does not support changing the protection of a part of an - // mmapped area, the mprotect calls below should just fail and not - // change any protection, so we're a little less safe against - // corruption, but everything should still work - if (Copy != s_mmapworks) { - unsigned char* p0 = reinterpret_cast(m_begin); - unsigned char* p1 = reinterpret_cast(p[0u]); - unsigned char* p2 = p1 + p.npages() * s_physpgsz; - unsigned char* p3 = reinterpret_cast(m_end); - if (p1 != p0) ::mprotect(p0, p1 - p0, PROT_NONE); - if (p2 != p3) ::mprotect(p2, p3 - p2, PROT_NONE); - } - m_parent = nullptr; - m_freelist.clear(); - m_nUsedGrp = 1; - p.m_pimpl->m_parent = nullptr; - m_begin = m_end = nullptr; - // commit suicide - delete this; - } - - PagePool::PagePool(unsigned nPgPerGroup) : m_nPgPerGrp(nPgPerGroup) - { - // if logical and physical page size differ, we may have to adjust - // m_nPgPerGrp to make things fit - if (PageChunk::pagesize() != PageChunk::physPgSz()) { - const unsigned mult = - PageChunk::physPgSz() / PageChunk::pagesize(); - const unsigned desired = nPgPerGroup * PageChunk::pagesize(); - // round up to to next physical page boundary - const unsigned actual = mult * - (desired / mult + bool(desired % mult)); - const unsigned newPgPerGrp = actual / PageChunk::pagesize(); - if (BidirMMapPipe::debugflag()) { - std::cerr << " INFO: In " << __func__ << " (" << - __FILE__ << ", line " << __LINE__ << - "): physical page size " << PageChunk::physPgSz() << - ", subdividing into logical pages of size " << - PageChunk::pagesize() << ", adjusting nPgPerGroup " << - m_nPgPerGrp << " -> " << newPgPerGrp << - std::endl; - } - assert(newPgPerGrp >= m_nPgPerGrp); - m_nPgPerGrp = newPgPerGrp; - } - std::fill(m_szmap, m_szmap + ((maxsz - minsz) / szincr), 0); - } - - PagePool::~PagePool() - { - m_freelist.clear(); - for (ChunkList::iterator it = m_chunks.begin(); m_chunks.end() != it; ++it) - delete *it; - m_chunks.clear(); - } - - void PagePool::zap(Pages& p) - { - // unmap all pages but those pointed to by p - m_freelist.clear(); - for (ChunkList::iterator it = m_chunks.begin(); m_chunks.end() != it; ++it) { - if ((*it)->contains(p)) { - (*it)->zap(p); - } else { - delete *it; - } - } - m_chunks.clear(); - std::fill(m_szmap, m_szmap + ((maxsz - minsz) / szincr), 0); - m_cursz = minsz; - } - - Pages PagePool::pop() - { - if (m_freelist.empty()) { - // allocate and register new chunk and put it on the freelist - const int sz = nextChunkSz(); - Chunk *c = new Chunk(this, - sz * m_nPgPerGrp * pagesize(), m_nPgPerGrp); - m_chunks.push_front(c); - m_freelist.push_back(c); - updateCurSz(sz, +1); - } - // get free element from first chunk on _freelist - Chunk* c = m_freelist.front(); - Pages p(c->pop()); - // full chunks are removed from _freelist - if (c->full()) m_freelist.pop_front(); - return p; - } - - void PagePool::release(PageChunk* chunk) - { - assert(chunk->empty()); - // find chunk on freelist and remove - ChunkList::iterator it = std::find( - m_freelist.begin(), m_freelist.end(), chunk); - if (m_freelist.end() == it) - throw Exception("PagePool::release(PageChunk*)", EINVAL); - m_freelist.erase(it); - // find chunk in m_chunks and remove - it = std::find(m_chunks.begin(), m_chunks.end(), chunk); - if (m_chunks.end() == it) - throw Exception("PagePool::release(PageChunk*)", EINVAL); - m_chunks.erase(it); - const unsigned sz = chunk->len() / (pagesize() * m_nPgPerGrp); - delete chunk; - updateCurSz(sz, -1); - } - - void PagePool::putOnFreeList(PageChunk* chunk) - { - assert(!chunk->full()); - m_freelist.push_back(chunk); - } - - void PagePool::updateCurSz(int sz, int incr) - { - m_szmap[(sz - minsz) / szincr] += incr; - m_cursz = minsz; - for (int i = (maxsz - minsz) / szincr; i--; ) { - if (m_szmap[i]) { - m_cursz += i * szincr; - break; - } - } - } - - int PagePool::nextChunkSz() const - { - // no chunks with space available, figure out chunk size - int sz = m_cursz; - if (m_chunks.empty()) { - // if we start allocating chunks, we start from minsz - sz = minsz; - } else { - if (minsz >= sz) { - // minimal sized chunks are always grown - sz = minsz + szincr; - } else { - if (1 != m_chunks.size()) { - // if we have more than one completely filled chunk, grow - sz += szincr; - } else { - // just one chunk left, try shrinking chunk size - sz -= szincr; - } - } - } - // clamp size to allowed range - if (sz > maxsz) sz = maxsz; - if (sz < minsz) sz = minsz; - return sz; - } -} - -// static BidirMMapPipe members -pthread_mutex_t BidirMMapPipe::s_openpipesmutex = PTHREAD_MUTEX_INITIALIZER; -std::list BidirMMapPipe::s_openpipes; -BidirMMapPipe_impl::PagePool* BidirMMapPipe::s_pagepool = nullptr; -unsigned BidirMMapPipe::s_pagepoolrefcnt = 0; -int BidirMMapPipe::s_debugflag = 0; - -BidirMMapPipe_impl::PagePool& BidirMMapPipe::pagepool() -{ - if (!s_pagepool) - s_pagepool = new BidirMMapPipe_impl::PagePool(TotPages); - return *s_pagepool; -} - -void BidirMMapPipe::teardownall(void) -{ - pthread_mutex_lock(&s_openpipesmutex); - while (!s_openpipes.empty()) { - BidirMMapPipe *p = s_openpipes.front(); - pthread_mutex_unlock(&s_openpipesmutex); - if (p->m_childPid) kill(p->m_childPid, SIGTERM); - p->doClose(true, true); - pthread_mutex_lock(&s_openpipesmutex); - } - pthread_mutex_unlock(&s_openpipesmutex); -} - -BidirMMapPipe::BidirMMapPipe(const BidirMMapPipe&) : - m_pages(pagepool().pop()) -{ - // free pages again - { BidirMMapPipe_impl::Pages p; p.swap(m_pages); } - if (!s_pagepoolrefcnt) { - delete s_pagepool; - s_pagepool = nullptr; - } -} - -BidirMMapPipe::BidirMMapPipe(bool useExceptions, bool useSocketpair) : - m_pages(pagepool().pop()), m_busylist(nullptr), m_freelist(nullptr), m_dirtylist(nullptr), - m_inpipe(-1), m_outpipe(-1), m_flags(failbit), m_childPid(0), - m_parentPid(::getpid()) - -{ - ++s_pagepoolrefcnt; - assert(0 < TotPages && 0 == (TotPages & 1) && TotPages <= 256); - int fds[4] = { -1, -1, -1, -1 }; - int myerrno; - static bool firstcall = true; - if (useExceptions) m_flags |= exceptionsbit; - - try { - if (firstcall) { - firstcall = false; - // register a cleanup handler to make sure all BidirMMapPipes are torn - // down, and child processes are sent a SIGTERM - if (0 != atexit(BidirMMapPipe::teardownall)) - throw Exception("atexit", errno); - } - - // build free lists - for (unsigned i = 1; i < TotPages; ++i) - m_pages[i - 1]->setNext(m_pages[i]); - m_pages[PagesPerEnd - 1]->setNext(nullptr); - if (!useSocketpair) { - // create pipes - if (0 != ::pipe(&fds[0])) throw Exception("pipe", errno); - if (0 != ::pipe(&fds[2])) throw Exception("pipe", errno); - } else { - if (0 != ::socketpair(AF_UNIX, SOCK_STREAM, 0, &fds[0])) - throw Exception("socketpair", errno); - } - // fork the child - pthread_mutex_lock(&s_openpipesmutex); - char c; - switch ((m_childPid = ::fork())) { - case -1: // error in fork() - myerrno = errno; - pthread_mutex_unlock(&s_openpipesmutex); - m_childPid = 0; - throw Exception("fork", myerrno); - case 0: // child - // put the ends in the right place - if (-1 != fds[2]) { - // pair of pipes - if (-1 == ::close(fds[0]) || (-1 == ::close(fds[3]))) { - myerrno = errno; - pthread_mutex_unlock(&s_openpipesmutex); - throw Exception("close", myerrno); - } - fds[0] = fds[3] = -1; - m_outpipe = fds[1]; - m_inpipe = fds[2]; - } else { - // socket pair - if (-1 == ::close(fds[0])) { - myerrno = errno; - pthread_mutex_unlock(&s_openpipesmutex); - throw Exception("close", myerrno); - } - fds[0] = -1; - m_inpipe = m_outpipe = fds[1]; - } - // close other pipes our parent may have open - we have no business - // reading from/writing to those... - for (std::list::iterator it = s_openpipes.begin(); - s_openpipes.end() != it; ) { - BidirMMapPipe* p = *it; - it = s_openpipes.erase(it); - p->doClose(true, true); - } - pagepool().zap(m_pages); - s_pagepoolrefcnt = 0; - delete s_pagepool; - s_pagepool = nullptr; - s_openpipes.push_front(this); - pthread_mutex_unlock(&s_openpipesmutex); - // ok, put our pages on freelist - m_freelist = m_pages[PagesPerEnd]; - // handshake with other end (to make sure it's alive)... - c = 'C'; // ...hild - if (1 != xferraw(m_outpipe, &c, 1, ::write)) - throw Exception("handshake: xferraw write", EPIPE); - if (1 != xferraw(m_inpipe, &c, 1, ::read)) - throw Exception("handshake: xferraw read", EPIPE); - if ('P' != c) throw Exception("handshake", EPIPE); - break; - default: // parent - // put the ends in the right place - if (-1 != fds[2]) { - // pair of pipes - if (-1 == ::close(fds[1]) || -1 == ::close(fds[2])) { - myerrno = errno; - pthread_mutex_unlock(&s_openpipesmutex); - throw Exception("close", myerrno); - } - fds[1] = fds[2] = -1; - m_outpipe = fds[3]; - m_inpipe = fds[0]; - } else { - // socketpair - if (-1 == ::close(fds[1])) { - myerrno = errno; - pthread_mutex_unlock(&s_openpipesmutex); - throw Exception("close", myerrno); - } - fds[1] = -1; - m_inpipe = m_outpipe = fds[0]; - } - // put on list of open pipes (so we can kill child processes - // if things go wrong) - s_openpipes.push_front(this); - pthread_mutex_unlock(&s_openpipesmutex); - // ok, put our pages on freelist - m_freelist = m_pages[0u]; - // handshake with other end (to make sure it's alive)... - c = 'P'; // ...arent - if (1 != xferraw(m_outpipe, &c, 1, ::write)) - throw Exception("handshake: xferraw write", EPIPE); - if (1 != xferraw(m_inpipe, &c, 1, ::read)) - throw Exception("handshake: xferraw read", EPIPE); - if ('C' != c) throw Exception("handshake", EPIPE); - break; - } - // mark file descriptors for close on exec (we do not want to leak the - // connection to anything we happen to exec) - int fdflags = 0; - if (-1 == ::fcntl(m_outpipe, F_GETFD, &fdflags)) - throw Exception("fcntl", errno); - fdflags |= FD_CLOEXEC; - if (-1 == ::fcntl(m_outpipe, F_SETFD, fdflags)) - throw Exception("fcntl", errno); - if (m_inpipe != m_outpipe) { - if (-1 == ::fcntl(m_inpipe, F_GETFD, &fdflags)) - throw Exception("fcntl", errno); - fdflags |= FD_CLOEXEC; - if (-1 == ::fcntl(m_inpipe, F_SETFD, fdflags)) - throw Exception("fcntl", errno); - } - // ok, finally, clear the failbit - m_flags &= ~failbit; - // all done - } catch (BidirMMapPipe::Exception&) { - if (0 != m_childPid) kill(m_childPid, SIGTERM); - for (int i = 0; i < 4; ++i) - if (-1 != fds[i] && 0 != fds[i]) ::close(fds[i]); - { - // free resources associated with mmapped pages - BidirMMapPipe_impl::Pages p; p.swap(m_pages); - } - if (!--s_pagepoolrefcnt) { - delete s_pagepool; - s_pagepool = nullptr; - } - throw; - } -} - -int BidirMMapPipe::close() -{ - assert(!(m_flags & failbit)); - return doClose(false); -} - -int BidirMMapPipe::doClose(bool force, bool holdlock) -{ - if (m_flags & failbit) return 0; - // flush data to be written - if (!force && -1 != m_outpipe && -1 != m_inpipe) flush(); - // shut down the write direction (no more writes from our side) - if (m_inpipe == m_outpipe) { - if (-1 != m_outpipe && !force && -1 == ::shutdown(m_outpipe, SHUT_WR)) - throw Exception("shutdown", errno); - m_outpipe = -1; - } else { - if (-1 != m_outpipe && -1 == ::close(m_outpipe)) - if (!force) throw Exception("close", errno); - m_outpipe = -1; - } - // shut down the write direction (no more writes from our side) - // drain anything the other end might still want to send - if (!force && -1 != m_inpipe) { - // **************** THIS IS EXTREMELY UGLY: **************** - // POLLHUP is not set reliably on pipe/socket shutdown on all - // platforms, unfortunately, so we poll for readability here until - // the other end closes, too - // - // the read loop below ensures that the other end sees the POLLIN that - // is set on shutdown instead, and goes ahead to close its end - // - // if we don't do this, and close straight away, the other end - // will catch a SIGPIPE or similar, and we don't want that - int err; - struct pollfd fds; - fds.fd = m_inpipe; - fds.events = POLLIN; - fds.revents = 0; - do { - while ((err = ::poll(&fds, 1, 1 << 20)) >= 0) { - if (fds.revents & (POLLERR | POLLHUP | POLLNVAL)) break; - if (fds.revents & POLLIN) { - char c; - if (1 > ::read(m_inpipe, &c, 1)) break; - } - } - } while (0 > err && EINTR == errno); - // ignore all other poll errors - } - // close read end - if (-1 != m_inpipe && -1 == ::close(m_inpipe)) - if (!force) throw Exception("close", errno); - m_inpipe = -1; - // unmap memory - try { - { BidirMMapPipe_impl::Pages p; p.swap(m_pages); } - if (!--s_pagepoolrefcnt) { - delete s_pagepool; - s_pagepool = nullptr; - } - } catch (std::exception&) { - if (!force) throw; - } - m_busylist = m_freelist = m_dirtylist = nullptr; - // wait for child process - int retVal = 0; - if (isParent()) { - int tmp; - do { - tmp = waitpid(m_childPid, &retVal, 0); - } while (-1 == tmp && EINTR == errno); - if (-1 == tmp) - if (!force) throw Exception("waitpid", errno); - m_childPid = 0; - } - // remove from list of open pipes - if (!holdlock) pthread_mutex_lock(&s_openpipesmutex); - std::list::iterator it = std::find( - s_openpipes.begin(), s_openpipes.end(), this); - if (s_openpipes.end() != it) s_openpipes.erase(it); - if (!holdlock) pthread_mutex_unlock(&s_openpipesmutex); - m_flags |= failbit; - return retVal; -} - -BidirMMapPipe::~BidirMMapPipe() -{ doClose(false); } - -BidirMMapPipe::size_type BidirMMapPipe::xferraw( - int fd, void* addr, size_type len, - ssize_t (*xferfn)(int, void*, std::size_t)) -{ - size_type xferred = 0; - unsigned char* buf = reinterpret_cast(addr); - while (len) { - ssize_t tmp = xferfn(fd, buf, len); - if (tmp > 0) { - xferred += tmp; - len -= tmp; - buf += tmp; - continue; - } else if (0 == tmp) { - // check for end-of-file on pipe - break; - } else if (-1 == tmp) { - // ok some error occurred, so figure out if we want to retry of throw - switch (errno) { - default: - // if anything was transferred, return number of bytes - // transferred so far, we can start throwing on the next - // transfer... - if (xferred) return xferred; - // else throw - throw Exception("xferraw", errno); - case EAGAIN: // fallthrough intended -#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN - case EWOULDBLOCK: // fallthrough intended -#endif - std::cerr << " ERROR: In " << __func__ << " (" << - __FILE__ << ", line " << __LINE__ << - "): expect transfer to block!" << std::endl; - case EINTR: - break; - } - continue; - } else { - throw Exception("xferraw: unexpected return value from read/write", - errno); - } - } - return xferred; -} - -void BidirMMapPipe::sendpages(Page* plist) -{ - if (plist) { - unsigned char pg = m_pages[plist]; - if (1 == xferraw(m_outpipe, &pg, 1, ::write)) { - if (BidirMMapPipe_impl::PageChunk::Copy == - BidirMMapPipe_impl::PageChunk::mmapVariety()) { - // ok, have to copy pages through pipe - for (Page* p = plist; p; p = p->next()) { - if (sizeof(Page) + p->size() != - xferraw(m_outpipe, p, sizeof(Page) + p->size(), - ::write)) { - throw Exception("sendpages: short write", EPIPE); - } - } - } - } else { - throw Exception("sendpages: short write", EPIPE); - } - } else { assert(plist); } -} - -unsigned BidirMMapPipe::recvpages() -{ - unsigned char pg; - unsigned retVal = 0; - Page *plisthead = nullptr; - Page *plisttail = nullptr; - if (1 == xferraw(m_inpipe, &pg, 1, ::read)) { - plisthead = plisttail = m_pages[pg]; - // ok, have number of pages - if (BidirMMapPipe_impl::PageChunk::Copy == - BidirMMapPipe_impl::PageChunk::mmapVariety()) { - // ok, need to copy pages through pipe - for (; plisttail; ++retVal) { - Page* p = plisttail; - if (sizeof(Page) == xferraw(m_inpipe, p, sizeof(Page), - ::read)) { - plisttail = p->next(); - if (p->empty()) continue; - // break in case of read error - if (p->size() != xferraw(m_inpipe, p->begin(), p->size(), - ::read)) break; - } - } - } else { - retVal = lenPageList(plisthead); - } - } - // put list of pages we just received into correct lists (busy/free) - if (plisthead) feedPageLists(plisthead); - // ok, retVal contains the number of pages read, so put them on the - // correct lists - return retVal; -} - -unsigned BidirMMapPipe::recvpages_nonblock() -{ - struct pollfd fds; - fds.fd = m_inpipe; - fds.events = POLLIN; - fds.revents = 0; - unsigned retVal = 0; - do { - int rc = ::poll(&fds, 1, 0); - if (0 > rc) { - if (EINTR == errno) continue; - break; - } - if (1 == retVal && fds.revents & POLLIN && - !(fds.revents & (POLLNVAL | POLLERR))) { - // ok, we can read without blocking, so the other end has - // something for us - return recvpages(); - } else { - break; - } - } while (true); - return retVal; -} - -unsigned BidirMMapPipe::lenPageList(const Page* p) -{ - unsigned n = 0; - for ( ; p; p = p->next()) ++n; - return n; -} - -void BidirMMapPipe::feedPageLists(Page* plist) -{ - assert(plist); - // get end of busy list - Page *blend = m_busylist; - while (blend && blend->next()) blend = blend->next(); - // ok, might have to send free pages to other end, and (if we do have to - // send something to the other end) while we're at it, send any dirty - // pages which are completely full, too - Page *sendlisthead = nullptr; - Page *sendlisttail = nullptr; - // loop over plist - while (plist) { - Page* p = plist; - plist = p->next(); - p->setNext(nullptr); - if (!p->empty()) { - // busy page... - p->pos() = 0; - // put at end of busy list - if (blend) blend->setNext(p); - else m_busylist = p; - blend = p; - } else { - // free page... - // Very simple algorithm: once we're done with a page, we send it back - // where it came from. If it's from our end, we put it on the free list, if - // it's from the other end, we send it back. - if ((isParent() && m_pages[p] >= PagesPerEnd) || - (isChild() && m_pages[p] < PagesPerEnd)) { - // page "belongs" to other end - if (!sendlisthead) sendlisthead = p; - if (sendlisttail) sendlisttail->setNext(p); - sendlisttail = p; - } else { - // add page to freelist - p->setNext(m_freelist); - m_freelist = p; - } - } - } - // check if we have to send stuff to the other end - if (sendlisthead) { - // go through our list of dirty pages, and see what we can - // send along - Page* dp; - while ((dp = m_dirtylist) && dp->full()) { - Page* p = dp; - // move head of dirty list - m_dirtylist = p->next(); - // queue for sending - p->setNext(nullptr); - sendlisttail->setNext(p); - sendlisttail = p; - } - // poll if the other end is still alive - this needs that we first - // close the write pipe of the other end when the remote end of the - // connection is shutting down in doClose; we'll see that because we - // get a POLLHUP on our inpipe - const int nfds = (m_outpipe == m_inpipe) ? 1 : 2; - struct pollfd fds[2]; - fds[0].fd = m_outpipe; - fds[0].events = fds[0].revents = 0; - if (m_outpipe != m_inpipe) { - fds[1].fd = m_inpipe; - fds[1].events = fds[1].revents = 0; - } else { - fds[0].events |= POLLIN; - } - int retVal = 0; - do { - retVal = ::poll(fds, nfds, 0); - if (0 > retVal && EINTR == errno) - continue; - break; - } while (true); - if (0 <= retVal) { - bool ok = !(fds[0].revents & (POLLERR | POLLNVAL | POLLHUP)); - if (m_outpipe != m_inpipe) { - ok = ok && !(fds[1].revents & (POLLERR | POLLNVAL | POLLHUP)); - } else { - if (ok && fds[0].revents & POLLIN) { - unsigned ret = recvpages(); - if (!ret) ok = false; - } - } - - if (ok) sendpages(sendlisthead); - // (if the pipe is dead already, we don't care that we leak the - // contents of the pages on the send list here, so that is why - // there's no else clause here) - } else { - throw Exception("feedPageLists: poll", errno); - } - } -} - -void BidirMMapPipe::markPageDirty(Page* p) -{ - assert(p); - assert(p == m_freelist); - // remove from freelist - m_freelist = p->next(); - p->setNext(nullptr); - // append to dirty list - Page* dl = m_dirtylist; - while (dl && dl->next()) dl = dl->next(); - if (dl) dl->setNext(p); - else m_dirtylist = p; -} - -BidirMMapPipe::Page* BidirMMapPipe::busypage() -{ - // queue any pages available for reading we can without blocking - recvpages_nonblock(); - Page* p; - // if there are no busy pages, try to get them from the other end, - // block if we have to... - while (!(p = m_busylist)) if (!recvpages()) return nullptr; - return p; -} - -BidirMMapPipe::Page* BidirMMapPipe::dirtypage() -{ - // queue any pages available for reading we can without blocking - recvpages_nonblock(); - Page* p = m_dirtylist; - // go to end of dirty list - if (p) while (p->next()) p = p->next(); - if (!p || p->full()) { - // need to append free page, so get one - while (!(p = m_freelist)) if (!recvpages()) return nullptr; - markPageDirty(p); - } - return p; -} - -void BidirMMapPipe::flush() -{ return doFlush(true); } - -void BidirMMapPipe::doFlush(bool forcePartialPages) -{ - assert(!(m_flags & failbit)); - // build a list of pages to flush - Page *flushlisthead = nullptr; - Page *flushlisttail = nullptr; - while (m_dirtylist) { - Page* p = m_dirtylist; - if (!forcePartialPages && !p->full()) break; - // remove dirty page from dirty list - m_dirtylist = p->next(); - p->setNext(nullptr); - // and send it to other end - if (!flushlisthead) flushlisthead = p; - if (flushlisttail) flushlisttail->setNext(p); - flushlisttail = p; - } - if (flushlisthead) sendpages(flushlisthead); -} - -void BidirMMapPipe::purge() -{ - assert(!(m_flags & failbit)); - // join busy and dirty lists - { - Page *l = m_busylist; - while (l && l->next()) l = l->next(); - if (l) l->setNext(m_dirtylist); - else m_busylist = m_dirtylist; - } - // empty busy and dirty pages - for (Page* p = m_busylist; p; p = p->next()) p->size() = 0; - // put them on the free list - if (m_busylist) feedPageLists(m_busylist); - m_busylist = m_dirtylist = nullptr; -} - -BidirMMapPipe::size_type BidirMMapPipe::bytesReadableNonBlocking() -{ - // queue all pages waiting for consumption in the pipe before we give an - // answer - recvpages_nonblock(); - size_type retVal = 0; - for (Page* p = m_busylist; p; p = p->next()) - retVal += p->size() - p->pos(); - return retVal; -} - -BidirMMapPipe::size_type BidirMMapPipe::bytesWritableNonBlocking() -{ - // queue all pages waiting for consumption in the pipe before we give an - // answer - recvpages_nonblock(); - // check if we could write to the pipe without blocking (we need to know - // because we might need to check if flushing of dirty pages would block) - bool couldwrite = false; - { - struct pollfd fds; - fds.fd = m_outpipe; - fds.events = POLLOUT; - fds.revents = 0; - int retVal = 0; - do { - retVal = ::poll(&fds, 1, 0); - if (0 > retVal) { - if (EINTR == errno) continue; - throw Exception("bytesWritableNonBlocking: poll", errno); - } - if (1 == retVal && fds.revents & POLLOUT && - !(fds.revents & (POLLNVAL | POLLERR | POLLHUP))) - couldwrite = true; - break; - } while (true); - } - // ok, start counting bytes - size_type retVal = 0; - unsigned npages = 0; - // go through the dirty list - for (Page* p = m_dirtylist; p; p = p->next()) { - ++npages; - // if page only partially filled - if (!p->full()) - retVal += p->free(); - if (npages >= FlushThresh && !couldwrite) break; - } - // go through the free list - for (Page* p = m_freelist; p && (!m_dirtylist || - npages < FlushThresh || couldwrite); p = p->next()) { - ++npages; - retVal += Page::capacity(); - } - return retVal; -} - -BidirMMapPipe::size_type BidirMMapPipe::read(void* addr, size_type sz) -{ - assert(!(m_flags & failbit)); - size_type nread = 0; - unsigned char *ap = reinterpret_cast(addr); - try { - while (sz) { - // find next page to read from - Page* p = busypage(); - if (!p) { - m_flags |= eofbit; - return nread; - } - unsigned char* pp = p->begin() + p->pos(); - size_type csz = std::min(size_type(p->remaining()), sz); - std::copy(pp, pp + csz, ap); - nread += csz; - ap += csz; - sz -= csz; - p->pos() += csz; - assert(p->size() >= p->pos()); - if (p->size() == p->pos()) { - // if no unread data remains, page is free - m_busylist = p->next(); - p->setNext(nullptr); - p->size() = 0; - feedPageLists(p); - } - } - } catch (Exception&) { - m_flags |= rderrbit; - if (m_flags & exceptionsbit) throw; - } - return nread; -} - -BidirMMapPipe::size_type BidirMMapPipe::write(const void* addr, size_type sz) -{ - assert(!(m_flags & failbit)); - size_type written = 0; - const unsigned char *ap = reinterpret_cast(addr); - try { - while (sz) { - // find next page to write to - Page* p = dirtypage(); - if (!p) { - m_flags |= eofbit; - return written; - } - unsigned char* pp = p->begin() + p->size(); - size_type csz = std::min(size_type(p->free()), sz); - std::copy(ap, ap + csz, pp); - written += csz; - ap += csz; - p->size() += csz; - sz -= csz; - assert(p->capacity() >= p->size()); - if (p->full()) { - // if page is full, see if we're above the flush threshold of - // 3/4 of our pages - if (lenPageList(m_dirtylist) >= FlushThresh) - doFlush(false); - } - } - } catch (Exception&) { - m_flags |= wrerrbit; - if (m_flags & exceptionsbit) throw; - } - return written; -} - -int BidirMMapPipe::poll(BidirMMapPipe::PollVector& pipes, int timeout) -{ - // go through pipes, and change flags where we already know without really - // polling - stuff where we don't need poll to wait for its timeout in the - // OS... - bool canskiptimeout = false; - std::vector masks(pipes.size(), ~(Readable | Writable)); - std::vector::iterator mit = masks.begin(); - for (PollVector::iterator it = pipes.begin(); pipes.end() != it; - ++it, ++mit) { - PollEntry& pe = *it; - pe.revents = None; - // null pipe is invalid - if (!pe.pipe) { - pe.revents |= Invalid; - canskiptimeout = true; - continue; - } - // closed pipe is invalid - if (pe.pipe->closed()) pe.revents |= Invalid; - // check for error - if (pe.pipe->bad()) pe.revents |= Error; - // check for end of file - if (pe.pipe->eof()) pe.revents |= EndOfFile; - // check if readable - if (pe.events & Readable) { - *mit |= Readable; - if (pe.pipe->m_busylist) pe.revents |= Readable; - } - // check if writable - if (pe.events & Writable) { - *mit |= Writable; - if (pe.pipe->m_freelist) { - pe.revents |= Writable; - } else { - Page *dl = pe.pipe->m_dirtylist; - while (dl && dl->next()) dl = dl->next(); - if (dl && dl->pos() < Page::capacity()) - pe.revents |= Writable; - } - } - if (pe.revents) canskiptimeout = true; - } - // set up the data structures required for the poll syscall - std::vector fds; - fds.reserve(2 * pipes.size()); - std::map fds2pipes; - for (PollVector::const_iterator it = pipes.begin(); - pipes.end() != it; ++it) { - const PollEntry& pe = *it; - struct pollfd tmp; - fds2pipes.insert(std::make_pair((tmp.fd = pe.pipe->m_inpipe), - const_cast(&pe))); - tmp.events = tmp.revents = 0; - // we always poll for readability; this allows us to queue pages - // early - tmp.events |= POLLIN; - if (pe.pipe->m_outpipe != tmp.fd) { - // ok, it's a pair of pipes - fds.push_back(tmp); - fds2pipes.insert(std::make_pair( - unsigned(tmp.fd = pe.pipe->m_outpipe), - const_cast(&pe))); - tmp.events = 0; - - } - if (pe.events & Writable) tmp.events |= POLLOUT; - fds.push_back(tmp); - } - // poll - int retVal = 0; - do { - retVal = ::poll(&fds[0], fds.size(), canskiptimeout ? 0 : timeout); - if (0 > retVal) { - if (EINTR == errno) continue; - throw Exception("poll", errno); - } - break; - } while (true); - // fds may have changed state, so update... - for (std::vector::iterator it = fds.begin(); - fds.end() != it; ++it) { - pollfd& fe = *it; - //if (!fe.revents) continue; - --retVal; - PollEntry& pe = *fds2pipes[fe.fd]; -oncemore: - if (fe.revents & POLLNVAL && fe.fd == pe.pipe->m_inpipe) - pe.revents |= ReadInvalid; - if (fe.revents & POLLNVAL && fe.fd == pe.pipe->m_outpipe) - pe.revents |= WriteInvalid; - if (fe.revents & POLLERR && fe.fd == pe.pipe->m_inpipe) - pe.revents |= ReadError; - if (fe.revents & POLLERR && fe.fd == pe.pipe->m_outpipe) - pe.revents |= WriteError; - if (fe.revents & POLLHUP && fe.fd == pe.pipe->m_inpipe) - pe.revents |= ReadEndOfFile; - if (fe.revents & POLLHUP && fe.fd == pe.pipe->m_outpipe) - pe.revents |= WriteEndOfFile; - if ((fe.revents & POLLIN) && fe.fd == pe.pipe->m_inpipe && - !(fe.revents & (POLLNVAL | POLLERR))) { - // ok, there is at least one page for us to receive from the - // other end - if (0 == pe.pipe->recvpages()) continue; - // more pages there? - do { - int tmp = ::poll(&fe, 1, 0); - if (tmp > 0) goto oncemore; // yippie! I don't even feel bad! - if (0 > tmp) { - if (EINTR == errno) continue; - throw Exception("poll", errno); - } - break; - } while (true); - } - if (pe.pipe->m_busylist) pe.revents |= Readable; - if (fe.revents & POLLOUT && fe.fd == pe.pipe->m_outpipe) { - if (pe.pipe->m_freelist) { - pe.revents |= Writable; - } else { - Page *dl = pe.pipe->m_dirtylist; - while (dl && dl->next()) dl = dl->next(); - if (dl && dl->pos() < Page::capacity()) - pe.revents |= Writable; - } - } - } - // apply correct masks, and count pipes with pending events - int npipes = 0; - mit = masks.begin(); - for (PollVector::iterator it = pipes.begin(); - pipes.end() != it; ++it, ++mit) - if ((it->revents &= *mit)) ++npipes; - return npipes; -} - -BidirMMapPipe& BidirMMapPipe::operator<<(const char* str) -{ - size_t sz = std::strlen(str); - *this << sz; - if (sz) write(str, sz); - return *this; -} - -BidirMMapPipe& BidirMMapPipe::operator>>(char* (&str)) -{ - size_t sz = 0; - *this >> sz; - if (good() && !eof()) { - str = reinterpret_cast(std::realloc(str, sz + 1)); - if (!str) throw Exception("realloc", errno); - if (sz) read(str, sz); - str[sz] = 0; - } - return *this; -} - -BidirMMapPipe& BidirMMapPipe::operator<<(const std::string& str) -{ - size_t sz = str.size(); - *this << sz; - write(str.data(), sz); - return *this; -} - -BidirMMapPipe& BidirMMapPipe::operator>>(std::string& str) -{ - str.clear(); - size_t sz = 0; - *this >> sz; - if (good() && !eof()) { - str.reserve(sz); - for (unsigned char c; sz--; str.push_back(c)) *this >> c; - } - return *this; -} - -END_NAMESPACE_ROOFIT - -#ifdef TEST_BIDIRMMAPPIPE -using namespace RooFit; - -int simplechild(BidirMMapPipe& pipe) -{ - // child does an echo loop - while (pipe.good() && !pipe.eof()) { - // read a string - std::string str; - pipe >> str; - if (!pipe) return -1; - if (pipe.eof()) break; - if (!str.empty()) { - std::cout << "[CHILD] : read: " << str << std::endl; - str = "... early in the morning?"; - } - pipe << str << BidirMMapPipe::flush; - // did our parent tell us to shut down? - if (str.empty()) break; - if (!pipe) return -1; - if (pipe.eof()) break; - std::cout << "[CHILD] : wrote: " << str << std::endl; - } - pipe.close(); - return 0; -} - -#include -int randomchild(BidirMMapPipe& pipe) -{ - // child sends out something at random intervals - ::srand48(::getpid()); - { - // wait for parent's go ahead signal - std::string s; - pipe >> s; - } - // no shutdown sequence needed on this side - we're producing the data, - // and the parent can just read until we're done (when it'll get EOF) - for (int i = 0; i < 5; ++i) { - // sleep a random time between 0 and .9 seconds - ::usleep(int(1e6 * ::drand48())); - std::ostringstream buf; - buf << "child pid " << ::getpid() << " sends message " << i; - std::string str = buf.str(); - std::cout << "[CHILD] : " << str << std::endl; - pipe << str << BidirMMapPipe::flush; - if (!pipe) return -1; - if (pipe.eof()) break; - } - // tell parent we're shutting down - pipe << "" << BidirMMapPipe::flush; - // wait for parent to acknowledge - std::string s; - pipe >> s; - pipe.close(); - return 0; -} - -int benchchildrtt(BidirMMapPipe& pipe) -{ - // child does the equivalent of listening for pings and sending the - // packet back - char* str = 0; - while (pipe && !pipe.eof()) { - pipe >> str; - if (!pipe) { - std::free(str); - pipe.close(); - return -1; - } - if (pipe.eof()) break; - pipe << str << BidirMMapPipe::flush; - // if we have just completed the shutdown handshake, we break here - if (!std::strlen(str)) break; - } - std::free(str); - pipe.close(); - return 0; -} - -int benchchildsink(BidirMMapPipe& pipe) -{ - // child behaves like a sink - char* str = 0; - while (pipe && !pipe.eof()) { - pipe >> str; - if (!std::strlen(str)) break; - } - pipe << "" << BidirMMapPipe::flush; - std::free(str); - pipe.close(); - return 0; -} - -int benchchildsource(BidirMMapPipe& pipe) -{ - // child behaves like a source - char* str = 0; - for (unsigned i = 0; i <= 24; ++i) { - str = reinterpret_cast(std::realloc(str, (1 << i) + 1)); - std::memset(str, '4', 1 << i); - str[1 << i] = 0; - for (unsigned j = 0; j < 1 << 7; ++j) { - pipe << str; - if (!pipe || pipe.eof()) { - std::free(str); - pipe.close(); - return -1; - } - } - // tell parent we're done with this block size - pipe << "" << BidirMMapPipe::flush; - } - // tell parent to shut down - pipe << "" << BidirMMapPipe::flush; - std::free(str); - pipe.close(); - return 0; -} - -BidirMMapPipe* spawnChild(int (*childexec)(BidirMMapPipe&)) -{ - // create a pipe with the given child at the remote end - BidirMMapPipe *p = new BidirMMapPipe(); - if (p->isChild()) { - int retVal = childexec(*p); - delete p; - std::exit(retVal); - } - return p; -} - -#include -#include -int main() -{ - // simple echo loop test - { - std::cout << "[PARENT]: simple challenge-response test, " - "one child:" << std::endl; - BidirMMapPipe* pipe = spawnChild(simplechild); - for (int i = 0; i < 5; ++i) { - std::string str("What shall we do with a drunken sailor..."); - *pipe << str << BidirMMapPipe::flush; - if (!*pipe) return -1; - std::cout << "[PARENT]: wrote: " << str << std::endl; - *pipe >> str; - if (!*pipe) return -1; - std::cout << "[PARENT]: read: " << str << std::endl; - } - // send shutdown string - *pipe << "" << BidirMMapPipe::flush; - // wait for shutdown handshake - std::string s; - *pipe >> s; - int retVal = pipe->close(); - std::cout << "[PARENT]: exit status of child: " << retVal << - std::endl; - if (retVal) return retVal; - delete pipe; - } - // simple poll test - children send 5 results in random intervals - { - unsigned nch = 20; - std::cout << std::endl << "[PARENT]: polling test, " << nch << - " children:" << std::endl; - typedef BidirMMapPipe::PollEntry PollEntry; - // poll data structure - BidirMMapPipe::PollVector pipes; - pipes.reserve(nch); - // spawn children - for (unsigned i = 0; i < nch; ++i) { - std::cout << "[PARENT]: spawning child " << i << std::endl; - pipes.push_back(PollEntry(spawnChild(randomchild), - BidirMMapPipe::Readable)); - } - // wake children up - std::cout << "[PARENT]: waking up children" << std::endl; - for (unsigned i = 0; i < nch; ++i) - *pipes[i].pipe << "" << BidirMMapPipe::flush; - std::cout << "[PARENT]: waiting for events on children's pipes" << std::endl; - // while at least some children alive - while (!pipes.empty()) { - // poll, wait until status change (infinite timeout) - int npipes = BidirMMapPipe::poll(pipes, -1); - // scan for pipes with changed status - for (std::vector::iterator it = pipes.begin(); - npipes && pipes.end() != it; ) { - if (!it->revents) { - // unchanged, next one - ++it; - continue; - } - --npipes; // maybe we can stop early... - // read from pipes which are readable - if (it->revents & BidirMMapPipe::Readable) { - std::string s; - *(it->pipe) >> s; - if (!s.empty()) { - std::cout << "[PARENT]: Read from pipe " << it->pipe << - ": " << s << std::endl; - ++it; - continue; - } else { - // child is shutting down... - *(it->pipe) << "" << BidirMMapPipe::flush; - goto childcloses; - } - } - // retire pipes with error or end-of-file condition - if (it->revents & (BidirMMapPipe::Error | - BidirMMapPipe::EndOfFile | - BidirMMapPipe::Invalid)) { - std::cerr << "[DEBUG]: Event on pipe " << it->pipe << - " revents" << - ((it->revents & BidirMMapPipe::Readable) ? " Readable" : "") << - ((it->revents & BidirMMapPipe::Writable) ? " Writable" : "") << - ((it->revents & BidirMMapPipe::ReadError) ? " ReadError" : "") << - ((it->revents & BidirMMapPipe::WriteError) ? " WriteError" : "") << - ((it->revents & BidirMMapPipe::ReadEndOfFile) ? " ReadEndOfFile" : "") << - ((it->revents & BidirMMapPipe::WriteEndOfFile) ? " WriteEndOfFile" : "") << - ((it->revents & BidirMMapPipe::ReadInvalid) ? " ReadInvalid" : "") << - ((it->revents & BidirMMapPipe::WriteInvalid) ? " WriteInvalid" : "") << - std::endl; -childcloses: - int retVal = it->pipe->close(); - std::cout << "[PARENT]: child exit status: " << - retVal << ", number of children still alive: " << - (pipes.size() - 1) << std::endl; - if (retVal) return retVal; - delete it->pipe; - it = pipes.erase(it); - continue; - } - } - } - } - // little benchmark - round trip time - { - std::cout << std::endl << "[PARENT]: benchmark: round-trip times vs block size" << std::endl; - for (unsigned i = 0; i <= 24; ++i) { - std::vector s(1 + (1 << i)); - std::memset(s, 'A', 1 << i); - s[1 << i] = 0; - const unsigned n = 1 << 7; - double avg = 0., min = 1e42, max = -1e42; - BidirMMapPipe *pipe = spawnChild(benchchildrtt); - for (unsigned j = n; j--; ) { - struct timeval t1; - ::gettimeofday(&t1, 0); - *pipe << s << BidirMMapPipe::flush; - if (!*pipe || pipe->eof()) break; - *pipe >> s; - if (!*pipe || pipe->eof()) break; - struct timeval t2; - ::gettimeofday(&t2, 0); - t2.tv_sec -= t1.tv_sec; - t2.tv_usec -= t1.tv_usec; - double dt = 1e-6 * double(t2.tv_usec) + double(t2.tv_sec); - if (dt < min) min = dt; - if (dt > max) max = dt; - avg += dt; - } - // send a shutdown string - *pipe << "" << BidirMMapPipe::flush; - // get child's shutdown ok - *pipe >> s; - avg /= double(n); - avg *= 1e6; min *= 1e6; max *= 1e6; - int retVal = pipe->close(); - if (retVal) { - std::cout << "[PARENT]: child exited with code " << retVal << std::endl; - return retVal; - } - delete pipe; - // there is a factor 2 in the formula for the transfer rate below, - // because we transfer data of twice the size of the block - once - // to the child, and once for the return trip - std::cout << "block size " << std::setw(9) << (1 << i) << - " avg " << std::setw(7) << avg << " us min " << - std::setw(7) << min << " us max " << std::setw(7) << max << - "us speed " << std::setw(9) << - 2. * (double(1 << i) / double(1 << 20) / (1e-6 * avg)) << - " MB/s" << std::endl; - } - std::cout << "[PARENT]: all children had exit code 0" << std::endl; - } - // little benchmark - child as sink - { - std::cout << std::endl << "[PARENT]: benchmark: raw transfer rate with child as sink" << std::endl; - for (unsigned i = 0; i <= 24; ++i) { - std::vector s(1 + (1 << i)); - std::memset(s, 'A', 1 << i); - s[1 << i] = 0; - const unsigned n = 1 << 7; - double avg = 0., min = 1e42, max = -1e42; - BidirMMapPipe *pipe = spawnChild(benchchildsink); - for (unsigned j = n; j--; ) { - struct timeval t1; - ::gettimeofday(&t1, 0); - // streaming mode - we do not flush here - *pipe << s; - if (!*pipe || pipe->eof()) break; - struct timeval t2; - ::gettimeofday(&t2, 0); - t2.tv_sec -= t1.tv_sec; - t2.tv_usec -= t1.tv_usec; - double dt = 1e-6 * double(t2.tv_usec) + double(t2.tv_sec); - if (dt < min) min = dt; - if (dt > max) max = dt; - avg += dt; - } - // send a shutdown string - *pipe << "" << BidirMMapPipe::flush; - // get child's shutdown ok - *pipe >> s; - avg /= double(n); - avg *= 1e6; min *= 1e6; max *= 1e6; - int retVal = pipe->close(); - if (retVal) { - std::cout << "[PARENT]: child exited with code " << retVal << std::endl; - return retVal; - } - delete pipe; - std::cout << "block size " << std::setw(9) << (1 << i) << - " avg " << std::setw(7) << avg << " us min " << - std::setw(7) << min << " us max " << std::setw(7) << max << - "us speed " << std::setw(9) << - (double(1 << i) / double(1 << 20) / (1e-6 * avg)) << - " MB/s" << std::endl; - } - std::cout << "[PARENT]: all children had exit code 0" << std::endl; - } - // little benchmark - child as source - { - std::cout << std::endl << "[PARENT]: benchmark: raw transfer rate with child as source" << std::endl; - char *s = 0; - double avg = 0., min = 1e42, max = -1e42; - unsigned n = 0, bsz = 0; - BidirMMapPipe *pipe = spawnChild(benchchildsource); - while (*pipe && !pipe->eof()) { - struct timeval t1; - ::gettimeofday(&t1, 0); - // streaming mode - we do not flush here - *pipe >> s; - if (!*pipe || pipe->eof()) break; - struct timeval t2; - ::gettimeofday(&t2, 0); - t2.tv_sec -= t1.tv_sec; - t2.tv_usec -= t1.tv_usec; - double dt = 1e-6 * double(t2.tv_usec) + double(t2.tv_sec); - if (std::strlen(s)) { - ++n; - if (dt < min) min = dt; - if (dt > max) max = dt; - avg += dt; - bsz = std::strlen(s); - } else { - if (!n) break; - // next block size - avg /= double(n); - avg *= 1e6; min *= 1e6; max *= 1e6; - - std::cout << "block size " << std::setw(9) << bsz << - " avg " << std::setw(7) << avg << " us min " << - std::setw(7) << min << " us max " << std::setw(7) << - max << "us speed " << std::setw(9) << - (double(bsz) / double(1 << 20) / (1e-6 * avg)) << - " MB/s" << std::endl; - n = 0; - avg = 0.; - min = 1e42; - max = -1e42; - } - } - int retVal = pipe->close(); - std::cout << "[PARENT]: child exited with code " << retVal << std::endl; - if (retVal) return retVal; - delete pipe; - std::free(s); - } - return 0; -} -#endif // TEST_BIDIRMMAPPIPE -#endif // _WIN32 - -// vim: ft=cpp:sw=4:tw=78:et - -/// \endcond diff --git a/roofit/roofitcore/src/BidirMMapPipe.h b/roofit/roofitcore/src/BidirMMapPipe.h deleted file mode 100644 index b04abc31294f1..0000000000000 --- a/roofit/roofitcore/src/BidirMMapPipe.h +++ /dev/null @@ -1,998 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/** @file BidirMMapPipe.h - * - * header file for BidirMMapPipe, a class which forks off a child process and - * serves as communications channel between parent and child - * - * @author Manuel Schiller - * @date 2013-07-07 - */ - -#ifndef BIDIRMMAPPIPE_H -#define BIDIRMMAPPIPE_H - -#include -#include -#include -#include -#include -#include - -#define BEGIN_NAMESPACE_ROOFIT namespace RooFit { -#define END_NAMESPACE_ROOFIT } - -BEGIN_NAMESPACE_ROOFIT - -/// namespace for implementation details of BidirMMapPipe -namespace BidirMMapPipe_impl { - // forward declarations - class BidirMMapPipeException; - class Page; - class PagePool; - class Pages; - - /** @brief class representing a chunk of pages - * - * @author Manuel Schiller - * @date 2013-07-24 - * - * allocating pages from the OS happens in chunks in order to not exhaust - * the maximum allowed number of memory mappings per process; this class - * takes care of such a chunk - * - * a page chunk allows callers to obtain or release pages in groups of - * continuous pages of fixed size - */ - class PageChunk { - public: - /// type of mmap support found - typedef enum { - Unknown, ///< don't know yet what'll work - Copy, ///< mmap doesn't work, have to copy back and forth - FileBacked, ///< mmapping a temp file works - DevZero, ///< mmapping /dev/zero works - Anonymous ///< anonymous mmap works - } MMapVariety; - - private: - static unsigned s_physpgsz; ///< system physical page size - static unsigned s_pagesize; ///< logical page size (run-time determined) - /// mmap variety that works on this system - static MMapVariety s_mmapworks; - - /// convenience typedef - typedef BidirMMapPipeException Exception; - - void* m_begin; ///< pointer to start of mmapped area - void* m_end; ///< pointer one behind end of mmapped area - // FIXME: cannot keep freelist inline - other end may need that - // data, and we'd end up overwriting the page header - std::list m_freelist; ///< free pages list - PagePool* m_parent; ///< parent page pool - unsigned m_nPgPerGrp; ///< number of pages per group - unsigned m_nUsedGrp; ///< number of used page groups - - /// determine page size at run time - static unsigned getPageSize(); - - /// mmap pages, len is length of mmapped area in bytes - static void* dommap(unsigned len); - /// munmap pages p, len is length of mmapped area in bytes - static void domunmap(void* p, unsigned len); - /// forbid copying - PageChunk(const PageChunk&) {} - /// forbid assignment - PageChunk& operator=(const PageChunk&) { return *this; } - public: - /// return the logical page size - static unsigned pagesize() { return s_pagesize; } - /// return the physical page size of the system - static unsigned physPgSz() { return s_physpgsz; } - /// return mmap variety support found - static MMapVariety mmapVariety() { return s_mmapworks; } - - /// constructor - PageChunk(PagePool* parent, unsigned length, unsigned nPgPerGroup); - - /// destructor - ~PageChunk(); - - /// return if p is contained in this PageChunk - bool contains(const Pages& p) const; - - /// pop a group of pages off the free list - Pages pop(); - - /// push a group of pages onto the free list - void push(const Pages& p); - - /// return length of chunk - unsigned len() const - { - return reinterpret_cast(m_end) - - reinterpret_cast(m_begin); - } - /// return number of pages per page group - unsigned nPagesPerGroup() const { return m_nPgPerGrp; } - - /// return true if no used page groups in this chunk - bool empty() const { return !m_nUsedGrp; } - - /// return true if no free page groups in this chunk - bool full() const { return m_freelist.empty(); } - - /// free all pages except for those pointed to by p - void zap(Pages& p); - }; - - /** @brief handle class for a number of Pages - * - * @author Manuel Schiller - * @date 2013-07-24 - * - * the associated pages are continuous in memory - */ - class Pages { - private: - /// implementation - typedef struct { - PageChunk *m_parent; ///< pointer to parent pool - Page* m_pages; ///< pointer to first page - unsigned m_refcnt; ///< reference counter - unsigned char m_npages; ///< length in pages - } impl; - public: - /// default constructor - Pages() = default; - - /// destructor - ~Pages(); - - /** @brief copy constructor - * - * copy Pages handle to new object - old object loses ownership, - * and becomes a dangling handle - */ - Pages(const Pages& other); - - /** @brief assignment operator - * - * assign Pages handle to new object - old object loses ownership, - * and becomes a dangling handle - */ - Pages& operator=(const Pages& other); - - /// return page size - static unsigned pagesize(); - - /// return number of pages accessible - unsigned npages() const { return m_pimpl->m_npages; } - - /// return page number pageno - Page* page(unsigned pgno) const; - - /// return page number pageno - Page* operator[](unsigned pgno) const { return page(pgno); } - - /// perform page to page number mapping - unsigned pageno(Page* p) const; - - /// perform page to page number mapping - unsigned operator[](Page* p) const { return pageno(p); } - - /// swap with other's contents - void swap(Pages& other) - { - impl* tmp = other.m_pimpl; - other.m_pimpl = m_pimpl; - m_pimpl = tmp; - } - - private: - /// page pool is our friend - it's allowed to construct Pages - friend class BidirMMapPipe_impl::PageChunk; - - /// pointer to implementation - impl* m_pimpl = nullptr; - - /// constructor - Pages(PageChunk* parent, Page* pages, unsigned npg); - }; -} - -/** @brief BidirMMapPipe creates a bidirectional channel between the current - * process and a child it forks. - * - * @author Manuel Schiller - * @date 2013-07-07 - * - * This class creates a bidirectional channel between this process and a child - * it creates with fork(). - * - * The channel is comrised of a small shared pool of buffer memory mmapped into - * both process spaces, and two pipes to synchronise the exchange of data. The - * idea behind using the pipes at all is to have some primitive which we can - * block on without having to worry about atomic operations or polling, leaving - * these tasks to the OS. In case the anonymous mmap cannot be performed on the - * OS the code is running on (for whatever reason), the code falls back to - * mmapping /dev/zero, mmapping a temporary file, or (if those all fail), a - * dynamically allocated buffer which is then transmitted through the pipe(s), - * a slightly slower alternative (because the data is copied more often). - * - * The channel supports five major operations: read(), write(), flush(), - * purge() and close(). Reading and writing may block until the required buffer - * space is available. Writes may queue up data to be sent to the other end - * until either enough pages are full, or the user calls flush which forces - * any unsent buffers to be sent to the other end. flush forces any data that - * is to be sent to be sent. purge discards any buffered data waiting to be - * read and/or sent. Closing the channel on the child returns zero, closing it - * on the parent returns the child's exit status. - * - * The class also provides operator<< and operator>> for C++-style I/O for - * basic data types (bool, char, short, int, long, long long, float, double - * and their unsigned counterparts). Data is transmitted binary (i.e. no - * formatting to strings like std::cout does). There are also overloads to - * support C-style zero terminated strings and std::string. In terms of - * performance, the former is to be preferred. - * - * If the caller needs to multiplex input and output to/from several pipes, the - * class provides the poll() method which allows to block until an event occurs - * on any of the polled pipes. - * - * After the BidirMMapPipe is closed, no further operations may be performed on - * that object, save for the destructor which may still be called. - * - * If the BidirMMapPipe has not properly been closed, the destructor will call - * close. However, the exit code of the child is lost in that case. - * - * Closing the object causes the mmapped memory to be unmapped and the two - * pipes to be closed. We also install an atexit handler in the process of - * creating BidirMMapPipes. This ensures that when the current process - * terminates, a SIGTERM signal is sent to the child processes created for all - * unclosed pipes to avoid leaving zombie processes in the OS's process table. - * - * BidirMMapPipe creation, closing and destruction are thread safe. If the - * BidirMMapPipe is used in more than one thread, the other operations have to - * be protected with a mutex (or something similar), though. - * - * End of file (other end closed its pipe, or died) is indicated with the eof() - * method, serious I/O errors set a flags (bad(), fail()), and also throw - * exceptions. For normal read/write operations, they can be suppressed (i.e. - * error reporting only using flags) with a constructor argument. - * - * Technicalities: - * - there is a pool of mmapped pages, half the pages are allocated to the - * parent process, half to the child - * - when one side has accumulated enough data (or a flush forces dirty pages - * out to the other end), it sends these pages to the other end by writing a - * byte containing the page number into the pipe - * - the other end (which has the pages mmapped, too) reads the page number(s) - * and puts the corresponding pages on its busy list - * - as the other ends reads, it frees busy pages, and eventually tries to put - * them on the its list; if a page belongs to the other end of the - * connection, it is sent back - * - lists of pages are sent across the pipe, not individual pages, in order - * to minimise the number of read/write operations needed - * - when mmap works properly, only one bytes containing the page number of - * the page list head is sent back and forth; the contents of that page - * allow to access the rest of the page list sent, and page headers on the - * list tell the receiving end if the page is free or has to be added to the - * busy list - * - when mmap does not work, we transfer one byte to indicate the head of the - * page list sent, and for each page on the list of sent pages, the page - * header and the page payload is sent (if the page is free, we only - * transmit the page header, and we never transmit more payload than - * the page actually contains) - * - in the child, all open BidirMMapPipes but the current one are closed. this - * is done for two reasons: first, to conserve file descriptors and address - * space. second, if more than one process is meant to use such a - * BidirMMapPipe, synchronisation issues arise which can lead to bugs that - * are hard to find and understand. it's much better to come up with a design - * which does not need pipes to be shared among more than two processes. - * - * Here is a trivial example of a parent and a child talking to each other over - * a BidirMMapPipe: - * @code - * #include - * #include - * #include - * - * #include "BidirMMapPipe.h" - * - * int simplechild(BidirMMapPipe& pipe) - * { - * // child does an echo loop - * while (pipe.good() && !pipe.eof()) { - * // read a string - * std::string str; - * pipe >> str; - * if (!pipe) return -1; - * if (pipe.eof()) break; - * // check if parent wants us to shut down - * if (!str.empty()) { - * std::cout << "[CHILD] : read: " << str << std::endl; - * str = "... early in the morning?"; - * } - * pipe << str << BidirMMapPipe::flush; - * if (str.empty()) break; - * if (!pipe) return -1; - * std::cout << "[CHILD] : wrote: " << str << std::endl; - * } - * // send shutdown request acknowledged - * pipe << "" << BidirMMapPipe::flush; - * - * pipe.close(); - * return 0; - * } - * - * BidirMMapPipe* spawnChild(int (*childexec)(BidirMMapPipe&)) - * { - * BidirMMapPipe *p = new BidirMMapPipe(); - * if (p->isChild()) { - * int retVal = childexec(*p); - * delete p; - * std::exit(retVal); - * } - * return p; - * } - * - * int main() - * { - * std::cout << "[PARENT]: simple challenge-response test, one child:" << - * std::endl; - * BidirMMapPipe* pipe = spawnChild(simplechild); - * for (int i = 0; i < 5; ++i) { - * std::string str("What shall we do with a drunken sailor..."); - * *pipe << str << BidirMMapPipe::flush; - * if (!*pipe) return -1; - * std::cout << "[PARENT]: wrote: " << str << std::endl; - * *pipe >> str; - * if (!*pipe) return -1; - * std::cout << "[PARENT]: read: " << str << std::endl; - * } - * // ask child to shut down - * pipe << "" << BidirMMapPipe::flush; - * // wait for it to see the shutdown request - * std::string s; - * pipe >> s; - * std::cout << "[PARENT]: exit status of child: " << pipe->close() << - * std::endl; - * delete pipe; - * return 0; - * } - * @endcode - * - * When designing your own protocols to use over the pipe, there are a few - * things to bear in mind: - * - Do as http does: When building a request, send all the options and - * properties of that request with the request itself in a single go (one - * flush). Then, the server has everything it needs, and hopefully, it'll - * shut up for a while and to let the client do something useful in the - * meantime... The same goes when the server replies to the request: include - * everything there is to know about the result of the request in the reply. - * - The expensive operation should be the request that is made, all other - * operations should somehow be formulated as options or properties to that - * request. - * - Include a shutdown handshake in whatever protocol you send over the - * pipe. That way, you can shut things down in a controlled way. Otherwise, - * and depending on your OS's scheduling quirks, you may catch a SIGPIPE if - * one end closes its pipe while the other is still trying to read. - */ -class BidirMMapPipe { -#ifndef _WIN32 - public: - /// type used to represent sizes - typedef std::size_t size_type; - /// convenience typedef for BidirMMapPipeException - typedef BidirMMapPipe_impl::BidirMMapPipeException Exception; - /// flag bits for partial C++ iostream compatibility - enum { - eofbit = 1, ///< end of file reached - failbit = 2, ///< logical failure (e.g. pipe closed) - rderrbit = 4, ///< read error - wrerrbit = 8, ///< write error - badbit = rderrbit | wrerrbit, ///< general I/O error - exceptionsbit = 16 ///< error reporting with exceptions - }; - - /** @brief constructor (forks!) - * - * Creates a bidirectional communications channel between this process - * and a child the constructor forks. On return from the constructor, - * isParent() and isChild() can be used to tell the parent end from the - * child end of the pipe. In the child, all other open BidirMMapPipes - * are closed. - * - * @param useExceptions read()/write() error reporting also done using - * exceptions - * @param useSocketpair use a socketpair instead of a pair or pipes - * - * Normally, exceptions are thrown for all serious I/O errors (apart - * from end of file). Setting useExceptions to false will force the - * read() and write() methods to only report serious I/O errors using - * flags. - * - * When useSocketpair is true, use a pair of Unix domain sockets - * created using socketpair instead a pair of pipes. The advantage is - * that only one pair of file descriptors is needed instead of two - * pairs which are needed for the pipe pair. Performance should very - * similar on most platforms, especially if mmap works, since only - * very little data is sent through the pipe(s)/socketpair. - */ - BidirMMapPipe(bool useExceptions = true, bool useSocketpair = false); - - /** @brief destructor - * - * closes this end of pipe - */ - ~BidirMMapPipe(); - - /** @brief return the current setting of the debug flag - * - * @returns an integer with the debug Setting - */ - static int debugflag() { return s_debugflag; } - - /** @brief set the debug flags - * - * @param flag debug flags (if zero, no messages are printed) - */ - static void setDebugflag(int flag) { s_debugflag = flag; } - - /** @brief read from pipe - * - * @param addr where to put read data - * @param sz size of data to read (in bytes) - * @returns size of data read, or 0 in case of end-of-file - * - * read may block until data from other end is available. It will - * return 0 if the other end closed the pipe. - */ - size_type read(void* addr, size_type sz); - - /** @brief write to pipe - * - * @param addr where to get data to write from - * @param sz size of data to write (in bytes) - * @returns size of data written, or 0 in case of end-of-file - * - * write may block until data can be written to other end (depends a - * bit on available buffer space). It will return 0 if the other end - * closed the pipe. The data is queued to be written on the next - * convenient occasion, or it can be forced out with flush(). - */ - size_type write(const void* addr, size_type sz); - - /** @brief flush buffers with unwritten data - * - * This forces unwritten data to be written to the other end. The call - * will block until this has been done (or the attempt failed with an - * error). - */ - void flush(); - - /** @brief purge buffered data waiting to be read and/or written - * - * Discards all internal buffers. - */ - void purge(); - - /** @brief number of bytes that can be read without blocking - * - * @returns number of bytes that can be read without blocking - */ - size_type bytesReadableNonBlocking(); - - /** @brief number of bytes that can be written without blocking - * - * @returns number of bytes that can be written without blocking - */ - size_type bytesWritableNonBlocking(); - - /** @brief flush buffers, close pipe - * - * Flush buffers, discard unread data, closes the pipe. If the pipe is - * in the parent process, it waits for the child. - * - * @returns exit code of child process in parent, zero in child - */ - int close(); - - /** @brief return PID of the process on the other end of the pipe - * - * @returns PID of the process running on the remote end - */ - pid_t pidOtherEnd() const - { return isChild() ? m_parentPid : m_childPid; } - - /// condition flags for poll - enum PollFlags { - None = 0, ///< nothing special on this pipe - Readable = 1, ///< pipe has data for reading - Writable = 2, ///< pipe can be written to - ReadError = 4, ///< pipe error read end - WriteError = 8, ///< pipe error Write end - Error = ReadError | WriteError, ///< pipe error - ReadEndOfFile = 32, ///< read pipe in end-of-file state - WriteEndOfFile = 64,///< write pipe in end-of-file state - EndOfFile = ReadEndOfFile | WriteEndOfFile, ///< end of file - ReadInvalid = 64, ///< read end of pipe invalid - WriteInvalid = 128, ///< write end of pipe invalid - Invalid = ReadInvalid | WriteInvalid ///< invalid pipe - }; - - /// for poll() interface - class PollEntry { - public: - BidirMMapPipe* pipe; ///< pipe of interest - unsigned events; ///< events of interest (or'ed bitmask) - unsigned revents; ///< events that happened (or'ed bitmask) - /// poll a pipe for all events - PollEntry(BidirMMapPipe* _pipe) : - pipe(_pipe), events(None), revents(None) { } - /// poll a pipe for specified events - PollEntry(BidirMMapPipe* _pipe, int _events) : - pipe(_pipe), events(_events), revents(None) { } - }; - /// convenience typedef for poll() interface - typedef std::vector PollVector; - - /** @brief poll a set of pipes for events (ready to read from, ready to - * write to, error) - * - * @param pipes set of pipes to check - * @param timeout timeout in milliseconds - * @returns positive number: number of pipes which have - * status changes, 0: timeout, or no pipes with - * status changed, -1 on error - * - * Timeout can be zero (check for specified events, and return), finite - * (wait at most timeout milliseconds before returning), or -1 - * (infinite). The poll method returns when the timeout has elapsed, - * or if an event occurs on one of the pipes being polled, whichever - * happens earlier. - * - * Pipes is a vector of one or more PollEntries, which each list a pipe - * and events to poll for. If events is left empty (zero), all - * conditions are polled for, otherwise only the indicated ones. On - * return, the revents fields contain the events that occurred for each - * pipe; error Error, EndOfFile or Invalid events are always set, - * regardless of whether they were in the set of requested events. - * - * poll may block slightly longer than specified by timeout due to OS - * timer granularity and OS scheduling. Due to its implementation, the - * poll call can also return early if the remote end of the page sends - * a free page while polling (which is put on that pipe's freelist), - * while that pipe is polled for e.g Reading. The status of the pipe is - * indicated correctly in revents, and the caller can simply poll - * again. (The reason this is done this way is because it helps to - * replenish the pool of free pages and queue busy pages without - * blocking.) - * - * Here's a piece of example code waiting on two pipes; if they become - * readable they are read: - * @code - * #include - * #include - * #include - * #include - * #include - * - * #include "BidirMMapPipe.h" - * - * // what to execute in the child - * int randomchild(BidirMMapPipe& pipe) - * { - * ::srand48(::getpid()); - * for (int i = 0; i < 5; ++i) { - * // sleep a random time between 0 and .9 seconds - * ::usleep(int(1e6 * ::drand48())); - * std::ostringstream buf; - * buf << "child pid " << ::getpid() << " sends message " << i; - * std::cout << "[CHILD] : " << buf.str() << std::endl; - * pipe << buf.str() << BidirMMapPipe::flush; - * if (!pipe) return -1; - * if (pipe.eof()) break; - * } - * // tell parent we're done - * pipe << "" << BidirMMapPipe::flush; - * // wait for parent to acknowledge - * std::string s; - * pipe >> s; - * pipe.close(); - * return 0; - * } - * - * // function to spawn a child - * BidirMMapPipe* spawnChild(int (*childexec)(BidirMMapPipe&)) - * { - * BidirMMapPipe *p = new BidirMMapPipe(); - * if (p->isChild()) { - * int retVal = childexec(*p); - * delete p; - * std::exit(retVal); - * } - * return p; - * } - * - * int main() - * { - * typedef BidirMMapPipe::PollEntry PollEntry; - * // poll data structure - * BidirMMapPipe::PollVector pipes; - * pipes.reserve(3); - * // spawn children - * for (int i = 0; i < 3; ++i) { - * pipes.push_back(PollEntry(spawnChild(randomchild), - * BidirMMapPipe::Readable)); - * } - * // while at least some children alive - * while (!pipes.empty()) { - * // poll, wait until status change (infinite timeout) - * int npipes = BidirMMapPipe::poll(pipes, -1); - * // scan for pipes with changed status - * for (std::vector::iterator it = pipes.begin(); - * npipes && pipes.end() != it; ) { - * if (!it->revents) { - * // unchanged, next one - * ++it; - * continue; - * } - * --npipes; // maybe we can stop early... - * // read from pipes which are readable - * if (it->revents & BidirMMapPipe::Readable) { - * std::string s; - * *(it->pipe) >> s; - * if (!s.empty()) { - * std::cout << "[PARENT]: Read from pipe " << - * it->pipe << ": " << s << std::endl; - * ++it; - * continue; - * } else { - * // child is shutting down... - * *(it->pipe) << "" << BidirMMapPipe::flush; - * goto childcloses; - * } - * } - * // retire pipes with error or end-of-file condition - * if (it->revents & (BidirMMapPipe::Error | - * BidirMMapPipe::EndOfFile | - * BidirMMapPipe::Invalid)) { - * std::cout << "[PARENT]: Error on pipe " << - * it->pipe << " revents " << it->revents << - * std::endl; - * childcloses: - * std::cout << "[PARENT]:\tchild exit status: " << - * it->pipe->close() << std::endl; - * if (retVal) return retVal; - * delete it->pipe; - * it = pipes.erase(it); - * continue; - * } - * } - * } - * return 0; - * } - * @endcode - */ - static int poll(PollVector& pipes, int timeout); - - /** @brief return if this end of the pipe is the parent end - * - * @returns true if parent end of pipe - */ - bool isParent() const { return m_childPid; } - - /** @brief return if this end of the pipe is the child end - * - * @returns true if child end of pipe - */ - bool isChild() const { return !m_childPid; } - - /** @brief if BidirMMapPipe uses a socketpair for communications - * - * @returns true if BidirMMapPipe uses a socketpair for communications - */ - bool usesSocketpair() const { return m_inpipe == m_outpipe; } - - /** @brief if BidirMMapPipe uses a pipe pair for communications - * - * @returns true if BidirMMapPipe uses a pipe pair for communications - */ - bool usesPipepair() const { return m_inpipe != m_outpipe; } - - /** @brief return flags (end of file, BidirMMapPipe closed, ...) - * - * @returns flags (end of file, BidirMMapPipe closed, ...) - */ - int rdstate() const { return m_flags; } - - /** @brief true if end-of-file - * - * @returns true if end-of-file - */ - bool eof() const { return m_flags & eofbit; } - - /** @brief logical failure (e.g. I/O on closed BidirMMapPipe) - * - * @returns true in case of grave logical error (I/O on closed pipe,...) - */ - bool fail() const { return m_flags & failbit; } - - /** @brief true on I/O error - * - * @returns true on I/O error - */ - bool bad() const { return m_flags & badbit; } - - /** @brief status of stream is good - * - * @returns true if pipe is good (no errors, eof, ...) - */ - bool good() const { return !(m_flags & (eofbit | failbit | badbit)); } - - /** @brief true if closed - * - * @returns true if stream is closed - */ - bool closed() const { return m_flags & failbit; } - - /** @brief return true if not serious error (fail/bad) - * - * @returns true if stream is does not have serious error (fail/bad) - * - * (if EOF, this is still true) - */ - operator bool() const { return !fail() && !bad(); } - - /** @brief return true if serious error (fail/bad) - * - * @returns true if stream has a serious error (fail/bad) - */ - bool operator!() const { return fail() || bad(); } - -#ifdef STREAMOP -#undef STREAMOP -#endif -#define STREAMOP(TYPE) \ - BidirMMapPipe& operator<<(const TYPE& val) \ - { write(&val, sizeof(TYPE)); return *this; } \ - BidirMMapPipe& operator>>(TYPE& val) \ - { read(&val, sizeof(TYPE)); return *this; } - STREAMOP(bool); ///< C++ style stream operators for bool - STREAMOP(char); ///< C++ style stream operators for char - STREAMOP(short); ///< C++ style stream operators for short - STREAMOP(int); ///< C++ style stream operators for int - STREAMOP(long); ///< C++ style stream operators for long - STREAMOP(long long); ///< C++ style stream operators for long long - STREAMOP(unsigned char); ///< C++ style stream operators for unsigned char - STREAMOP(unsigned short); ///< C++ style stream operators for unsigned short - STREAMOP(unsigned int); ///< C++ style stream operators for unsigned int - STREAMOP(unsigned long); ///< C++ style stream operators for unsigned long - STREAMOP(unsigned long long); ///< C++ style stream operators for unsigned long long - STREAMOP(float); ///< C++ style stream operators for float - STREAMOP(double); ///< C++ style stream operators for double -#undef STREAMOP - - /** @brief write a C-style string - * - * @param str C-style string - * @returns pipe written to - */ - BidirMMapPipe& operator<<(const char* str); - - /** @brief read a C-style string - * - * @param str pointer to string (space allocated with malloc!) - * @returns pipe read from - * - * since this is for C-style strings, we use malloc/realloc/free for - * strings. passing in a nullptr pointer is valid here, and the routine - * will use realloc to allocate a chunk of memory of the right size. - */ - BidirMMapPipe& operator>>(char* (&str)); - - /** @brief write a std::string object - * - * @param str string to write - * @returns pipe written to - */ - BidirMMapPipe& operator<<(const std::string& str); - - /** @brief read a std::string object - * - * @param str string to be read - * @returns pipe read from - */ - BidirMMapPipe& operator>>(std::string& str); - - /** @brief write raw pointer to T to other side - * - * NOTE: This will not write the pointee! Only the value of the - * pointer is transferred. - * - * @param tptr pointer to be written - * @returns pipe written to - */ - template BidirMMapPipe& operator<<(const T* tptr) - { write(&tptr, sizeof(tptr)); return *this; } - - /** @brief read raw pointer to T from other side - * - * NOTE: This will not read the pointee! Only the value of the - * pointer is transferred. - * - * @param tptr pointer to be read - * @returns pipe read from - */ - template BidirMMapPipe& operator>>(T* &tptr) - { read(&tptr, sizeof(tptr)); return *this; } - - /** @brief I/O manipulator support - * - * @param manip manipulator - * @returns pipe with manipulator applied - * - * example: - * @code - * pipe << BidirMMapPipe::flush; - * @endcode - */ - BidirMMapPipe& operator<<(BidirMMapPipe& (*manip)(BidirMMapPipe&)) - { return manip(*this); } - - /** @brief I/O manipulator support - * - * @param manip manipulator - * @returns pipe with manipulator applied - * - * example: - * @code - * pipe >> BidirMMapPipe::purge; - * @endcode - */ - BidirMMapPipe& operator>>(BidirMMapPipe& (*manip)(BidirMMapPipe&)) - { return manip(*this); } - - /// for usage a la "pipe << flush;" - static BidirMMapPipe& flush(BidirMMapPipe& pipe) { pipe.flush(); return pipe; } - /// for usage a la "pipe << purge;" - static BidirMMapPipe& purge(BidirMMapPipe& pipe) { pipe.purge(); return pipe; } - - private: - /// copy-construction forbidden - BidirMMapPipe(const BidirMMapPipe&); - /// assignment forbidden - BidirMMapPipe& operator=(const BidirMMapPipe&) { return *this; } - - /// page is our friend - friend class BidirMMapPipe_impl::Page; - /// convenience typedef for Page - typedef BidirMMapPipe_impl::Page Page; - - /// tuning constants - enum { - // TotPages = 16 will give 32k buffers at 4k page size for both - // parent and child; if your average message to send is larger - // than this, consider raising the value (max 256) - TotPages = 16, ///< pages shared (child + parent) - - PagesPerEnd = TotPages / 2, ///< pages per pipe end - - // if FlushThresh pages are filled, the code forces a flush; 3/4 - // of the pages available seems to work quite well - FlushThresh = (3 * PagesPerEnd) / 4 ///< flush threshold - }; - - // per-class members - static pthread_mutex_t s_openpipesmutex; ///< protects s_openpipes - /// list of open BidirMMapPipes - static std::list s_openpipes; - /// pool of mmapped pages - static BidirMMapPipe_impl::PagePool* s_pagepool; - /// page pool reference counter - static unsigned s_pagepoolrefcnt; - /// debug flag - static int s_debugflag; - - /// return page pool - static BidirMMapPipe_impl::PagePool& pagepool(); - - // per-instance members - BidirMMapPipe_impl::Pages m_pages; ///< mmapped pages - Page* m_busylist; ///< linked list: busy pages (data to be read) - Page* m_freelist; ///< linked list: free pages - Page* m_dirtylist; ///< linked list: dirty pages (data to be sent) - int m_inpipe; ///< pipe end from which data may be read - int m_outpipe; ///< pipe end to which data may be written - int m_flags; ///< flags (e.g. end of file) - pid_t m_childPid; ///< pid of the child (zero if we're child) - pid_t m_parentPid; ///< pid of the parent - - /// cleanup routine - at exit, we want our children to get a SIGTERM... - static void teardownall(void); - - /// return length of a page list - static unsigned lenPageList(const Page* list); - - /** "feed" the busy and free lists with a list of pages - * - * @param plist linked list of pages - * - * goes through plist, puts free pages from plist onto the freelist - * (or sends them to the remote end if they belong there), and puts - * non-empty pages on plist onto the busy list - */ - void feedPageLists(Page* plist); - - /// put on dirty pages list - void markPageDirty(Page* p); - - /// transfer bytes through the pipe (reading, writing, may block) - static size_type xferraw(int fd, void* addr, size_type len, - ssize_t (*xferfn)(int, void*, std::size_t)); - /// transfer bytes through the pipe (reading, writing, may block) - static size_type xferraw(int fd, void* addr, const size_type len, - ssize_t (*xferfn)(int, const void*, std::size_t)) - { - return xferraw(fd, addr, len, - reinterpret_cast(xferfn)); - } - - /** @brief send page(s) to the other end (may block) - * - * @param plist linked list of pages to send - * - * the implementation gathers the different write(s) wherever - * possible; if mmap works, this results in a single write to transfer - * the list of pages sent, if we need to copy things through the pipe, - * we have one write to transfer which pages are sent, and then one - * write per page. - */ - void sendpages(Page* plist); - - /** @brief receive a pages from the other end (may block), queue them - * - * @returns number of pages received - * - * this is an application-level scatter read, which gets the list of - * pages to read from the pipe. if mmap works, it needs only one read - * call (to get the head of the list of pages transferred). if we need - * to copy pages through the pipe, we need to add one read for each - * empty page, and two reads for each non-empty page. - */ - unsigned recvpages(); - - /** @brief receive pages from other end (non-blocking) - * - * @returns number of pages received - * - * like recvpages(), but does not block if nothing is available for - * reading - */ - unsigned recvpages_nonblock(); - - /// get a busy page to read data from (may block) - Page* busypage(); - /// get a dirty page to write data to (may block) - Page* dirtypage(); - - /// close the pipe (no flush if forced) - int doClose(bool force, bool holdlock = false); - /// perform the flush - void doFlush(bool forcePartialPages = true); -#endif //_WIN32 -}; - -END_NAMESPACE_ROOFIT - -#undef BEGIN_NAMESPACE_ROOFIT -#undef END_NAMESPACE_ROOFIT - -#endif // BIDIRMMAPPIPE_H - -// vim: ft=cpp:sw=4:tw=78:et - -/// \endcond diff --git a/roofit/roofitcore/src/FitHelpers.cxx b/roofit/roofitcore/src/FitHelpers.cxx index 7431401a962c2..2608b66a9bb3b 100644 --- a/roofit/roofitcore/src/FitHelpers.cxx +++ b/roofit/roofitcore/src/FitHelpers.cxx @@ -45,14 +45,9 @@ #include "RooFitImplHelpers.h" #include "RooFit/Detail/RooNLLVarNew.h" -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#include "RooChi2Var.h" -#include "RooNLLVar.h" - #ifdef ROOFIT_MULTIPROCESS #include "RooFit/MultiProcess/Config.h" #endif -#endif using RooFit::Detail::RooNLLVarNew; @@ -792,7 +787,6 @@ std::unique_ptr createNLL(RooAbsPdf &pdf, RooAbsData &data, const Ro const bool ext = interpretExtendedCmdArg(pdf, pc.getInt("ext")); int splitRange = pc.getInt("splitRange"); - int cloneData = pc.getInt("cloneData"); auto offset = static_cast(pc.getInt("doOffset")); if (pc.hasProcessed("Range")) { @@ -844,150 +838,42 @@ std::unique_ptr createNLL(RooAbsPdf &pdf, RooAbsData &data, const Ro auto evalBackend = static_cast(pc.getInt("EvalBackend")); - // Construct BatchModeNLL if requested - if (evalBackend != RooFit::EvalBackend::Value::Legacy) { + RooArgSet normSet; + pdf.getObservables(data.get(), normSet); - RooArgSet normSet; - pdf.getObservables(data.get(), normSet); - - auto *simPdfForProjDeps = dynamic_cast(&pdf); - if (simPdfForProjDeps && simPdfForProjDeps->indexCatIsObservable(normSet)) { - for (auto i : projDeps) { - auto res = normSet.find(i->GetName()); - if (res != nullptr) { - res->setAttribute("__conditional__"); - } + auto *simPdfForProjDeps = dynamic_cast(&pdf); + if (simPdfForProjDeps && simPdfForProjDeps->indexCatIsObservable(normSet)) { + for (auto i : projDeps) { + auto res = normSet.find(i->GetName()); + if (res != nullptr) { + res->setAttribute("__conditional__"); } - } else { - normSet.remove(projDeps); - } - - std::unique_ptr pdfClone = - compilePdfForFit(pdf, normSet, rangeName, splitRange, addCoefRangeName, /*likelihoodMode=*/true); - - if (addCoefRangeName) { - oocxcoutI(&pdf, Fitting) << "RooAbsPdf::fitTo(" << pdf.GetName() - << ") fixing interpretation of coefficients of any component to range " - << addCoefRangeName << "\n"; - } - - std::unique_ptr compiledConstr; - if (std::unique_ptr constr = createConstr()) { - compiledConstr = RooFit::Detail::compileForNormSet(*constr, *data.get()); - compiledConstr->addOwnedComponents(std::move(constr)); } - - auto nll = createNLLNew(*pdfClone, data, std::move(compiledConstr), rangeName ? rangeName : "", projDeps, ext, - pc.getDouble("IntegrateBins"), offset); - - const double correction = pdfClone->getCorrection(); - - if (correction > 0) { - oocoutI(&pdf, Fitting) << "[FitHelpers] Detected correction term from RooAbsPdf::getCorrection(). " - << "Adding penalty to NLL." << std::endl; - - // Convert the multiplicative correction to an additive term in -log L - auto penaltyTerm = std::make_unique((baseName + "_Penalty").c_str(), - "Penalty term from getCorrection()", correction); - - // add penalty and NLL - auto correctedNLL = std::make_unique((baseName + "_corrected").c_str(), "NLL + penalty", - RooArgSet{*nll, *penaltyTerm}); - - // transfer ownership of terms - correctedNLL->addOwnedComponents(std::move(nll), std::move(penaltyTerm)); - nll = std::move(correctedNLL); - } - - auto nllWrapper = std::make_unique( - *nll, &data, evalBackend == RooFit::EvalBackend::Value::Cuda, rangeName ? rangeName : "", pdfClone.get(), - takeGlobalObservablesFromData); - - // We destroy the timing scrope for createNLL prematurely, because we - // separately measure the time for jitting and gradient creation - // inside the RooFuncWrapper. - timingScope.reset(); - - if (evalBackend == RooFit::EvalBackend::Value::Codegen) { - nllWrapper->generateGradient(); - } - if (evalBackend == RooFit::EvalBackend::Value::CodegenNoGrad) { - nllWrapper->setUseGeneratedFunctionCode(true); - } - - nllWrapper->addOwnedComponents(std::move(nll)); - nllWrapper->addOwnedComponents(std::move(pdfClone)); - - return nllWrapper; + } else { + normSet.remove(projDeps); } - std::unique_ptr nll; + std::unique_ptr pdfClone = + compilePdfForFit(pdf, normSet, rangeName, splitRange, addCoefRangeName, /*likelihoodMode=*/true); -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - bool verbose = pc.getInt("verbose"); - - int numcpu = pc.getInt("numcpu"); - int numcpu_strategy = pc.getInt("interleave"); - // strategy 3 works only for RooSimultaneous. - if (numcpu_strategy == 3 && !pdf.InheritsFrom("RooSimultaneous")) { - oocoutW(&pdf, Minimization) << "Cannot use a NumCpu Strategy = 3 when the pdf is not a RooSimultaneous, " - "falling back to default strategy = 0" - << std::endl; - numcpu_strategy = 0; + if (addCoefRangeName) { + oocxcoutI(&pdf, Fitting) << "RooAbsPdf::fitTo(" << pdf.GetName() + << ") fixing interpretation of coefficients of any component to range " + << addCoefRangeName << "\n"; } - RooFit::MPSplit interl = (RooFit::MPSplit)numcpu_strategy; - - auto binnedLInfo = RooHelpers::getBinnedL(pdf); - RooAbsPdf &actualPdf = binnedLInfo.binnedPdf ? *binnedLInfo.binnedPdf : pdf; - - // Construct NLL - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); - RooAbsTestStatistic::Configuration cfg; - cfg.addCoefRangeName = addCoefRangeName ? addCoefRangeName : ""; - cfg.nCPU = numcpu; - cfg.interleave = interl; - cfg.verbose = verbose; - cfg.splitCutRange = static_cast(splitRange); - cfg.cloneInputData = static_cast(cloneData); - cfg.integrateOverBinsPrecision = pc.getDouble("IntegrateBins"); - cfg.binnedL = binnedLInfo.isBinnedL; - cfg.takeGlobalObservablesFromData = takeGlobalObservablesFromData; - cfg.rangeName = rangeName ? rangeName : ""; - auto nllVar = std::make_unique(baseName.c_str(), "-log(likelihood)", actualPdf, data, projDeps, ext, cfg); - nllVar->enableBinOffsetting(offset == RooFit::OffsetMode::Bin); - nll = std::move(nllVar); - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::PrintErrors); - // Include constraints, if any, in likelihood - if (std::unique_ptr constraintTerm = createConstr()) { - - // Even though it is technically only required when the computation graph - // is changed because global observables are taken from data, it is safer - // to clone the constraint model in general to reset the normalization - // integral caches and avoid ASAN build failures (the PDF of the main - // measurement is cloned too anyway, so not much overhead). This can be - // reconsidered after the caching of normalization sets by pointer is changed - // to a more memory-safe solution. - constraintTerm = RooHelpers::cloneTreeWithSameParameters(*constraintTerm, data.get()); - - // Redirect the global observables to the ones from the dataset if applicable. - constraintTerm->setData(data, false); - - // The computation graph for the constraints is very small, no need to do - // the tracking of clean and dirty nodes here. - constraintTerm->setOperMode(RooAbsArg::ADirty); - - auto orignll = std::move(nll); - nll = std::make_unique((baseName + "_with_constr").c_str(), "nllWithCons", - RooArgSet(*orignll, *constraintTerm)); - nll->addOwnedComponents(std::move(orignll), std::move(constraintTerm)); + std::unique_ptr compiledConstr; + if (std::unique_ptr constr = createConstr()) { + compiledConstr = RooFit::Detail::compileForNormSet(*constr, *data.get()); + compiledConstr->addOwnedComponents(std::move(constr)); } - if (offset == RooFit::OffsetMode::Initial) { - nll->enableOffsetting(true); - } + auto nll = createNLLNew(*pdfClone, data, std::move(compiledConstr), rangeName ? rangeName : "", projDeps, ext, + pc.getDouble("IntegrateBins"), offset); - if (const double correction = pdf.getCorrection(); correction > 0) { + const double correction = pdfClone->getCorrection(); + + if (correction > 0) { oocoutI(&pdf, Fitting) << "[FitHelpers] Detected correction term from RooAbsPdf::getCorrection(). " << "Adding penalty to NLL." << std::endl; @@ -995,19 +881,35 @@ std::unique_ptr createNLL(RooAbsPdf &pdf, RooAbsData &data, const Ro auto penaltyTerm = std::make_unique((baseName + "_Penalty").c_str(), "Penalty term from getCorrection()", correction); - auto correctedNLL = std::make_unique( - // add penalty and NLL - (baseName + "_corrected").c_str(), "NLL + penalty", RooArgSet(*nll, *penaltyTerm)); + // add penalty and NLL + auto correctedNLL = std::make_unique((baseName + "_corrected").c_str(), "NLL + penalty", + RooArgSet{*nll, *penaltyTerm}); // transfer ownership of terms correctedNLL->addOwnedComponents(std::move(nll), std::move(penaltyTerm)); nll = std::move(correctedNLL); } -#else - throw std::runtime_error("RooFit was not built with the legacy evaluation backend"); -#endif - return nll; + auto nllWrapper = std::make_unique( + *nll, &data, evalBackend == RooFit::EvalBackend::Value::Cuda, rangeName ? rangeName : "", pdfClone.get(), + takeGlobalObservablesFromData); + + // We destroy the timing scrope for createNLL prematurely, because we + // separately measure the time for jitting and gradient creation + // inside the RooFuncWrapper. + timingScope.reset(); + + if (evalBackend == RooFit::EvalBackend::Value::Codegen) { + nllWrapper->generateGradient(); + } + if (evalBackend == RooFit::EvalBackend::Value::CodegenNoGrad) { + nllWrapper->setUseGeneratedFunctionCode(true); + } + + nllWrapper->addOwnedComponents(std::move(nll)); + nllWrapper->addOwnedComponents(std::move(pdfClone)); + + return nllWrapper; } std::unique_ptr createChi2(RooAbsReal &real, RooDataHist &data, const RooLinkedList &cmdList) @@ -1063,116 +965,78 @@ std::unique_ptr createChi2(RooAbsReal &real, RooDataHist &data, cons rangeName = "fit"; } - if (evalBackend != RooFit::EvalBackend::Value::Legacy) { - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); + RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); + + const int splitRange = pc.getInt("splitRange"); + resetFitrangeAttributes(real, data, baseName, rangeName, splitRange); + + std::unique_ptr wrapper; + + // Function mode: the input is a non-pdf RooAbsReal. We can short-circuit + // the pdf-compilation pipeline since there's no real pdf to normalize. + if (!pdf) { + RooArgSet observables; + real.getObservables(data.get(), observables); + RooNLLVarNew::Config cfg; + cfg.statistic = RooNLLVarNew::Statistic::Chi2; + cfg.chi2ErrorType = etype; + auto chi2 = std::make_unique(baseName.c_str(), baseName.c_str(), real, observables, cfg); + wrapper = std::make_unique( + *chi2, &data, evalBackend == RooFit::EvalBackend::Value::Cuda, rangeName ? rangeName : "", + /*simPdf=*/nullptr, + /*takeGlobalObservablesFromData=*/true); + wrapper->addOwnedComponents(std::move(chi2)); + } else { + const bool extended = interpretExtendedCmdArg(*pdf, pc.getInt("extended")); - const int splitRange = pc.getInt("splitRange"); - resetFitrangeAttributes(real, data, baseName, rangeName, splitRange); + RooArgSet normSet; + pdf->getObservables(data.get(), normSet); - std::unique_ptr wrapper; + oocxcoutI(pdf, Fitting) << "createChi2(" << pdf->GetName() + << ") fixing normalization set for coefficient determination to observables in data\n"; + pdf->fixAddCoefNormalization(normSet, false); - // Function mode: the input is a non-pdf RooAbsReal. We can short-circuit - // the pdf-compilation pipeline since there's no real pdf to normalize. - if (!pdf) { + std::unique_ptr pdfClone = + compilePdfForFit(*pdf, normSet, rangeName, splitRange, pc.getString("addCoefRange", nullptr, true), + /*likelihoodMode=*/false); + + RooArgList binSamplingPdfs; + RooAbsPdf &finalPdf = applyIntegrateBinsWrapping(*pdfClone, data, pc.getDouble("integrate_bins"), binSamplingPdfs); + + std::unique_ptr chi2; + auto *simPdfClone = dynamic_cast(&finalPdf); + // Like in createNLLNew(): a "switch"-mode RooSimultaneous (index + // category not among the data columns) is treated as an ordinary pdf. + if (simPdfClone && simPdfClone->indexCatIsObservable(*data.get())) { + chi2 = std::unique_ptr{dynamic_cast( + createSimultaneousChi2(*simPdfClone, rangeName ? rangeName : "", extended, etype).release())}; + } else { RooArgSet observables; - real.getObservables(data.get(), observables); + finalPdf.getObservables(data.get(), observables); RooNLLVarNew::Config cfg; cfg.statistic = RooNLLVarNew::Statistic::Chi2; + cfg.extended = extended; cfg.chi2ErrorType = etype; - auto chi2 = std::make_unique(baseName.c_str(), baseName.c_str(), real, observables, cfg); - wrapper = std::make_unique( - *chi2, &data, evalBackend == RooFit::EvalBackend::Value::Cuda, rangeName ? rangeName : "", - /*simPdf=*/nullptr, - /*takeGlobalObservablesFromData=*/true); - wrapper->addOwnedComponents(std::move(chi2)); - } else { - const bool extended = interpretExtendedCmdArg(*pdf, pc.getInt("extended")); - - RooArgSet normSet; - pdf->getObservables(data.get(), normSet); - - oocxcoutI(pdf, Fitting) << "createChi2(" << pdf->GetName() - << ") fixing normalization set for coefficient determination to observables in data\n"; - pdf->fixAddCoefNormalization(normSet, false); - - std::unique_ptr pdfClone = - compilePdfForFit(*pdf, normSet, rangeName, splitRange, pc.getString("addCoefRange", nullptr, true), - /*likelihoodMode=*/false); - - RooArgList binSamplingPdfs; - RooAbsPdf &finalPdf = - applyIntegrateBinsWrapping(*pdfClone, data, pc.getDouble("integrate_bins"), binSamplingPdfs); - - std::unique_ptr chi2; - auto *simPdfClone = dynamic_cast(&finalPdf); - // Like in createNLLNew(): a "switch"-mode RooSimultaneous (index - // category not among the data columns) is treated as an ordinary pdf. - if (simPdfClone && simPdfClone->indexCatIsObservable(*data.get())) { - chi2 = std::unique_ptr{dynamic_cast( - createSimultaneousChi2(*simPdfClone, rangeName ? rangeName : "", extended, etype).release())}; - } else { - RooArgSet observables; - finalPdf.getObservables(data.get(), observables); - RooNLLVarNew::Config cfg; - cfg.statistic = RooNLLVarNew::Statistic::Chi2; - cfg.extended = extended; - cfg.chi2ErrorType = etype; - chi2 = std::make_unique(baseName.c_str(), baseName.c_str(), finalPdf, observables, cfg); - } - - wrapper = std::make_unique( - *chi2, &data, evalBackend == RooFit::EvalBackend::Value::Cuda, rangeName ? rangeName : "", pdfClone.get(), - /*takeGlobalObservablesFromData=*/true); - wrapper->addOwnedComponents(std::move(binSamplingPdfs)); - wrapper->addOwnedComponents(std::move(chi2)); - wrapper->addOwnedComponents(std::move(pdfClone)); + chi2 = std::make_unique(baseName.c_str(), baseName.c_str(), finalPdf, observables, cfg); } - if (evalBackend == RooFit::EvalBackend::Value::Codegen) { - wrapper->generateGradient(); - } - if (evalBackend == RooFit::EvalBackend::Value::CodegenNoGrad) { - wrapper->setUseGeneratedFunctionCode(true); - } - - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::PrintErrors); - return wrapper; + wrapper = std::make_unique( + *chi2, &data, evalBackend == RooFit::EvalBackend::Value::Cuda, rangeName ? rangeName : "", pdfClone.get(), + /*takeGlobalObservablesFromData=*/true); + wrapper->addOwnedComponents(std::move(binSamplingPdfs)); + wrapper->addOwnedComponents(std::move(chi2)); + wrapper->addOwnedComponents(std::move(pdfClone)); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - RooAbsTestStatistic::Configuration cfg; - - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); - - bool extended = false; - if (pdf) { - extended = interpretExtendedCmdArg(*pdf, pc.getInt("extended")); + if (evalBackend == RooFit::EvalBackend::Value::Codegen) { + wrapper->generateGradient(); + } + if (evalBackend == RooFit::EvalBackend::Value::CodegenNoGrad) { + wrapper->setUseGeneratedFunctionCode(true); } - - const char *addCoefRangeName = pc.getString("addCoefRange", nullptr, true); - int splitRange = pc.getInt("splitRange"); - - // Set the fitrange attribute of th PDF, add observables ranges for plotting - resetFitrangeAttributes(real, data, baseName, rangeName, splitRange); - - cfg.rangeName = rangeName ? rangeName : ""; - cfg.nCPU = pc.getInt("numcpu"); - cfg.interleave = RooFit::Interleave; - cfg.verbose = static_cast(pc.getInt("verbose")); - cfg.cloneInputData = false; - cfg.integrateOverBinsPrecision = pc.getDouble("integrate_bins"); - cfg.addCoefRangeName = addCoefRangeName ? addCoefRangeName : ""; - cfg.splitCutRange = static_cast(splitRange); - auto chi2 = std::make_unique(baseName.c_str(), baseName.c_str(), real, static_cast(data), - extended, etype, cfg); RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::PrintErrors); - - return chi2; -#else - throw std::runtime_error("createChi2() is not supported without the legacy evaluation backend"); - return nullptr; -#endif + return wrapper; } std::unique_ptr fitTo(RooAbsReal &real, RooAbsData &data, const RooLinkedList &cmdList, bool chi2) diff --git a/roofit/roofitcore/src/RooAbsOptTestStatistic.cxx b/roofit/roofitcore/src/RooAbsOptTestStatistic.cxx deleted file mode 100644 index 3870ec8fca329..0000000000000 --- a/roofit/roofitcore/src/RooAbsOptTestStatistic.cxx +++ /dev/null @@ -1,556 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/***************************************************************************** - * Project: RooFit * - * Package: RooFitCore * - * @(#)root/roofitcore:$Id$ - * Authors: * - * WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu * - * DK, David Kirkby, UC Irvine, dkirkby@uci.edu * - * * - * Copyright (c) 2000-2005, Regents of the University of California * - * and Stanford University. All rights reserved. * - * * - * Redistribution and use in source and binary forms, * - * with or without modification, are permitted according to the terms * - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * - *****************************************************************************/ - -/** -\file RooAbsOptTestStatistic.cxx -\class RooAbsOptTestStatistic -\ingroup Roofitcore - -Abstract base class for test -statistics objects that evaluate a function or PDF at each point of a given -dataset. This class provides generic optimizations, such as -caching and precalculation of constant terms that can be made for -all such quantities. - -Implementations should define evaluatePartition(), which calculates the -value of a (sub)range of the dataset and optionally combinedValue(), -which combines the values calculated for each partition. If combinedValue() -is not overloaded, the default implementation will add the partition results -to obtain the combined result. - -Support for calculation in partitions is needed to allow multi-core -parallelized calculation of test statistics. -**/ - -#include "RooAbsOptTestStatistic.h" - -#include "Riostream.h" -#include "TClass.h" -#include - -#include "RooAbsData.h" -#include "RooAbsDataStore.h" -#include "RooAbsPdf.h" -#include "RooAddPdf.h" -#include "RooArgSet.h" -#include "RooBinSamplingPdf.h" -#include "RooBinning.h" -#include "RooCategory.h" -#include "RooDataHist.h" -#include "RooDataSet.h" -#include "RooErrorHandler.h" -#include "RooFitImplHelpers.h" -#include "RooGlobalFunc.h" -#include "RooMsgService.h" -#include "RooProdPdf.h" -#include "RooProduct.h" -#include "RooRealSumPdf.h" -#include "RooRealVar.h" -#include "RooVectorDataStore.h" - -#include "ROOT/StringUtils.hxx" - -using std::ostream; - -//////////////////////////////////////////////////////////////////////////////// -/// Create a test statistic, and optimise its calculation. -/// \param[in] name Name of the instance. -/// \param[in] title Title (for e.g. plotting). -/// \param[in] real Function to evaluate. -/// \param[in] indata Dataset for which to compute test statistic. -/// \param[in] projDeps A set of projected observables. -/// \param[in] cfg the statistic configuration -/// -/// cfg contains: -/// - rangeName If not null, only events in the dataset inside the range will be used in the test -/// statistic calculation. -/// - addCoefRangeName If not null, all RooAddPdf components of `real` will be -/// instructed to fix their fraction definitions to the given named range. -/// - nCPU If > 1, the test statistic calculation will be parallelised over multiple processes. By default, the data -/// is split with 'bulk' partitioning (each process calculates a contiguous block of fraction 1/nCPU -/// of the data). For binned data, this approach may be suboptimal as the number of bins with >0 entries -/// in each processing block may vary greatly; thereby distributing the workload rather unevenly. -/// - interleave Strategy how to distribute events among workers. If an interleave partitioning strategy is used where each partition -/// i takes all bins for which (ibin % ncpu == i), an even distribution of work is more likely. -/// - splitCutRange If true, a different rangeName constructed as `rangeName_{catName}` will be used -/// as range definition for each index state of a RooSimultaneous. -/// - cloneInputData Not used. Data is always cloned. -/// - integrateOverBinsPrecision If > 0, PDF in binned fits are integrated over the bins. This sets the precision. If = 0, -/// only unbinned PDFs fit to RooDataHist are integrated. If < 0, PDFs are never integrated. -RooAbsOptTestStatistic::RooAbsOptTestStatistic(const char *name, const char *title, RooAbsReal &real, - RooAbsData &indata, const RooArgSet &projDeps, - RooAbsTestStatistic::Configuration const &cfg) - : RooAbsTestStatistic(name, title, real, indata, projDeps, cfg), - _integrateBinsPrecision(cfg.integrateOverBinsPrecision) -{ - // Don't do a thing in master mode - if (operMode() != Slave) { - return; - } - - initSlave(real, indata, projDeps, _rangeName.c_str(), _addCoefRangeName.c_str()); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Copy constructor - -RooAbsOptTestStatistic::RooAbsOptTestStatistic(const RooAbsOptTestStatistic &other, const char *name) - : RooAbsTestStatistic(other, name), - _sealed(other._sealed), - _sealNotice(other._sealNotice), - _integrateBinsPrecision(other._integrateBinsPrecision) -{ - // Don't do a thing in master mode - if (operMode() != Slave) { - - if (other._normSet) { - _normSet = new RooArgSet; - other._normSet->snapshot(*_normSet); - } - return; - } - - initSlave(*other._funcClone, *other._dataClone, other._projDeps ? *other._projDeps : RooArgSet(), - other._rangeName.c_str(), other._addCoefRangeName.c_str()); -} - - - -//////////////////////////////////////////////////////////////////////////////// - -void RooAbsOptTestStatistic::initSlave(RooAbsReal& real, RooAbsData& indata, const RooArgSet& projDeps, const char* rangeName, - const char* addCoefRangeName) { - // ****************************************************************** - // *** PART 1 *** Clone incoming pdf, attach to each other * - // ****************************************************************** - - // Clone FUNC - _funcClone = RooHelpers::cloneTreeWithSameParameters(real, indata.get()).release(); - _funcCloneSet = nullptr ; - - // Attach FUNC to data set - _funcObsSet = std::unique_ptr{_funcClone->getObservables(indata)}.release(); - - if (_funcClone->getAttribute("BinnedLikelihood")) { - _funcClone->setAttribute("BinnedLikelihoodActive") ; - } - - // Mark all projected dependents as such - if (!projDeps.empty()) { - std::unique_ptr projDataDeps{_funcObsSet->selectCommon(projDeps)}; - projDataDeps->setAttribAll("projectedDependent") ; - } - - // If PDF is a RooProdPdf (with possible constraint terms) - // analyze pdf for actual parameters (i.e those in unconnected constraint terms should be - // ignored as here so that the test statistic will not be recalculated if those - // are changed - RooProdPdf* pdfWithCons = dynamic_cast(_funcClone) ; - if (pdfWithCons) { - - std::unique_ptr connPars{pdfWithCons->getConnectedParameters(*indata.get())}; - // Add connected parameters as servers - _paramSet.add(*connPars) ; - - } else { - // Add parameters as servers - _funcClone->getParameters(indata.get(), _paramSet); - } - - // Store normalization set - _normSet = new RooArgSet; - indata.get()->snapshot(*_normSet, false); - - // Expand list of observables with any observables used in parameterized ranges. - // This NEEDS to be a counting loop since we are inserting during the loop. - for (std::size_t i = 0; i < _funcObsSet->size(); ++i) { - auto realDepRLV = dynamic_cast((*_funcObsSet)[i]); - if (realDepRLV && realDepRLV->isDerived()) { - RooArgSet tmp2; - realDepRLV->leafNodeServerList(&tmp2, nullptr, true); - _funcObsSet->add(tmp2,true); - } - } - - - - // ****************************************************************** - // *** PART 2 *** Clone and adjust incoming data, attach to PDF * - // ****************************************************************** - - // Check if the fit ranges of the dependents in the data and in the FUNC are consistent - const RooArgSet* dataDepSet = indata.get() ; - for (const auto arg : *_funcObsSet) { - - // Check that both dataset and function argument are of type RooRealVar - RooRealVar* realReal = dynamic_cast(arg) ; - if (!realReal) continue ; - RooRealVar* datReal = dynamic_cast(dataDepSet->find(realReal->GetName())) ; - if (!datReal) continue ; - - // Check that range of observables in pdf is equal or contained in range of observables in data - - if (!realReal->getBinning().lowBoundFunc() && realReal->getMin()<(datReal->getMin()-1e-6)) { - coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR minimum of FUNC observable " << arg->GetName() - << "(" << realReal->getMin() << ") is smaller than that of " - << arg->GetName() << " in the dataset (" << datReal->getMin() << ")" << std::endl ; - RooErrorHandler::softAbort() ; - return ; - } - - if (!realReal->getBinning().highBoundFunc() && realReal->getMax()>(datReal->getMax()+1e-6)) { - coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR maximum of FUNC observable " << arg->GetName() - << " is larger than that of " << arg->GetName() << " in the dataset" << std::endl ; - RooErrorHandler::softAbort() ; - return ; - } - } - - // Copy data and strip entries lost by adjusted fit range, _dataClone ranges will be copied from realDepSet ranges - if (rangeName && strlen(rangeName)) { - _dataClone = std::unique_ptr{indata.reduce(RooFit::SelectVars(*_funcObsSet),RooFit::CutRange(rangeName))}.release(); - } else { - _dataClone = static_cast(indata.Clone()) ; - } - _ownData = true ; - - - // ****************************************************************** - // *** PART 3 *** Make adjustments for fit ranges, if specified * - // ****************************************************************** - - std::unique_ptr origObsSet( real.getObservables(indata) ); - if (rangeName && strlen(rangeName)) { - cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") constructing test statistic for sub-range named " << rangeName << std::endl ; - - if(auto pdfClone = dynamic_cast(_funcClone)) { - pdfClone->setNormRange(rangeName); - } - - // Print warnings if the requested ranges are not available for the observable - for (const auto arg : *_funcObsSet) { - - if (auto realObs = dynamic_cast(arg)) { - - auto tokens = ROOT::Split(rangeName, ","); - for(std::string const& token : tokens) { - if(!realObs->hasRange(token.c_str())) { - std::stringstream errMsg; - errMsg << "The observable \"" << realObs->GetName() << "\" doesn't define the requested range \"" - << token << "\". Replacing it with the default range." << std::endl; - coutI(Fitting) << errMsg.str() << std::endl; - } - } - } - } - } - - - // ****************************************************************** - // *** PART 3.2 *** Binned fits * - // ****************************************************************** - - setUpBinSampling(); - - - // Fix RooAddPdf coefficients to original normalization range - if (rangeName && strlen(rangeName)) { - - // WVE Remove projected dependents from normalization - _funcClone->fixAddCoefNormalization(*_dataClone->get(),false) ; - - if (addCoefRangeName && strlen(addCoefRangeName)) { - cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() - << ") fixing interpretation of coefficients of any RooAddPdf component to range " << addCoefRangeName << std::endl ; - _funcClone->fixAddCoefRange(addCoefRangeName,false) ; - } - } - - - // This is deferred from part 2 - but must happen after part 3 - otherwise invalid bins cannot be properly marked in cacheValidEntries - _dataClone->attachBuffers(*_funcObsSet) ; - setEventCount(_dataClone->numEntries()) ; - - - - - // ********************************************************************* - // *** PART 4 *** Adjust normalization range for projected observables * - // ********************************************************************* - - // Remove projected dependents from normalization set - if (!projDeps.empty()) { - - _projDeps = new RooArgSet; - projDeps.snapshot(*_projDeps, false) ; - - //RooArgSet* tobedel = (RooArgSet*) _normSet->selectCommon(*_projDeps) ; - _normSet->remove(*_projDeps,true,true) ; - - // Mark all projected dependents as such - RooArgSet projDataDeps; - _funcObsSet->selectCommon(*_projDeps, projDataDeps); - projDataDeps.setAttribAll("projectedDependent") ; - } - - - coutI(Optimization) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") optimizing internal clone of p.d.f for likelihood evaluation." - << "Lazy evaluation and associated change tracking will disabled for all nodes that depend on observables" << std::endl ; - - - // ********************************************************************* - // *** PART 4 *** Finalization and activation of optimization * - // ********************************************************************* - - // Redirect pointers of base class to clone - _func = _funcClone ; - _data = _dataClone ; - - _funcClone->getVal(_normSet) ; - - optimizeCaching() ; - - // It would be unusual if the global observables are used in the likelihood - // outside of the constraint terms, but if they are we have to be consistent - // and also redirect them to the snapshots in the dataset if appropriate. - if(_takeGlobalObservablesFromData && _data->getGlobalObservables()) { - recursiveRedirectServers(*_data->getGlobalObservables()) ; - } - -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Destructor - -RooAbsOptTestStatistic::~RooAbsOptTestStatistic() -{ - if (operMode()==Slave) { - delete _funcClone ; - delete _funcObsSet ; - if (_projDeps) { - delete _projDeps ; - } - if (_ownData) { - delete _dataClone ; - } - } - delete _normSet ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Method to combined test statistic results calculated into partitions into -/// the global result. This default implementation adds the partition return -/// values - -double RooAbsOptTestStatistic::combinedValue(RooAbsReal** array, Int_t n) const -{ - // Default implementation returns sum of components - double sum(0); - double carry(0); - for (Int_t i = 0; i < n; ++i) { - double y = array[i]->getValV(); - carry += reinterpret_cast(array[i])->getCarry(); - y -= carry; - const double t = sum + y; - carry = (t - sum) - y; - sum = t; - } - _evalCarry = carry; - return sum ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Catch server redirect calls and forward to internal clone of function - -bool RooAbsOptTestStatistic::redirectServersHook(const RooAbsCollection& newServerList, bool mustReplaceAll, bool nameChange, bool isRecursive) -{ - RooAbsTestStatistic::redirectServersHook(newServerList,mustReplaceAll,nameChange,isRecursive) ; - if (operMode()!=Slave) return false ; - bool ret = _funcClone->recursiveRedirectServers(newServerList,false,nameChange) ; - return ret || RooAbsReal::redirectServersHook(newServerList, mustReplaceAll, nameChange, isRecursive); -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Catch print hook function and forward to function clone - -void RooAbsOptTestStatistic::printCompactTreeHook(ostream& os, const char* indent) -{ - RooAbsTestStatistic::printCompactTreeHook(os,indent) ; - if (operMode()!=Slave) return ; - TString indent2(indent) ; - indent2 += "opt >>" ; - _funcClone->printCompactTree(os,indent2.Data()) ; - os << indent2 << " dataset clone = " << _dataClone << " first obs = " << _dataClone->get()->first() << std::endl ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// This method changes the value caching logic for all nodes that depends on any of the observables -/// as defined by the given dataset. When evaluating a test statistic constructed from the RooAbsReal -/// with a dataset the observables are guaranteed to change with every call, thus there is no point -/// in tracking these changes which result in a net overhead. Thus for observable-dependent nodes, -/// the evaluation mechanism is changed from being dependent on a 'valueDirty' flag to guaranteed evaluation. -/// On the dataset side, the observables objects are modified to no longer send valueDirty messages -/// to their client - -void RooAbsOptTestStatistic::optimizeCaching() -{ - // Trigger create of all object caches now in nodes that have deferred object creation - // so that cache contents can be processed immediately - _funcClone->getVal(_normSet) ; - - // Set value caching mode for all nodes that depend on any of the observables to ADirty - _funcClone->optimizeCacheMode(*_funcObsSet) ; - - // Disable propagation of dirty state flags for observables - _dataClone->setDirtyProp(false) ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Change dataset that is used to given one. If cloneData is true, a clone of -/// in the input dataset is made. If the test statistic was constructed with -/// a range specification on the data, the cloneData argument is ignored and -/// the data is always cloned. -bool RooAbsOptTestStatistic::setDataSlave(RooAbsData& indata, bool cloneData, bool ownNewData) -{ - - if (operMode()==SimMaster) { - return false ; - } - - - // If the current dataset is owned, transfer the ownership to unique pointer - // that will get out of scope at the end of this function. We can't delete it - // right now, because there might be global observables in the model that - // first need to be redirected to the new dataset with a later call to - // RooAbsArg::recursiveRedirectServers. - std::unique_ptr oldOwnedData; - if (_ownData) { - oldOwnedData.reset(_dataClone); - _dataClone = nullptr ; - } - - if (!cloneData && !_rangeName.empty()) { - coutW(InputArguments) << "RooAbsOptTestStatistic::setData(" << GetName() << ") WARNING: test statistic was constructed with range selection on data, " - << "ignoring request to _not_ clone the input dataset" << std::endl ; - cloneData = true ; - } - - if (cloneData) { - // Cloning input dataset - _dataClone = std::unique_ptr{indata.reduce(RooFit::SelectVars(*indata.get()),RooFit::CutRange(_rangeName.c_str()))}.release(); - _ownData = true ; - - } else { - - // Taking input dataset - _dataClone = &indata ; - _ownData = ownNewData ; - - } - - // Attach function clone to dataset - _dataClone->attachBuffers(*_funcObsSet) ; - _dataClone->setDirtyProp(false) ; - _data = _dataClone ; - - // Adjust internal event count - setEventCount(indata.numEntries()) ; - - setValueDirty() ; - - // It would be unusual if the global observables are used in the likelihood - // outside of the constraint terms, but if they are we have to be consistent - // and also redirect them to the snapshots in the dataset if appropriate. - if(_takeGlobalObservablesFromData && _data->getGlobalObservables()) { - recursiveRedirectServers(*_data->getGlobalObservables()) ; - } - - return true ; -} - - - - -//////////////////////////////////////////////////////////////////////////////// - -RooAbsData& RooAbsOptTestStatistic::data() -{ - if (_sealed) { - bool notice = (sealNotice() && strlen(sealNotice())) ; - coutW(ObjectHandling) << "RooAbsOptTestStatistic::data(" << GetName() - << ") WARNING: object sealed by creator - access to data is not permitted: " - << (notice?sealNotice():"") << std::endl ; - static RooDataSet dummy ("dummy","dummy",RooArgSet()) ; - return dummy ; - } - return *_dataClone ; -} - - -//////////////////////////////////////////////////////////////////////////////// - -const RooAbsData& RooAbsOptTestStatistic::data() const -{ - if (_sealed) { - bool notice = (sealNotice() && strlen(sealNotice())) ; - coutW(ObjectHandling) << "RooAbsOptTestStatistic::data(" << GetName() - << ") WARNING: object sealed by creator - access to data is not permitted: " - << (notice?sealNotice():"") << std::endl ; - static RooDataSet dummy ("dummy","dummy",RooArgSet()) ; - return dummy ; - } - return *_dataClone ; -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Inspect PDF to find out if we are doing a binned fit to a 1-dimensional unbinned PDF. -/// If this is the case, enable finer sampling of bins by wrapping PDF into a RooBinSamplingPdf. -/// The member _integrateBinsPrecision decides how we act: -/// - < 0: Don't do anything. -/// - = 0: Only enable feature if fitting unbinned PDF to RooDataHist. -/// - > 0: Enable as requested. -void RooAbsOptTestStatistic::setUpBinSampling() { - - auto& pdf = static_cast(*_funcClone); - if (auto newPdf = RooBinSamplingPdf::create(pdf, *_dataClone, _integrateBinsPrecision)) { - newPdf->addOwnedComponents(*_funcClone); - _funcClone = newPdf.release(); - } - -} - - -/// Returns a suffix string that is unique for RooAbsOptTestStatistic -/// instances that don't share the same cloned input data object. -const char* RooAbsOptTestStatistic::cacheUniqueSuffix() const { - return Form("_%lx", _dataClone->uniqueId().value()) ; -} - -/// \endcond diff --git a/roofit/roofitcore/src/RooAbsOptTestStatistic.h b/roofit/roofitcore/src/RooAbsOptTestStatistic.h deleted file mode 100644 index 7c20885dbeee9..0000000000000 --- a/roofit/roofitcore/src/RooAbsOptTestStatistic.h +++ /dev/null @@ -1,86 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/* - * Project: RooFit - * - * Copyright (c) 2024, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ROO_ABS_OPT_TEST_STATISTIC -#define ROO_ABS_OPT_TEST_STATISTIC - -#include "RooAbsTestStatistic.h" -#include "RooSetProxy.h" -#include "RooCategoryProxy.h" -#include "TString.h" - -class RooArgSet ; -class RooAbsData ; -class RooAbsReal ; - -class RooAbsOptTestStatistic : public RooAbsTestStatistic { -public: - - // Constructors, assignment etc - RooAbsOptTestStatistic(const char *name, const char *title, RooAbsReal& real, RooAbsData& data, - const RooArgSet& projDeps, - RooAbsTestStatistic::Configuration const& cfg); - RooAbsOptTestStatistic(const RooAbsOptTestStatistic& other, const char* name=nullptr); - ~RooAbsOptTestStatistic() override; - - double combinedValue(RooAbsReal** gofArray, Int_t nVal) const override ; - - RooAbsReal& function() { return *_funcClone ; } - const RooAbsReal& function() const { return *_funcClone ; } - - RooAbsData& data() ; - const RooAbsData& data() const ; - - - const char* cacheUniqueSuffix() const override; - - // Override this to be always true to force calculation of likelihood without parameters - bool isDerived() const override { return true ; } - - void seal(const char* notice="") { _sealed = true ; _sealNotice = notice ; } - bool isSealed() const { return _sealed ; } - const char* sealNotice() const { return _sealNotice.Data() ; } - -private: - void setUpBinSampling(); - -protected: - - bool setDataSlave(RooAbsData& data, bool cloneData=true, bool ownNewDataAnyway=false) override ; - void initSlave(RooAbsReal& real, RooAbsData& indata, const RooArgSet& projDeps, const char* rangeName, - const char* addCoefRangeName) ; - - friend class RooAbsReal ; - friend class RooAbsTestStatistic ; - - bool redirectServersHook(const RooAbsCollection& newServerList, bool mustReplaceAll, bool nameChange, bool isRecursive) override ; - void printCompactTreeHook(std::ostream& os, const char* indent="") override ; - void optimizeCaching() ; - - RooArgSet* _normSet = nullptr; ///< Pointer to set with observables used for normalization - RooArgSet* _funcCloneSet = nullptr; ///< Set owning all components of internal clone of input function - RooAbsData* _dataClone = nullptr; ///< Pointer to internal clone if input data - RooAbsReal* _funcClone = nullptr; ///< Pointer to internal clone of input function - RooArgSet* _projDeps = nullptr; ///< Set of projected observable - bool _ownData = false; ///< Do we own the dataset - bool _sealed = false; ///< Is test statistic sealed -- i.e. no access to data - TString _sealNotice ; ///< User-defined notice shown when reading a sealed likelihood - RooArgSet* _funcObsSet = nullptr; ///< List of observables in the pdf expression - - RooAbsReal* _origFunc = nullptr; ///< Original function - RooAbsData* _origData = nullptr; ///< Original data - double _integrateBinsPrecision{-1.}; // Precision for finer sampling of bins. -}; - -#endif - -/// \endcond diff --git a/roofit/roofitcore/src/RooAbsPdf.cxx b/roofit/roofitcore/src/RooAbsPdf.cxx index 74e111b66e3ca..dc002f514c747 100644 --- a/roofit/roofitcore/src/RooAbsPdf.cxx +++ b/roofit/roofitcore/src/RooAbsPdf.cxx @@ -826,39 +826,18 @@ double RooAbsPdf::extendedTerm(RooAbsData const& data, bool weightSquared, bool * \f] * `Range(double lo, double hi)` Fit only data inside given range. A range named "fit" is created on the fly on all observables. * `SumCoefRange(const char* name)` Set the range in which to interpret the coefficients of RooAddPdf components - * `NumCPU(int num, int istrat)` Parallelize NLL calculation on num CPUs. (Currently, this setting is ignored with the **cpu** Backend.) - * - *
Strategy Effect - *
0 = RooFit::BulkPartition - *default* Divide events in N equal chunks - *
1 = RooFit::Interleave Process event i%N in process N. Recommended for binned data with - * a substantial number of zero-bins, which will be distributed across processes more equitably in this strategy - *
2 = RooFit::SimComponents Process each component likelihood of a RooSimultaneous fully in a single process - * and distribute components over processes. This approach can be beneficial if normalization calculation time - * dominates the total computation time of a component (since the normalization calculation must be performed - * in each process in strategies 0 and 1. However beware that if the RooSimultaneous components do not share many - * parameters this strategy is inefficient: as most minuit-induced likelihood calculations involve changing - * a single parameter, only 1 of the N processes will be active most of the time if RooSimultaneous components - * do not share many parameters - *
3 = RooFit::Hybrid Follow strategy 0 for all RooSimultaneous components, except those with less than - * 30 dataset entries, for which strategy 2 is followed. - *
+ * `NumCPU(int num, int istrat)` \warning Deprecated option that is ignored. + * It was used to parallelize the NLL calculation of the removed legacy evaluation backend. * `EvalBackend(std::string const&)` Choose a likelihood evaluation backend: * *
Backend Description - *
**cpu** - *default* New vectorized evaluation mode, using faster math functions and auto-vectorisation (currently on a single thread). - * Since ROOT 6.23, this is the default if `EvalBackend()` is not passed, succeeding the **legacy** backend. - * If all RooAbsArg objects in the model support vectorized evaluation, - * likelihood computations are 2 to 10 times faster than with the **legacy** backend (each on a single thread). - * - unless your dataset is so small that the vectorization is not worth it. - * The relative difference of the single log-likelihoods with respect to the legacy mode is usually better than \f$10^{-12}\f$, - * and for fit parameters it's usually better than \f$10^{-6}\f$. In past ROOT releases, this backend could be activated with the now deprecated `BatchMode()` option. + *
**cpu** - *default* Vectorized evaluation mode, using faster math functions and auto-vectorisation (currently on a single thread). + * Since ROOT 6.23, this is the default if `EvalBackend()` is not passed. + * In past ROOT releases, this backend could be activated with the now deprecated `BatchMode()` option. *
**cuda** Evaluate the likelihood on a GPU that supports CUDA. * This backend re-uses code from the **cpu** backend, but compiled in CUDA kernels. * Hence, the results are expected to be identical, modulo some numerical differences that can arise from the different order in which the GPU is summing the log probabilities. * This backend can drastically speed up the fit if all RooAbsArg object in the model support it. - *
**legacy** The original likelihood evaluation method. - * Evaluates the PDF for each single data entry at a time before summing the negative log probabilities. - * It supports multi-threading, but you might need more than 20 threads to maybe see about 10% performance gain over the default cpu-backend (that runs currently only on a single thread). *
**codegen** **Experimental** - Generates and compiles minimal C++ code for the NLL on-the-fly and wraps it in the returned RooAbsReal. * Also generates and compiles the code for the gradient using Automatic Differentiation (AD) with [Clad](https://github.com/vgvassilev/clad). * This analytic gradient is passed to the minimizer, which can result in significant speedups for many-parameter fits, diff --git a/roofit/roofitcore/src/RooAbsReal.cxx b/roofit/roofitcore/src/RooAbsReal.cxx index ad4f7fde38ddb..7a5dc9ddd9ba6 100644 --- a/roofit/roofitcore/src/RooAbsReal.cxx +++ b/roofit/roofitcore/src/RooAbsReal.cxx @@ -3307,7 +3307,7 @@ double RooAbsReal::maxVal(Int_t /*code*/) const //////////////////////////////////////////////////////////////////////////////// -/// Interface to insert remote error logging messages received by RooRealMPFE into current error logging stream. +/// Interface to insert error logging messages from an external originator into the current error logging stream. void RooAbsReal::logEvalError(const RooAbsReal* originator, const char* origName, const char* message, const char* serverValueString) { @@ -3988,7 +3988,7 @@ double RooAbsReal::findRoot(RooRealVar& x, double xmin, double xmax, double yval ///
`Range(const char* name)` Fit only data inside range with given name ///
`Range(double lo, double hi)` Fit only data inside given range. A range named "fit" is created on the fly on all observables. /// Multiple comma separated range names can be specified. -///
`NumCPU(int num)` Parallelize NLL calculation on num CPUs +///
`NumCPU(int num)` \warning Deprecated option that is ignored. ///
`IntegrateBins()` Integrate PDF within each bin. This sets the desired precision. ///
`Verbose()` Verbose output of GOF framework ///
`SumCoefRange()` Set the range in which to interpret the coefficients of RooAddPdf components @@ -4055,15 +4055,10 @@ std::unique_ptr RooAbsReal::chi2FitToImpl(RooDataHist &data, const /// expected number of events that the PDF predicts. /// /// \note If the dataset has errors stored, empty bins will prevent the calculation of \f$ \chi^2 \f$, because those have -/// zero error. This leads to messages like: -/// ``` -/// [#0] ERROR:Eval -- RooChi2Var::RooChi2Var(chi2_GenPdf_data_hist) INFINITY ERROR: bin 2 has zero error -/// ``` +/// zero error. /// -/// \note In this case, one can use the expected errors of the PDF instead of the data errors: -/// ```{.cpp} -/// RooChi2Var chi2(..., ..., RooFit::DataError(RooAbsData::Expected), ...); -/// ``` +/// \note In this case, one can use the expected errors of the PDF instead of the data errors, +/// by passing `RooFit::DataError(RooAbsData::Expected)` as a command argument. /// /// \param data Histogram with data /// \param arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8 ordered arguments diff --git a/roofit/roofitcore/src/RooAbsTestStatistic.cxx b/roofit/roofitcore/src/RooAbsTestStatistic.cxx deleted file mode 100644 index 65e31b3f5b4e5..0000000000000 --- a/roofit/roofitcore/src/RooAbsTestStatistic.cxx +++ /dev/null @@ -1,606 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/***************************************************************************** - * Project: RooFit * - * Package: RooFitCore * - * @(#)root/roofitcore:$Id$ - * Authors: * - * WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu * - * DK, David Kirkby, UC Irvine, dkirkby@uci.edu * - * * - * Copyright (c) 2000-2005, Regents of the University of California * - * and Stanford University. All rights reserved. * - * * - * Redistribution and use in source and binary forms, * - * with or without modification, are permitted according to the terms * - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * - *****************************************************************************/ - -/** -\file RooAbsTestStatistic.cxx -\class RooAbsTestStatistic -\ingroup Roofitcore - -Abstract base class for all test -statistics. Test statistics that evaluate the PDF at each data -point should inherit from the RooAbsOptTestStatistic class which -implements several generic optimizations that can be done for such -quantities. - -This test statistic base class organizes calculation of test -statistic values for RooSimultaneous PDF as a combination of test -statistic values for the PDF components of the simultaneous PDF and -organizes multi-processor parallel calculation of test statistic -values. For the latter, the test statistic value is calculated in -partitions in parallel executing processes and a posteriori -combined in the main thread. -**/ - -#include "RooAbsTestStatistic.h" - -#include "RooAbsPdf.h" -#include "RooSimultaneous.h" -#include "RooAbsData.h" -#include "RooArgSet.h" -#include "RooRealVar.h" -#include "RooRealMPFE.h" -#include "RooErrorHandler.h" -#include "RooMsgService.h" -#include "RooAbsCategoryLValue.h" -#include "RooFitImplHelpers.h" -#include "RooAbsOptTestStatistic.h" -#include "RooCategory.h" - -#include "TTimeStamp.h" -#include "TClass.h" -#include -#include - -using std::endl, std::ostream; - -//////////////////////////////////////////////////////////////////////////////// -/// Create a test statistic from the given function and the data. -/// \param[in] name Name of the test statistic -/// \param[in] title Title (for plotting) -/// \param[in] real Function to be used for tests -/// \param[in] data Data to fit function to -/// \param[in] projDeps A set of projected observables -/// \param[in] cfg statistic configuration object -/// -/// cfg contains: -/// - rangeName Fit data only in range with given name -/// - addCoefRangeName If not null, all RooAddPdf components of `real` will be instructed to fix their fraction definitions to the given named range. -/// - nCPU If larger than one, the test statistic calculation will be parallelized over multiple processes. -/// By default the data is split with 'bulk' partitioning (each process calculates a contiguous block of fraction 1/nCPU -/// of the data). For binned data this approach may be suboptimal as the number of bins with >0 entries -/// in each processing block many vary greatly thereby distributing the workload rather unevenly. -/// - interleave is set to true, the interleave partitioning strategy is used where each partition -/// i takes all bins for which (ibin % ncpu == i) which is more likely to result in an even workload. -/// - verbose Be more verbose. -/// - splitCutRange If true, a different rangeName constructed as rangeName_{catName} will be used -/// as range definition for each index state of a RooSimultaneous. This means that a different range can be defined -/// for each category such as -/// ``` -/// myVariable.setRange("range_pi0", 135, 210); -/// myVariable.setRange("range_gamma", 50, 210); -/// ``` -/// if the categories are called "pi0" and "gamma". - -namespace { - -/// A RooSimultaneous implies a simultaneous fit over the states of its index -/// category only if the index category is among the data columns. Otherwise, -/// it acts as a "switch" pdf that evaluates to the component selected by the -/// current index state (analogous to RooMultiPdf), and the test statistic has -/// to treat it like any ordinary pdf instead of splitting the data. -bool isSimultaneousFit(RooAbsReal &real, RooAbsData &data) -{ - auto *simPdf = dynamic_cast(&real); - return simPdf && simPdf->indexCatIsObservable(*data.get()); -} - -} // namespace - -RooAbsTestStatistic::RooAbsTestStatistic(const char *name, const char *title, RooAbsReal& real, RooAbsData& data, - const RooArgSet& projDeps, RooAbsTestStatistic::Configuration const& cfg) : - RooAbsReal(name,title), - _paramSet("paramSet","Set of parameters",this), - _func(&real), - _data(&data), - _projDeps(static_cast(projDeps.Clone())), - _rangeName(cfg.rangeName), - _addCoefRangeName(cfg.addCoefRangeName), - _splitRange(cfg.splitCutRange), - _verbose(cfg.verbose), - // Determine if RooAbsReal implies a simultaneous fit over channels - _gofOpMode{(cfg.nCPU>1 || cfg.nCPU==-1) ? MPMaster : (isSimultaneousFit(real, data) ? SimMaster : Slave)}, - _nEvents{data.numEntries()}, - _nCPU(cfg.nCPU != -1 ? cfg.nCPU : 1), - _mpinterl(cfg.interleave), - _takeGlobalObservablesFromData{cfg.takeGlobalObservablesFromData} -{ - // Register all parameters as servers - _paramSet.add(*std::unique_ptr{real.getParameters(&data)}); -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Copy constructor - -RooAbsTestStatistic::RooAbsTestStatistic(const RooAbsTestStatistic& other, const char* name) : - RooAbsReal(other,name), - _paramSet("paramSet","Set of parameters",this), - _func(other._func), - _data(other._data), - _projDeps(static_cast(other._projDeps->Clone())), - _rangeName(other._rangeName), - _addCoefRangeName(other._addCoefRangeName), - _splitRange(other._splitRange), - _verbose(other._verbose), - // Determine if RooAbsReal implies a simultaneous fit over channels - _gofOpMode{(other._nCPU>1 || other._nCPU==-1) ? MPMaster - : (isSimultaneousFit(*other._func, *other._data) ? SimMaster : Slave)}, - _nEvents{_data->numEntries()}, - _nCPU(other._nCPU != -1 ? other._nCPU : 1), - _mpinterl(other._mpinterl), - _doOffset(other._doOffset), - _takeGlobalObservablesFromData{other._takeGlobalObservablesFromData}, - _offset(other._offset), - _evalCarry(other._evalCarry) -{ - // Our parameters are those of original - _paramSet.add(other._paramSet) ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Destructor - -RooAbsTestStatistic::~RooAbsTestStatistic() -{ - if (MPMaster == _gofOpMode && _init) { - for (Int_t i = 0; i < _nCPU; ++i) delete _mpfeArray[i]; - delete[] _mpfeArray ; - } - - delete _projDeps ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Calculate and return value of test statistic. If the test statistic -/// is calculated from a RooSimultaneous, the test statistic calculation -/// is performed separately on each simultaneous p.d.f component and associated -/// data, and then combined. If the test statistic calculation is parallelized, -/// partitions are calculated in nCPU processes and combined a posteriori. - -double RooAbsTestStatistic::evaluate() const -{ - // One-time Initialization - if (!_init) { - const_cast(this)->initialize() ; - } - - if (SimMaster == _gofOpMode) { - // Evaluate array of owned GOF objects - double ret = 0.; - - if (_mpinterl == RooFit::BulkPartition || _mpinterl == RooFit::Interleave ) { - ret = combinedValue(reinterpret_cast(const_cast*>(_gofArray.data())),_gofArray.size()); - } else { - double sum = 0.; - double carry = 0.; - int i = 0; - for (auto& gof : _gofArray) { - if (i % _numSets == _setNum || (_mpinterl==RooFit::Hybrid && gof->_mpinterl != RooFit::SimComponents )) { - double y = gof->getValV(); - carry += gof->getCarry(); - y -= carry; - const double t = sum + y; - carry = (t - sum) - y; - sum = t; - } - ++i; - } - ret = sum ; - _evalCarry = carry; - } - - // Only apply global normalization if SimMaster doesn't have MP master - if (numSets()==1) { - const double norm = globalNormalization(); - ret /= norm; - _evalCarry /= norm; - } - - return ret ; - - } else if (MPMaster == _gofOpMode) { - - // Start calculations in parallel - for (Int_t i = 0; i < _nCPU; ++i) _mpfeArray[i]->calculate(); - - double sum(0); - double carry = 0.; - for (Int_t i = 0; i < _nCPU; ++i) { - double y = _mpfeArray[i]->getValV(); - carry += _mpfeArray[i]->getCarry(); - y -= carry; - const double t = sum + y; - carry = (t - sum) - y; - sum = t; - } - - double ret = sum ; - _evalCarry = carry; - - const double norm = globalNormalization(); - ret /= norm; - _evalCarry /= norm; - - return ret ; - - } else { - - // Evaluate as straight FUNC - Int_t nFirst(0); - Int_t nLast(_nEvents); - Int_t nStep(1); - - switch (_mpinterl) { - case RooFit::BulkPartition: - nFirst = _nEvents * _setNum / _numSets ; - nLast = _nEvents * (_setNum+1) / _numSets ; - nStep = 1 ; - break; - - case RooFit::Interleave: - nFirst = _setNum ; - nLast = _nEvents ; - nStep = _numSets ; - break ; - - case RooFit::SimComponents: - nFirst = 0 ; - nLast = _nEvents ; - nStep = 1 ; - break ; - - case RooFit::Hybrid: - throw std::logic_error("this should never happen"); - break ; - } - - double ret = evaluatePartition(nFirst,nLast,nStep); - - if (numSets()==1) { - const double norm = globalNormalization(); - ret /= norm; - _evalCarry /= norm; - } - - return ret ; - - } -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// One-time initialization of the test statistic. Setup -/// infrastructure for simultaneous p.d.f processing and/or -/// parallelized processing if requested - -bool RooAbsTestStatistic::initialize() -{ - if (_init) return false; - - if (MPMaster == _gofOpMode) { - initMPMode(_func,_data,_projDeps,_rangeName,_addCoefRangeName) ; - } else if (SimMaster == _gofOpMode) { - initSimMode(static_cast(_func),_data,_projDeps,_rangeName,_addCoefRangeName) ; - } - _init = true; - return false; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Forward server redirect calls to component test statistics - -bool RooAbsTestStatistic::redirectServersHook(const RooAbsCollection& newServerList, bool mustReplaceAll, bool nameChange, bool isRecursive) -{ - if (SimMaster == _gofOpMode) { - // Forward to slaves - for(auto& gof : _gofArray) { - gof->recursiveRedirectServers(newServerList,mustReplaceAll,nameChange); - } - } else if (MPMaster == _gofOpMode&& _mpfeArray) { - // Forward to slaves - for (Int_t i = 0; i < _nCPU; ++i) { - if (_mpfeArray[i]) { - _mpfeArray[i]->recursiveRedirectServers(newServerList,mustReplaceAll,nameChange); -// std::cout << "redirecting servers on " << _mpfeArray[i]->GetName() << std::endl; - } - } - } - return RooAbsReal::redirectServersHook(newServerList, mustReplaceAll, nameChange, isRecursive); -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Add extra information on component test statistics when printing -/// itself as part of a tree structure - -void RooAbsTestStatistic::printCompactTreeHook(ostream& os, const char* indent) -{ - if (SimMaster == _gofOpMode) { - // Forward to slaves - os << indent << "RooAbsTestStatistic begin GOF contents" << std::endl ; - for (std::size_t i = 0; i < _gofArray.size(); ++i) { - TString indent2(indent); - indent2 += "[" + std::to_string(i) + "] "; - _gofArray[i]->printCompactTreeHook(os,indent2); - } - os << indent << "RooAbsTestStatistic end GOF contents" << std::endl; - } else if (MPMaster == _gofOpMode) { - // WVE implement this - } -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Set MultiProcessor set number identification of this instance - -void RooAbsTestStatistic::setMPSet(Int_t inSetNum, Int_t inNumSets) -{ - _setNum = inSetNum; _numSets = inNumSets; - _extSet = _mpinterl==RooFit::SimComponents ? _setNum : (_numSets - 1); - - if (SimMaster == _gofOpMode) { - // Forward to slaves - initialize(); - for(auto& gof : _gofArray) { - gof->setMPSet(inSetNum,inNumSets); - } - } -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Initialize multi-processor calculation mode. Create component test statistics in separate -/// processed that are connected to this process through a RooAbsRealMPFE front-end class. - -void RooAbsTestStatistic::initMPMode(RooAbsReal* real, RooAbsData* data, const RooArgSet* projDeps, std::string const& rangeName, std::string const& addCoefRangeName) -{ - _mpfeArray = new pRooRealMPFE[_nCPU]; - - // Create proto-goodness-of-fit - Configuration cfg; - cfg.rangeName = rangeName; - cfg.addCoefRangeName = addCoefRangeName; - cfg.nCPU = 1; - cfg.interleave = _mpinterl; - cfg.verbose = _verbose; - cfg.splitCutRange = _splitRange; - cfg.takeGlobalObservablesFromData = _takeGlobalObservablesFromData; - // This configuration parameter is stored in the RooAbsOptTestStatistic. - // It would have been cleaner to move the member variable into RooAbsTestStatistic, - // but to avoid incrementing the class version we do the dynamic_cast trick. - if(auto thisAsRooAbsOptTestStatistic = dynamic_cast(this)) { - cfg.integrateOverBinsPrecision = thisAsRooAbsOptTestStatistic->_integrateBinsPrecision; - } - RooAbsTestStatistic* gof = create(GetName(),GetTitle(),*real,*data,*projDeps,cfg); - gof->recursiveRedirectServers(_paramSet); - - for (Int_t i = 0; i < _nCPU; ++i) { - gof->setMPSet(i,_nCPU); - gof->SetName(Form("%s_GOF%d",GetName(),i)); - gof->SetTitle(Form("%s_GOF%d",GetTitle(),i)); - - ccoutD(Eval) << "RooAbsTestStatistic::initMPMode: starting remote server process #" << i << std::endl; - _mpfeArray[i] = new RooRealMPFE(Form("%s_%zx_MPFE%d",GetName(),reinterpret_cast(this),i),Form("%s_%zx_MPFE%d",GetTitle(),reinterpret_cast(this),i),*gof,false); - //_mpfeArray[i]->setVerbose(true,true); - _mpfeArray[i]->initialize(); - if (i > 0) { - _mpfeArray[i]->followAsSlave(*_mpfeArray[0]); - } - } - _mpfeArray[_nCPU - 1]->addOwnedComponents(*gof); - coutI(Eval) << "RooAbsTestStatistic::initMPMode: started " << _nCPU << " remote server process." << std::endl; - //cout << "initMPMode --- done" << std::endl ; - return ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Initialize simultaneous p.d.f processing mode. Strip simultaneous -/// p.d.f into individual components, split dataset in subset -/// matching each component and create component test statistics for -/// each of them. - -void RooAbsTestStatistic::initSimMode(RooSimultaneous* simpdf, RooAbsData* data, - const RooArgSet* projDeps, - std::string const& rangeName, std::string const& addCoefRangeName) -{ - - RooAbsCategoryLValue& simCat = const_cast(simpdf->indexCat()); - - std::vector> dsetList{const_cast(data)->split(*simpdf,processEmptyDataSets())}; - - // Create array of regular fit contexts, containing subset of data and single fitCat PDF - for (const auto& catState : simCat) { - const std::string& catName = catState.first; - RooAbsCategory::value_type catIndex = catState.second; - - // If the channel is not in the selected range of the category variable, we - // won't create a slave calculator for this channel. - if(!rangeName.empty()) { - // Only the RooCategory supports ranges, not the other - // RooAbsCategoryLValue-derived classes. - auto simCatAsRooCategory = dynamic_cast(&simCat); - if(simCatAsRooCategory && !simCatAsRooCategory->isStateInRange(rangeName.c_str(), catIndex)) { - continue; - } - } - - // Retrieve the PDF for this simCat state - RooAbsPdf* pdf = simpdf->getPdf(catName.c_str()); - auto found = std::find_if(dsetList.begin(), dsetList.end(), [&](auto const &item) { - return catName == item->GetName(); - }); - RooAbsData *dset = found != dsetList.end() ? found->get() : nullptr; - - if (pdf && dset && (0. != dset->sumEntries() || processEmptyDataSets())) { - ccoutI(Fitting) << "RooAbsTestStatistic::initSimMode: creating slave calculator #" << _gofArray.size() << " for state " << catName - << " (" << dset->numEntries() << " dataset entries)" << std::endl; - - - // *** START HERE - // WVE HACK determine if we have a RooRealSumPdf and then treat it like a binned likelihood - auto binnedInfo = RooHelpers::getBinnedL(*pdf); - RooAbsReal &actualPdf = binnedInfo.binnedPdf ? *binnedInfo.binnedPdf : *pdf; - // WVE END HACK - // Below here directly pass binnedPdf instead of PROD(binnedPdf,constraints) as constraints are evaluated elsewhere anyway - // and omitting them reduces model complexity and associated handling/cloning times - Configuration cfg; - cfg.addCoefRangeName = addCoefRangeName; - cfg.interleave = _mpinterl; - cfg.verbose = _verbose; - cfg.splitCutRange = _splitRange; - cfg.binnedL = binnedInfo.isBinnedL; - cfg.takeGlobalObservablesFromData = _takeGlobalObservablesFromData; - // This configuration parameter is stored in the RooAbsOptTestStatistic. - // It would have been cleaner to move the member variable into RooAbsTestStatistic, - // but to avoid incrementing the class version we do the dynamic_cast trick. - if(auto thisAsRooAbsOptTestStatistic = dynamic_cast(this)) { - cfg.integrateOverBinsPrecision = thisAsRooAbsOptTestStatistic->_integrateBinsPrecision; - } - cfg.rangeName = RooHelpers::getRangeNameForSimComponent(rangeName, _splitRange, catName); - cfg.nCPU = _nCPU; - _gofArray.emplace_back(create(catName.c_str(),catName.c_str(),actualPdf,*dset,*projDeps,cfg)); - // *** END HERE - - // Fill per-component split mode with Bulk Partition for now so that Auto will map to bulk-splitting of all components - if (_mpinterl==RooFit::Hybrid) { - _gofArray.back()->_mpinterl = dset->numEntries()<10 ? RooFit::SimComponents : RooFit::BulkPartition; - } - - // Servers may have been redirected between instantiation and (deferred) initialization - - RooArgSet actualParams; - actualPdf.getParameters(dset->get(), actualParams); - RooArgSet selTargetParams; - _paramSet.selectCommon(actualParams, selTargetParams); - - _gofArray.back()->recursiveRedirectServers(selTargetParams); - } - } - for(auto& gof : _gofArray) { - gof->setSimCount(_gofArray.size()); - } - coutI(Fitting) << "RooAbsTestStatistic::initSimMode: created " << _gofArray.size() << " slave calculators." << std::endl; -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Change dataset that is used to given one. If cloneData is true, a clone of -/// in the input dataset is made. If the test statistic was constructed with -/// a range specification on the data, the cloneData argument is ignored and -/// the data is always cloned. -bool RooAbsTestStatistic::setData(RooAbsData& indata, bool cloneData) -{ - // Trigger refresh of likelihood offsets - if (isOffsetting()) { - enableOffsetting(false); - enableOffsetting(true); - } - - switch(operMode()) { - case Slave: - // Delegate to implementation - return setDataSlave(indata, cloneData); - case SimMaster: - // Forward to slaves - if (indata.canSplitFast()) { - for(auto& gof : _gofArray) { - RooAbsData* compData = indata.getSimData(gof->GetName()); - gof->setDataSlave(*compData, cloneData); - } - } else if (0 == indata.numEntries()) { - // For an unsplit empty dataset, simply assign empty dataset to each component - for(auto& gof : _gofArray) { - gof->setDataSlave(indata, cloneData); - } - } else { - std::vector> dlist{indata.split(*static_cast(_func), processEmptyDataSets())}; - - for(auto& gof : _gofArray) { - auto found = std::find_if(dlist.begin(), dlist.end(), [&](auto const &item) { - return strcmp(gof->GetName(), item->GetName()) == 0; - }); - RooAbsData *compData = found != dlist.end() ? found->get() : nullptr; - if (compData) { - gof->setDataSlave(*compData,false,true); - } else { - coutE(DataHandling) << "RooAbsTestStatistic::setData(" << GetName() << ") ERROR: Cannot find component data for state " << gof->GetName() << std::endl; - } - } - } - break; - case MPMaster: - // Not supported - coutF(DataHandling) << "RooAbsTestStatistic::setData(" << GetName() << ") FATAL: setData() is not supported in multi-processor mode" << std::endl; - throw std::runtime_error("RooAbsTestStatistic::setData is not supported in MPMaster mode"); - break; - } - - return true; -} - - - -void RooAbsTestStatistic::enableOffsetting(bool flag) -{ - // Apply internal value offsetting to control numeric precision - if (!_init) { - const_cast(this)->initialize() ; - } - - switch(operMode()) { - case Slave: - _doOffset = flag ; - // Clear offset if feature is disabled to that it is recalculated next time it is enabled - if (!_doOffset) { - _offset = ROOT::Math::KahanSum{0.} ; - } - setValueDirty() ; - break ; - case SimMaster: - _doOffset = flag; - for(auto& gof : _gofArray) { - gof->enableOffsetting(flag); - } - break ; - case MPMaster: - _doOffset = flag; - for (Int_t i = 0; i < _nCPU; ++i) { - _mpfeArray[i]->enableOffsetting(flag); - } - break; - } -} - - -double RooAbsTestStatistic::getCarry() const -{ return _evalCarry; } - -/// \endcond diff --git a/roofit/roofitcore/src/RooAbsTestStatistic.h b/roofit/roofitcore/src/RooAbsTestStatistic.h deleted file mode 100644 index 4e63cbfe519e1..0000000000000 --- a/roofit/roofitcore/src/RooAbsTestStatistic.h +++ /dev/null @@ -1,157 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/* - * Project: RooFit - * - * Copyright (c) 2024, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ROO_ABS_TEST_STATISTIC -#define ROO_ABS_TEST_STATISTIC - -#include "RooAbsReal.h" -#include "RooSetProxy.h" -#include "RooRealProxy.h" -#include "Math/Util.h" - -#include -#include - -class RooArgSet ; -class RooAbsData ; -class RooAbsReal ; -class RooSimultaneous ; -class RooRealMPFE ; - -class RooAbsTestStatistic ; -typedef RooAbsData* pRooAbsData ; -typedef RooRealMPFE* pRooRealMPFE ; - -class RooAbsTestStatistic : public RooAbsReal { - friend class RooRealMPFE; -public: - - struct Configuration { - /// Stores the configuration parameters for RooAbsTestStatistic. - std::string rangeName; - std::string addCoefRangeName; - int nCPU = 1; - RooFit::MPSplit interleave = RooFit::BulkPartition; - bool verbose = true; - bool splitCutRange = false; - bool cloneInputData = true; - double integrateOverBinsPrecision = -1.; - bool binnedL = false; - bool takeGlobalObservablesFromData = false; - }; - - // Constructors, assignment etc - RooAbsTestStatistic(const char *name, const char *title, RooAbsReal& real, RooAbsData& data, - const RooArgSet& projDeps, Configuration const& cfg); - RooAbsTestStatistic(const RooAbsTestStatistic& other, const char* name=nullptr); - ~RooAbsTestStatistic() override; - virtual RooAbsTestStatistic* create(const char *name, const char *title, RooAbsReal& real, RooAbsData& data, - const RooArgSet& projDeps, Configuration const& cfg) = 0; - - virtual double combinedValue(RooAbsReal** gofArray, Int_t nVal) const = 0 ; - virtual double globalNormalization() const { - // Default value of global normalization factor is 1.0 - return 1.0 ; - } - - bool setData(RooAbsData& data, bool cloneData=true) override ; - - void enableOffsetting(bool flag) override ; - bool isOffsetting() const override { return _doOffset ; } - double offset() const override { return _offset.Sum() ; } - virtual double offsetCarry() const { return _offset.Carry(); } - - enum GOFOpMode { SimMaster,MPMaster,Slave } ; - GOFOpMode operMode() const { - // Return test statistic operation mode of this instance (SimMaster, MPMaster or Slave) - return _gofOpMode ; - } - -protected: - - void printCompactTreeHook(std::ostream& os, const char* indent="") override ; - - bool redirectServersHook(const RooAbsCollection& newServerList, bool mustReplaceAll, bool nameChange, bool isRecursive) override ; - double evaluate() const override ; - - virtual double evaluatePartition(std::size_t firstEvent, std::size_t lastEvent, std::size_t stepSize) const = 0 ; - virtual double getCarry() const; - - void setMPSet(Int_t setNum, Int_t numSets) ; - void setSimCount(Int_t simCount) { - // Store total number of components p.d.f. of a RooSimultaneous in this component test statistic - _simCount = simCount ; - } - - void setEventCount(Int_t nEvents) { - // Store total number of events in this component test statistic - _nEvents = nEvents ; - } - - Int_t numSets() const { - // Return total number of sets for parallel calculation - return _numSets ; - } - Int_t setNum() const { - // Return parallel calculation set number for this instance - return _setNum ; - } - - RooSetProxy _paramSet ; ///< Parameters of the test statistic (=parameters of the input function) - - - // Original arguments - RooAbsReal* _func = nullptr; ///< Pointer to original input function - RooAbsData* _data = nullptr; ///< Pointer to original input dataset - const RooArgSet* _projDeps = nullptr; ///< Pointer to set with projected observables - std::string _rangeName ; ///< Name of range in which to calculate test statistic - std::string _addCoefRangeName ; ///< Name of reference to be used for RooAddPdf components - bool _splitRange = false; ///< Split rangeName in RooSimultaneous index labels if true - Int_t _simCount = 1; ///< Total number of component p.d.f.s in RooSimultaneous (if any) - bool _verbose = false; ///< Verbose messaging if true - - virtual bool setDataSlave(RooAbsData& /*data*/, bool /*cloneData*/=true, bool /*ownNewDataAnyway*/=false) { return true ; } - - //private: - - - virtual bool processEmptyDataSets() const { return true ; } - - bool initialize() ; - void initSimMode(RooSimultaneous* pdf, RooAbsData* data, const RooArgSet* projDeps, std::string const& rangeName, std::string const& addCoefRangeName) ; - void initMPMode(RooAbsReal* real, RooAbsData* data, const RooArgSet* projDeps, std::string const& rangeName, std::string const& addCoefRangeName) ; - - mutable bool _init = false; ///> _gofArray; /// _offset {0.0}; /// #include @@ -154,50 +150,31 @@ void RooAddition::doEval(RooFit::EvalContext &ctx) const } //////////////////////////////////////////////////////////////////////////////// -/// Return the default error level for MINUIT error analysis -/// If the addition contains one or more RooNLLVars and -/// no RooChi2Vars, return the defaultErrorLevel() of -/// RooNLLVar. If the addition contains one ore more RooChi2Vars -/// and no RooNLLVars, return the defaultErrorLevel() of -/// RooChi2Var. If the addition contains neither or both -/// issue a warning message and return a value of 1 +/// Return the default error level for MINUIT error analysis. +/// If the addition contains a test statistic implemented by RooNLLVarNew, +/// return its defaultErrorLevel(). Otherwise, issue a warning message and +/// return a value of 1. double RooAddition::defaultErrorLevel() const { RooAbsReal* nllArg(nullptr) ; - RooAbsReal* chi2Arg(nullptr) ; std::unique_ptr comps{getComponents()}; for(RooAbsArg * arg : *comps) { if (dynamic_cast(arg)) { nllArg = static_cast(arg) ; } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - if (dynamic_cast(arg)) { - nllArg = static_cast(arg) ; - } - if (dynamic_cast(arg)) { - chi2Arg = static_cast(arg) ; - } -#endif } - if (nllArg && !chi2Arg) { + if (nllArg) { coutI(Fitting) << "RooAddition::defaultErrorLevel(" << GetName() - << ") Summation contains a RooNLLVar, using its error level" << std::endl; + << ") Summation contains a test statistic, using its error level" << std::endl; return nllArg->defaultErrorLevel() ; - } else if (chi2Arg && !nllArg) { - coutI(Fitting) << "RooAddition::defaultErrorLevel(" << GetName() - << ") Summation contains a RooChi2Var, using its error level" << std::endl; - return chi2Arg->defaultErrorLevel() ; - } else if (!nllArg && !chi2Arg) { - coutI(Fitting) << "RooAddition::defaultErrorLevel(" << GetName() << ") WARNING: " - << "Summation contains neither RooNLLVar nor RooChi2Var server, using default level of 1.0" << std::endl; - } else { - coutI(Fitting) << "RooAddition::defaultErrorLevel(" << GetName() << ") WARNING: " - << "Summation contains BOTH RooNLLVar and RooChi2Var server, using default level of 1.0" << std::endl; } + coutI(Fitting) << "RooAddition::defaultErrorLevel(" << GetName() << ") WARNING: " + << "Summation contains no test statistic server, using default level of 1.0" << std::endl; + return 1.0 ; } diff --git a/roofit/roofitcore/src/RooChi2Var.cxx b/roofit/roofitcore/src/RooChi2Var.cxx deleted file mode 100644 index 89020e32ece56..0000000000000 --- a/roofit/roofitcore/src/RooChi2Var.cxx +++ /dev/null @@ -1,150 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/***************************************************************************** - * Project: RooFit * - * Package: RooFitCore * - * @(#)root/roofitcore:$Id$ - * Authors: * - * WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu * - * DK, David Kirkby, UC Irvine, dkirkby@uci.edu * - * * - * Copyright (c) 2000-2005, Regents of the University of California * - * and Stanford University. All rights reserved. * - * * - * Redistribution and use in source and binary forms, * - * with or without modification, are permitted according to the terms * - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * - *****************************************************************************/ - -#include "RooChi2Var.h" - -#include "FitHelpers.h" -#include "RooDataHist.h" -#include "RooAbsPdf.h" -#include "RooCmdConfig.h" -#include "RooMsgService.h" - -#include "Riostream.h" -#include "TClass.h" - -#include "RooRealVar.h" -#include "RooAbsDataStore.h" - -#include - -RooChi2Var::RooChi2Var(const char *name, const char *title, RooAbsReal &func, RooDataHist &data, bool extended, - RooDataHist::ErrorType etype, RooAbsTestStatistic::Configuration const &cfg) - : RooAbsOptTestStatistic(name, title, func, data, RooArgSet{}, cfg), - _etype{etype == RooAbsData::Auto ? (data.isNonPoissonWeighted() ? RooAbsData::SumW2 : RooAbsData::Expected) - : etype}, - _funcMode{dynamic_cast(&func) ? (extended ? ExtendedPdf : Pdf) : Function} -{ -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Copy constructor - -RooChi2Var::RooChi2Var(const RooChi2Var& other, const char* name) : - RooAbsOptTestStatistic(other,name), - _etype(other._etype), - _funcMode(other._funcMode) -{ -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Calculate chi^2 in partition from firstEvent to lastEvent using given stepSize -/// Throughout the calculation, we use Kahan's algorithm for summing to -/// prevent loss of precision - this is a factor four more expensive than -/// straight addition, but since evaluating the PDF is usually much more -/// expensive than that, we tolerate the additional cost... - -double RooChi2Var::evaluatePartition(std::size_t firstEvent, std::size_t lastEvent, std::size_t stepSize) const -{ - double result(0); - double carry(0); - - // Also consider the composite case of multiple ranges - std::vector rangeTokens; - if (!_rangeName.empty()) { - rangeTokens = ROOT::Split(_rangeName, ","); - } - - // Determine normalization factor depending on type of input function - double normFactor(1) ; - switch (_funcMode) { - case Function: normFactor=1 ; break ; - case Pdf: normFactor = _dataClone->sumEntries() ; break ; - case ExtendedPdf: normFactor = (static_cast(_funcClone))->expectedEvents(_dataClone->get()) ; break ; - } - - // Loop over bins of dataset - RooDataHist* hdata = static_cast(_dataClone) ; - for (auto i=firstEvent ; iget(i); - - // Skip bins that are outside of the selected range - bool doSelect(true) ; - if (!_rangeName.empty()) { - doSelect = false; - // A row is selected if it is inside at least one complete named range. - for (const auto &rangeName : rangeTokens) { - bool inThisRange = true; - for (const auto arg : *row) { - if (!arg->inRange(rangeName.c_str())) { - inThisRange = false; - break; - } - } - if (inThisRange) { - doSelect = true; - break; - } - } - } - if (!doSelect) continue ; - - const double nData = hdata->weight(i) ; - - const double nPdf = _funcClone->getVal(_normSet) * normFactor * hdata->binVolume(i) ; - - const double eExt = nPdf-nData ; - - - double eInt ; - if (_etype != RooAbsData::Expected) { - double eIntLo; - double eIntHi; - hdata->weightError(eIntLo, eIntHi, _etype); - eInt = (eExt > 0) ? eIntHi : eIntLo; - } else { - eInt = sqrt(nPdf) ; - } - - // Skip cases where pdf=0 and there is no data - if (0. == eInt * eInt && 0. == nData * nData && 0. == nPdf * nPdf) continue ; - - // Return 0 if eInt=0, special handling in MINUIT will follow - if (0. == eInt * eInt) { - coutE(Eval) << "RooChi2Var::RooChi2Var(" << GetName() << ") INFINITY ERROR: bin " << i - << " has zero error" << std::endl; - return 0.; - } - -// std::cout << "Chi2Var[" << i << "] nData = " << nData << " nPdf = " << nPdf << " errorExt = " << eExt << " errorInt = " << eInt << " contrib = " << eExt*eExt/(eInt*eInt) << std::endl ; - - double term = eExt*eExt/(eInt*eInt) ; - double y = term - carry; - double t = result + y; - carry = (t - result) - y; - result = t; - } - - _evalCarry = carry; - return result ; -} - -/// \endcond diff --git a/roofit/roofitcore/src/RooChi2Var.h b/roofit/roofitcore/src/RooChi2Var.h deleted file mode 100644 index f89848c987c18..0000000000000 --- a/roofit/roofitcore/src/RooChi2Var.h +++ /dev/null @@ -1,65 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/* - * Project: RooFit - * - * Copyright (c) 2024, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ROO_CHI2_VAR -#define ROO_CHI2_VAR - -#include "RooAbsOptTestStatistic.h" -#include "RooCmdArg.h" -#include "RooDataHist.h" -#include "RooAbsPdf.h" - -class RooChi2Var : public RooAbsOptTestStatistic { -public: - enum FuncMode { Function, Pdf, ExtendedPdf } ; - - // Constructors, assignment etc - RooChi2Var(const char *name, const char *title, RooAbsReal& func, RooDataHist& data, - bool extended, RooDataHist::ErrorType etype, - RooAbsTestStatistic::Configuration const& cfg=RooAbsTestStatistic::Configuration{}); - - RooChi2Var(const RooChi2Var& other, const char* name=nullptr); - TObject* clone(const char* newname=nullptr) const override { return new RooChi2Var(*this,newname); } - - RooAbsTestStatistic* create(const char *name, const char *title, RooAbsReal& pdf, RooAbsData& dhist, - const RooArgSet& projDeps, RooAbsTestStatistic::Configuration const& cfg) override { - // Virtual constructor - return new RooChi2Var(name,title,(RooAbsPdf&)pdf,(RooDataHist&)dhist,projDeps,_funcMode,cfg,_etype) ; - } - - double defaultErrorLevel() const override { - // The default error level for MINUIT error analysis for a chi^2 is 1.0 - return 1.0 ; - } - -private: - - RooChi2Var(const char *name, const char *title, RooAbsReal& func, RooDataHist& data, - const RooArgSet& projDeps, FuncMode funcMode, - RooAbsTestStatistic::Configuration const& cfg, - RooDataHist::ErrorType etype) - : RooAbsOptTestStatistic(name,title,func,data,projDeps,cfg), _etype(etype), _funcMode(funcMode) {} - -protected: - - double evaluatePartition(std::size_t firstEvent, std::size_t lastEvent, std::size_t stepSize) const override ; - - static RooArgSet _emptySet ; ///< Supports named argument constructor - - RooDataHist::ErrorType _etype ; ///< Error type store in associated RooDataHist - FuncMode _funcMode ; ///< Function, P.d.f. or extended p.d.f? -}; - - -#endif - -/// \endcond diff --git a/roofit/roofitcore/src/RooFactoryWSTool.cxx b/roofit/roofitcore/src/RooFactoryWSTool.cxx index adcd24acf0889..eaaa30526befd 100644 --- a/roofit/roofitcore/src/RooFactoryWSTool.cxx +++ b/roofit/roofitcore/src/RooFactoryWSTool.cxx @@ -58,10 +58,6 @@ It interprets all expressions for RooWorkspace::factory(const char*). #include "TROOT.h" #include "RooFitImplHelpers.h" -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#include "RooChi2Var.h" -#include "RooNLLVar.h" -#endif using namespace RooFit; using std::string, std::map, std::list, std::pair, std::endl, std::vector; @@ -2054,20 +2050,6 @@ std::string RooFactoryWSTool::SpecialsIFace::create(RooFactoryWSTool& ft, const // nconv::name[var,pdf1,pdf2] ft.createArg("RooNumConvolution",instName,pargs) ; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - } else if (cl=="nll") { - - // nll::name[pdf,data] - RooNLLVar nll(instName,instName,ft.asPDF(pargv[0].c_str()),ft.asDATA(pargv[1].c_str()), /*extended=*/false) ; - if (ft.ws().import(nll,Silence())) ft.logError() ; - - } else if (cl=="chi2") { - - // chi2::name[pdf,data] - RooChi2Var nll(instName,instName,ft.asPDF(pargv[0].c_str()),ft.asDHIST(pargv[1].c_str()), /*extended=*/false, /*etype=*/RooAbsData::Auto); - if (ft.ws().import(nll,Silence())) ft.logError() ; - -#endif } else if (cl=="profile") { // profile::name[func,vars] diff --git a/roofit/roofitcore/src/RooFormulaVar.cxx b/roofit/roofitcore/src/RooFormulaVar.cxx index ca740c48e6dc9..2c67511dc0369 100644 --- a/roofit/roofitcore/src/RooFormulaVar.cxx +++ b/roofit/roofitcore/src/RooFormulaVar.cxx @@ -53,10 +53,6 @@ #include "TFormula.h" -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#include "RooNLLVar.h" -#include "RooChi2Var.h" -#endif using std::ostream, std::istream, std::list; @@ -282,50 +278,6 @@ std::list* RooFormulaVar::plotSamplingHint(RooAbsRealLValue& obs, double -//////////////////////////////////////////////////////////////////////////////// -/// Return the default error level for MINUIT error analysis -/// If the formula contains one or more RooNLLVars and -/// no RooChi2Vars, return the defaultErrorLevel() of -/// RooNLLVar. If the addition contains one ore more RooChi2Vars -/// and no RooNLLVars, return the defaultErrorLevel() of -/// RooChi2Var. If the addition contains neither or both -/// issue a warning message and return a value of 1 - -double RooFormulaVar::defaultErrorLevel() const -{ - RooAbsReal* nllArg(nullptr) ; - RooAbsReal* chi2Arg(nullptr) ; - -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - for (const auto arg : _actualVars) { - if (dynamic_cast(arg)) { - nllArg = static_cast(arg) ; - } - if (dynamic_cast(arg)) { - chi2Arg = static_cast(arg) ; - } - } -#endif - - if (nllArg && !chi2Arg) { - coutI(Minimization) << "RooFormulaVar::defaultErrorLevel(" << GetName() - << ") Formula contains a RooNLLVar, using its error level" << std::endl ; - return nllArg->defaultErrorLevel() ; - } else if (chi2Arg && !nllArg) { - coutI(Minimization) << "RooFormulaVar::defaultErrorLevel(" << GetName() - << ") Formula contains a RooChi2Var, using its error level" << std::endl ; - return chi2Arg->defaultErrorLevel() ; - } else if (!nllArg && !chi2Arg) { - coutI(Minimization) << "RooFormulaVar::defaultErrorLevel(" << GetName() << ") WARNING: " - << "Formula contains neither RooNLLVar nor RooChi2Var server, using default level of 1.0" << std::endl ; - } else { - coutI(Minimization) << "RooFormulaVar::defaultErrorLevel(" << GetName() << ") WARNING: " - << "Formula contains BOTH RooNLLVar and RooChi2Var server, using default level of 1.0" << std::endl ; - } - - return 1.0 ; -} - std::string RooFormulaVar::getUniqueFuncName() const { return evaluator().getTFormula()->GetUniqueFuncName().Data(); diff --git a/roofit/roofitcore/src/RooGlobalFunc.cxx b/roofit/roofitcore/src/RooGlobalFunc.cxx index 1ed27e1a2d82c..64d7b74c189fc 100644 --- a/roofit/roofitcore/src/RooGlobalFunc.cxx +++ b/roofit/roofitcore/src/RooGlobalFunc.cxx @@ -447,7 +447,7 @@ RooCmdArg Link(const std::map &arg) return processMap("LinkDataSliceMany", processLinkItem, arg); } -// RooChi2Var::ctor / RooNLLVar arguments +// createChi2() / createNLL() arguments RooCmdArg Extended(bool flag) { return RooCmdArg("Extended", flag, 0, 0, 0, nullptr, nullptr, nullptr, nullptr); @@ -482,14 +482,13 @@ RooCmdArg BatchMode(std::string const &batchMode) << "The BatchMode() command argument is deprecated. Please use EvalBackend() instead." << std::endl; std::string lower = batchMode; std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return std::tolower(c); }); - if (lower == "off") { - return EvalBackend::Legacy(); - } else if (lower == "cpu") { + if (lower == "cpu") { return EvalBackend::Cpu(); } else if (lower == "cuda") { return EvalBackend::Cuda(); } - throw std::runtime_error("Only supported string values for BatchMode() are \"off\", \"cpu\", or \"cuda\"."); + throw std::runtime_error("Only supported string values for BatchMode() are \"cpu\" or \"cuda\". The legacy " + "evaluation backend that corresponded to BatchMode(\"off\") was removed from RooFit."); } /// Integrate the PDF over bins. Improves accuracy for binned fits. Switch off using `0.` as argument. \see /// RooAbsPdf::fitTo(). @@ -575,8 +574,6 @@ EvalBackend::Value EvalBackend::toValue(std::string const &name) { std::string lower = name; std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return std::tolower(c); }); - if (lower == toName(Value::Legacy)) - return Value::Legacy; if (lower == toName(Value::Cpu)) return Value::Cpu; if (lower == toName(Value::Cuda)) @@ -585,13 +582,9 @@ EvalBackend::Value EvalBackend::toValue(std::string const &name) return Value::Codegen; if (lower == toName(Value::CodegenNoGrad)) return Value::CodegenNoGrad; - throw std::runtime_error("Only supported string values for EvalBackend() are \"legacy\", \"cpu\", \"cuda\", " + throw std::runtime_error("Only supported string values for EvalBackend() are \"cpu\", \"cuda\", " "\"codegen\", or \"codegen_no_grad\"."); } -EvalBackend EvalBackend::Legacy() -{ - return EvalBackend(Value::Legacy); -} EvalBackend EvalBackend::Cpu() { return EvalBackend(Value::Cpu); @@ -614,8 +607,6 @@ std::string EvalBackend::name() const } std::string EvalBackend::toName(EvalBackend::Value value) { - if (value == Value::Legacy) - return "legacy"; if (value == Value::Cpu) return "cpu"; if (value == Value::Cuda) diff --git a/roofit/roofitcore/src/RooMinimizer.cxx b/roofit/roofitcore/src/RooMinimizer.cxx index 6a62be7ef4725..24b718a659a04 100644 --- a/roofit/roofitcore/src/RooMinimizer.cxx +++ b/roofit/roofitcore/src/RooMinimizer.cxx @@ -142,9 +142,8 @@ std::unique_ptr setOperModesDirty(RooAbsReal &function) //////////////////////////////////////////////////////////////////////////////// /// Construct MINUIT interface to given function. Function can be anything, -/// but is typically a -log(likelihood) implemented by RooNLLVar or a chi^2 -/// (implemented by RooChi2Var). Other frequent use cases are a RooAddition -/// of a RooNLLVar plus a penalty or constraint term. This class propagates +/// but is typically a -log(likelihood) or a chi^2 as returned by +/// RooAbsPdf::createNLL() or RooAbsReal::createChi2(). This class propagates /// all RooFit information (floating parameters, their values and errors) /// to MINUIT before each MINUIT call and propagates all MINUIT information /// back to the RooFit object at the end of each call (updated parameter diff --git a/roofit/roofitcore/src/RooNLLVar.cxx b/roofit/roofitcore/src/RooNLLVar.cxx deleted file mode 100644 index 77247fab180fb..0000000000000 --- a/roofit/roofitcore/src/RooNLLVar.cxx +++ /dev/null @@ -1,358 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/***************************************************************************** - * Project: RooFit * - * Package: RooFitCore * - * @(#)root/roofitcore:$Id$ - * Authors: * - * WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu * - * DK, David Kirkby, UC Irvine, dkirkby@uci.edu * - * * - * Copyright (c) 2000-2005, Regents of the University of California * - * and Stanford University. All rights reserved. * - * * - * Redistribution and use in source and binary forms, * - * with or without modification, are permitted according to the terms * - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * - *****************************************************************************/ - -/** -\file RooNLLVar.cxx -\class RooNLLVar -\ingroup Roofitcore - -Implements a -log(likelihood) calculation from a dataset -and a PDF. The NLL is calculated as -\f[ - \sum_\mathrm{data} -\log( \mathrm{pdf}(x_\mathrm{data})) -\f] -In extended mode, a -\f$ N_\mathrm{expect} - N_\mathrm{observed}*log(N_\mathrm{expect}) \f$ term is added. -**/ - -#include "RooNLLVar.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "RooRealMPFE.h" -#include -#include - -#include "TMath.h" -#include "Math/Util.h" - -#include - -RooNLLVar::~RooNLLVar() {} - - -//////////////////////////////////////////////////////////////////////////////// -/// Construct likelihood from given p.d.f and (binned or unbinned dataset) -/// For internal use. - -RooNLLVar::RooNLLVar(const char *name, const char *title, RooAbsPdf& pdf, RooAbsData& indata, - bool extended, RooAbsTestStatistic::Configuration const& cfg) : - RooNLLVar{name, title, pdf, indata, RooArgSet(), extended, cfg} {} - - -//////////////////////////////////////////////////////////////////////////////// -/// Construct likelihood from given p.d.f and (binned or unbinned dataset) -/// For internal use. - -RooNLLVar::RooNLLVar(const char *name, const char *title, RooAbsPdf &pdf, RooAbsData &indata, const RooArgSet &projDeps, - bool extended, RooAbsTestStatistic::Configuration const &cfg) - : RooAbsOptTestStatistic(name, title, pdf, indata, projDeps, cfg), - _extended(extended), - _binnedPdf(cfg.binnedL ? static_cast(_funcClone) : nullptr) -{ - // If binned likelihood flag is set, pdf is a RooRealSumPdf representing a yield vector - // for a binned likelihood calculation - - // Retrieve and cache bin widths needed to convert un-normalized binnedPdf values back to yields - if (_binnedPdf) { - - // The Active label will disable pdf integral calculations - _binnedPdf->setAttribute("BinnedLikelihoodActive") ; - - RooArgSet obs; - _funcClone->getObservables(_dataClone->get(), obs); - if (obs.size()!=1) { - _binnedPdf = nullptr; - } else { - auto* var = static_cast(obs.first()); - std::unique_ptr> boundaries{_binnedPdf->binBoundaries(*var,var->getMin(),var->getMax())}; - auto biter = boundaries->begin() ; - _binw.reserve(boundaries->size()-1) ; - double lastBound = (*biter) ; - ++biter ; - while (biter!=boundaries->end()) { - _binw.push_back((*biter) - lastBound); - lastBound = (*biter) ; - ++biter ; - } - } - } -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Copy constructor - -RooNLLVar::RooNLLVar(const RooNLLVar& other, const char* name) : - RooAbsOptTestStatistic(other,name), - _extended(other._extended), - _weightSq(other._weightSq), - _offsetSaveW2(other._offsetSaveW2), - _binw(other._binw), - _binnedPdf{other._binnedPdf} -{ -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Create a test statistic using several properties of the current instance. This is used to duplicate -/// the test statistic in multi-processing scenarios. -RooAbsTestStatistic* RooNLLVar::create(const char *name, const char *title, RooAbsReal& pdf, RooAbsData& adata, - const RooArgSet& projDeps, RooAbsTestStatistic::Configuration const& cfg) { - RooAbsPdf & thePdf = dynamic_cast(pdf); - // check if pdf can be extended - bool extendedPdf = _extended && thePdf.canBeExtended(); - - auto testStat = new RooNLLVar(name, title, thePdf, adata, projDeps, extendedPdf, cfg); - return testStat; -} - - -//////////////////////////////////////////////////////////////////////////////// - -void RooNLLVar::applyWeightSquared(bool flag) -{ - if (_gofOpMode==Slave) { - if (flag != _weightSq) { - _weightSq = flag; - std::swap(_offset, _offsetSaveW2); - } - setValueDirty(); - } else if ( _gofOpMode==MPMaster) { - for (int i=0 ; i<_nCPU ; i++) - _mpfeArray[i]->applyNLLWeightSquared(flag); - } else if ( _gofOpMode==SimMaster) { - for(auto& gof : _gofArray) - static_cast(*gof).applyWeightSquared(flag); - } -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Calculate and return likelihood on subset of data. -/// \param[in] firstEvent First event to be processed. -/// \param[in] lastEvent First event not to be processed, any more. -/// \param[in] stepSize Steps between events. -/// \note For batch computations, the step size **must** be one. -/// -/// If this an extended likelihood, the extended term is added to the return likelihood -/// in the batch that encounters the event with index 0. - -double RooNLLVar::evaluatePartition(std::size_t firstEvent, std::size_t lastEvent, std::size_t stepSize) const -{ - // Throughout the calculation, we use Kahan's algorithm for summing to - // prevent loss of precision - this is a factor four more expensive than - // straight addition, but since evaluating the PDF is usually much more - // expensive than that, we tolerate the additional cost... - ROOT::Math::KahanSum result{0.0}; - double sumWeight{0.0}; - - auto * pdfClone = static_cast(_funcClone); - - - // If pdf is marked as binned - do a binned likelihood calculation here (sum of log-Poisson for each bin) - if (_binnedPdf) { - ROOT::Math::KahanSum sumWeightKahanSum{0.0}; - for (auto i=firstEvent ; iget(i) ; - - double eventWeight = _dataClone->weight(); - - - // Calculate log(Poisson(N|mu) for this bin - double N = eventWeight ; - double mu = _binnedPdf->getVal()*_binw[i] ; - //cout << "RooNLLVar::binnedL(" << GetName() << ") N=" << N << " mu = " << mu << std::endl ; - - if (mu<=0 && N>0) { - - // Catch error condition: data present where zero events are predicted - logEvalError(Form("Observed %f events in bin %lu with zero event yield",N,(unsigned long)i)) ; - - } else if (std::abs(mu)<1e-10 && std::abs(N)<1e-10) { - - // Special handling of this case since log(Poisson(0,0)=0 but can't be calculated with usual log-formula - // since log(mu)=0. No update of result is required since term=0. - - } else { - - double term = 0.0; - if(_doBinOffset) { - term -= -mu + N + N * (std::log(mu) - std::log(N)); - } else { - term -= -mu + N * std::log(mu) - TMath::LnGamma(N+1); - } - result += term; - sumWeightKahanSum += eventWeight; - - } - } - - sumWeight = sumWeightKahanSum.Sum(); - - } else { //unbinned PDF - - std::tie(result, sumWeight) = computeScalar(stepSize, firstEvent, lastEvent); - - // include the extended maximum likelihood term, if requested - if(_extended && _setNum==_extSet) { - result += pdfClone->extendedTerm(*_dataClone, _weightSq, _doBinOffset); - } - } //unbinned PDF - - - // If part of simultaneous PDF normalize probability over - // number of simultaneous PDFs: -sum(log(p/n)) = -sum(log(p)) + N*log(n) - // If we do bin-by bin offsetting, we don't do this because it cancels out - if (!_doBinOffset && _simCount>1) { - result += sumWeight * std::log(static_cast(_simCount)); - } - - - // At the end of the first full calculation, wire the caches - if (_first) { - _first = false ; - _funcClone->wireAllCaches() ; - } - - - // Check if value offset flag is set. - if (_doOffset) { - - // If no offset is stored enable this feature now - if (_offset.Sum() == 0 && _offset.Carry() == 0 && (result.Sum() != 0 || result.Carry() != 0)) { - coutI(Minimization) << "RooNLLVar::evaluatePartition(" << GetName() << ") first = "<< firstEvent << " last = " << lastEvent << " Likelihood offset now set to " << result.Sum() << std::endl ; - _offset = result ; - } - - // Subtract offset - result -= _offset; - } - - _evalCarry = result.Carry(); - return result.Sum() ; -} - -RooNLLVar::ComputeResult RooNLLVar::computeScalar(std::size_t stepSize, std::size_t firstEvent, std::size_t lastEvent) const { - auto pdfClone = static_cast(_funcClone); - return computeScalarFunc(pdfClone, _dataClone, _normSet, _weightSq, stepSize, firstEvent, lastEvent, _offsetPdf.get()); -} - -RooNLLVar::ComputeResult RooNLLVar::computeScalarFunc(const RooAbsPdf *pdfClone, RooAbsData *dataClone, - RooArgSet *normSet, bool weightSq, std::size_t stepSize, - std::size_t firstEvent, std::size_t lastEvent, RooAbsPdf const* offsetPdf) -{ - ROOT::Math::KahanSum kahanWeight; - ROOT::Math::KahanSum kahanProb; - RooNaNPacker packedNaN(0.f); - - for (auto i=firstEvent; iget(i) ; - - double weight = dataClone->weight(); //FIXME - - if (0. == weight * weight) continue ; - if (weightSq) weight = dataClone->weightSquared() ; - - double logProba = pdfClone->getLogVal(normSet); - - if(offsetPdf) { - logProba -= offsetPdf->getLogVal(normSet); - } - - const double term = -weight * logProba; - - kahanWeight.Add(weight); - kahanProb.Add(term); - packedNaN.accumulate(term); - } - - if (packedNaN.getPayload() != 0.) { - // Some events with evaluation errors. Return "badness" of errors. - return {ROOT::Math::KahanSum{packedNaN.getNaNWithPayload()}, kahanWeight.Sum()}; - } - - return {kahanProb, kahanWeight.Sum()}; -} - -bool RooNLLVar::setDataSlave(RooAbsData &indata, bool cloneData, bool ownNewData) -{ - bool ret = RooAbsOptTestStatistic::setDataSlave(indata, cloneData, ownNewData); - // To re-create the data template pdf if necessary - _offsetPdf.reset(); - enableBinOffsetting(_doBinOffset); - return ret; -} - -void RooNLLVar::enableBinOffsetting(bool flag) -{ - if (!_init) { - initialize(); - } - - _doBinOffset = flag; - - // If this is a "master" that delegates the actual work to "slaves", the - // _offsetPdf will not be reset. - bool needsResetting = true; - - switch (operMode()) { - case Slave: break; - case SimMaster: { - for (auto &gof : _gofArray) { - static_cast(*gof).enableBinOffsetting(flag); - } - needsResetting = false; - break; - } - case MPMaster: { - for (int i = 0; i < _nCPU; ++i) { - static_cast(_mpfeArray[i]->arg()).enableBinOffsetting(flag); - } - needsResetting = false; - break; - } - } - - if (!needsResetting) - return; - - if (flag && !_offsetPdf) { - std::string name = std::string{GetName()} + "_offsetPdf"; - std::unique_ptr dataTemplate; - if (auto dh = dynamic_cast(_dataClone)) { - dataTemplate = std::make_unique(*dh); - } else { - dataTemplate = std::unique_ptr(static_cast(*_dataClone).binnedClone()); - } - _offsetPdf = std::make_unique(name.c_str(), name.c_str(), *_funcObsSet, std::move(dataTemplate)); - _offsetPdf->setOperMode(ADirty); - } - setValueDirty(); -} - -/// \endcond diff --git a/roofit/roofitcore/src/RooNLLVar.h b/roofit/roofitcore/src/RooNLLVar.h deleted file mode 100644 index 774311b6925d0..0000000000000 --- a/roofit/roofitcore/src/RooNLLVar.h +++ /dev/null @@ -1,79 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/* - * Project: RooFit - * - * Copyright (c) 2024, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ROO_NLL_VAR -#define ROO_NLL_VAR - -#include "RooAbsOptTestStatistic.h" -#include "RooCmdArg.h" -#include "RooAbsPdf.h" -#include -#include - -class RooNLLVar : public RooAbsOptTestStatistic { -public: - - // Constructors, assignment etc - RooNLLVar(const char *name, const char *title, RooAbsPdf& pdf, RooAbsData& data, - bool extended, - RooAbsTestStatistic::Configuration const& cfg=RooAbsTestStatistic::Configuration{}); - - RooNLLVar(const char *name, const char *title, RooAbsPdf& pdf, RooAbsData& data, - const RooArgSet& projDeps, bool extended = false, - RooAbsTestStatistic::Configuration const& cfg=RooAbsTestStatistic::Configuration{}); - - RooNLLVar(const RooNLLVar& other, const char* name=nullptr); - TObject* clone(const char* newname=nullptr) const override { return new RooNLLVar(*this,newname); } - - RooAbsTestStatistic* create(const char *name, const char *title, RooAbsReal& pdf, RooAbsData& adata, - const RooArgSet& projDeps, RooAbsTestStatistic::Configuration const& cfg) override; - - ~RooNLLVar() override; - - void applyWeightSquared(bool flag) override; - - double defaultErrorLevel() const override { return 0.5 ; } - - void enableBinOffsetting(bool on = true); - - using ComputeResult = std::pair, double>; - - static RooNLLVar::ComputeResult computeScalarFunc(const RooAbsPdf *pdfClone, RooAbsData *dataClone, RooArgSet *normSet, - bool weightSq, std::size_t stepSize, std::size_t firstEvent, - std::size_t lastEvent, RooAbsPdf const* offsetPdf = nullptr); - - bool setDataSlave(RooAbsData& data, bool cloneData=true, bool ownNewDataAnyway=false) override; - -protected: - - bool processEmptyDataSets() const override { return _extended ; } - double evaluatePartition(std::size_t firstEvent, std::size_t lastEvent, std::size_t stepSize) const override; - - static RooArgSet _emptySet ; // Supports named argument constructor - -private: - ComputeResult computeScalar(std::size_t stepSize, std::size_t firstEvent, std::size_t lastEvent) const; - - bool _extended{false}; - bool _doBinOffset{false}; - bool _weightSq{false}; ///< Apply weights squared? - mutable bool _first{true}; /// _offsetSaveW2{0.0}; /// _binw ; /// _offsetPdf; ///getVal() // Evaluate slowFunc in current process - -RooRealMPFE mpfe("mpfe","frontend to slowFunc",*slowFunc) ; -mpfe.calculate() ; // Start calculation of slow-func in remote process - // .. do other stuff here .. -double val = mpfe.getVal() // Wait for remote calculation to finish and retrieve value -~~~ - -For general multiprocessing in ROOT, please refer to the TProcessExecutor class. - -**/ - -#include "Riostream.h" - -#ifndef _WIN32 -#include "BidirMMapPipe.h" -#endif - -#include -#include -#include -#include "RooRealMPFE.h" -#include "RooArgSet.h" -#include "RooAbsCategory.h" -#include "RooRealVar.h" -#include "RooCategory.h" -#include "RooMsgService.h" -#include "RooNLLVar.h" - -#include "Rtypes.h" -#include "TSystem.h" - - -class RooRealMPFE ; - -// RooMPSentinel is a singleton class that keeps track of all -// parallel execution processes for goodness-of-fit calculations. -// The primary task of RooMPSentinel is to terminate all server processes -// when the main ROOT process is exiting. -struct RooMPSentinel { - - static RooMPSentinel& instance(); - - ~RooMPSentinel(); - - void add(RooRealMPFE& mpfe) ; - void remove(RooRealMPFE& mpfe) ; - - RooArgSet _mpfeSet ; -}; - -RooMPSentinel& RooMPSentinel::instance() { - static RooMPSentinel inst; - return inst; -} - - -using std::string, std::ostringstream, std::list; -using namespace RooFit; - - -//////////////////////////////////////////////////////////////////////////////// -/// Construct front-end object for object 'arg' whose evaluation will be calculated -/// asynchronously in a separate process. If calcInline is true the value of 'arg' -/// is calculate synchronously in the current process. - -RooRealMPFE::RooRealMPFE(const char *name, const char *title, RooAbsReal& arg, bool calcInline) : - RooAbsReal(name,title), - _state(Initialize), - _arg("arg","arg",this,arg), - _vars("vars","vars",this), - _calcInProgress(false), - _verboseClient(false), - _verboseServer(false), - _inlineMode(calcInline), - _remoteEvalErrorLoggingState(RooAbsReal::PrintErrors), - _pipe(nullptr), - _updateMaster(nullptr), - _retrieveDispatched(false), _evalCarry(0.) -{ -#ifdef _WIN32 - _inlineMode = true; -#endif - initVars() ; - RooMPSentinel::instance().add(*this) ; - -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Copy constructor. Initializes in clean state so that upon eval -/// this instance will create its own server processes - -RooRealMPFE::RooRealMPFE(const RooRealMPFE& other, const char* name) : - RooAbsReal(other, name), - _state(Initialize), - _arg("arg",this,other._arg), - _vars("vars",this,other._vars), - _calcInProgress(false), - _verboseClient(other._verboseClient), - _verboseServer(other._verboseServer), - _inlineMode(other._inlineMode), - _forceCalc(other._forceCalc), - _remoteEvalErrorLoggingState(other._remoteEvalErrorLoggingState), - _pipe(nullptr), - _updateMaster(nullptr), - _retrieveDispatched(false), _evalCarry(other._evalCarry) -{ - initVars() ; - RooMPSentinel::instance().add(*this) ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Destructor - -RooRealMPFE::~RooRealMPFE() -{ - if (_state==Client) standby(); - RooMPSentinel::instance().remove(*this); -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Initialize list of variables of front-end argument 'arg' - -void RooRealMPFE::initVars() -{ - // Empty current lists - _vars.removeAll() ; - _saveVars.removeAll() ; - - // Retrieve non-constant parameters - std::unique_ptr vars{_arg->getParameters(RooArgSet())}; - // RooArgSet *ncVars = vars->selectByAttrib("Constant", false); - RooArgList varList(*vars) ; - - // Save in lists - _vars.add(varList) ; - _saveVars.addClone(varList) ; - _valueChanged.resize(_vars.size()) ; - _constChanged.resize(_vars.size()) ; - - // Force next calculation - _forceCalc = true ; -} - -double RooRealMPFE::getCarry() const -{ - if (_inlineMode) { - RooAbsTestStatistic* tmp = dynamic_cast(_arg.absArg()); - if (tmp) return tmp->getCarry(); - else return 0.; - } else { - return _evalCarry; - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// Initialize the remote process and message passing -/// pipes between current process and remote process - -void RooRealMPFE::initialize() -{ - // Trivial case: Inline mode - if (_inlineMode) { - _state = Inline ; - return ; - } - -#ifndef _WIN32 - // Clear eval error log prior to forking - // to avoid confusions... - clearEvalErrorLog() ; - // Fork server process and setup IPC - _pipe = new BidirMMapPipe(); - - if (_pipe->isChild()) { - // Start server loop - _state = Server ; - serverLoop(); - - // Kill server at end of service - if (_verboseServer) ccoutD(Minimization) << "RooRealMPFE::initialize(" << - GetName() << ") server process terminating" << std::endl ; - - delete _arg.absArg(); - delete _pipe; - _exit(0) ; - } else { - // Client process - fork successful - if (_verboseClient) { - ccoutD(Minimization) << "RooRealMPFE::initialize(" << GetName() << ") successfully forked server process " - << _pipe->pidOtherEnd() << std::endl; - } - _state = Client ; - _calcInProgress = false ; - } -#endif // _WIN32 -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Server loop of remote processes. This function will return -/// only when an incoming TERMINATE message is received. - -void RooRealMPFE::serverLoop() -{ -#ifndef _WIN32 - int msg ; - - Int_t idx; - Int_t index; - Int_t numErrors; - double value ; - bool isConst ; - - clearEvalErrorLog() ; - - while(*_pipe && !_pipe->eof()) { - *_pipe >> msg; - if (Terminate == msg) { - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> Terminate" << std::endl; - // send terminate acknowledged to client - *_pipe << msg << BidirMMapPipe::flush; - break; - } - - switch (msg) { - case SendReal: - { - *_pipe >> idx >> value >> isConst; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> SendReal [" << idx << "]=" << value << std::endl ; - RooRealVar* rvar = static_cast(_vars.at(idx)) ; - rvar->setVal(value) ; - if (rvar->isConstant() != isConst) { - rvar->setConstant(isConst) ; - } - } - break ; - - case SendCat: - { - *_pipe >> idx >> index; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> SendCat [" << idx << "]=" << index << std::endl ; - (static_cast(_vars.at(idx)))->setIndex(index) ; - } - break ; - - case Calculate: - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> Calculate" << std::endl ; - _value = _arg ; - break ; - - case CalculateNoOffset: - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> Calculate" << std::endl ; - - RooAbsReal::setHideOffset(false) ; - _value = _arg ; - RooAbsReal::setHideOffset(true) ; - break ; - - case Retrieve: - { - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> Retrieve" << std::endl ; - msg = ReturnValue; - numErrors = numEvalErrors(); - *_pipe << msg << _value << getCarry() << numErrors; - - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC toClient> ReturnValue " << _value << " NumError " << numErrors << std::endl ; - - if (numErrors) { - // Loop over errors - std::string objidstr; - { - ostringstream oss2; - // Format string with object identity as this cannot be evaluated on the other side - oss2 << "PID" << gSystem->GetPid() << "/"; - printStream(oss2,kName|kClassName|kArgs,kInline); - objidstr = oss2.str(); - } - std::map > >::const_iterator iter = evalErrorIter(); - const RooAbsArg* ptr = nullptr; - for (int i = 0; i < numEvalErrorItems(); ++i) { - list::const_iterator iter2 = iter->second.second.begin(); - for (; iter->second.second.end() != iter2; ++iter2) { - ptr = iter->first; - *_pipe << ptr << iter2->_msg << iter2->_srvval << objidstr; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC toClient> sending error log Arg " << iter->first << " Msg " << iter2->_msg << std::endl ; - } - } - // let other end know that we're done with the list of errors - ptr = nullptr; - *_pipe << ptr; - // Clear error list on local side - clearEvalErrorLog(); - } - *_pipe << BidirMMapPipe::flush; - } - break; - - case Verbose: - { - bool flag ; - *_pipe >> flag; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> Verbose " << (flag?1:0) << std::endl ; - _verboseServer = flag ; - } - break ; - - - case ApplyNLLW2: - { - bool flag ; - *_pipe >> flag; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> ApplyNLLW2 " << (flag?1:0) << std::endl ; - - // Do application of weight-squared here - doApplyNLLW2(flag) ; - } - break ; - - case EnableOffset: - { - bool flag ; - *_pipe >> flag; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> EnableOffset " << (flag?1:0) << std::endl ; - - // Enable likelihoof offsetting here - ((RooAbsReal&)_arg.arg()).enableOffsetting(flag) ; - } - break ; - - case LogEvalError: - { - int iflag2; - *_pipe >> iflag2; - RooAbsReal::ErrorLoggingMode flag2 = static_cast(iflag2); - RooAbsReal::setEvalErrorLoggingMode(flag2) ; - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> LogEvalError flag = " << flag2 << std::endl ; - } - break ; - - - default: - if (_verboseServer) std::cout << "RooRealMPFE::serverLoop(" << GetName() - << ") IPC fromClient> Unknown message (code = " << msg << ")" << std::endl ; - break ; - } - } - -#endif // _WIN32 -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Client-side function that instructs server process to start -/// asynchronous (re)calculation of function value. This function -/// returns immediately. The calculated value can be retrieved -/// using getVal() - -void RooRealMPFE::calculate() const -{ - - // Start asynchronous calculation of arg value - if (_state==Initialize) { - const_cast(this)->initialize() ; - } - - // Inline mode -- Calculate value now - if (_state==Inline) { - _value = _arg ; - clearValueDirty() ; - } - -#ifndef _WIN32 - // Compare current value of variables with saved values and send changes to server - if (_state==Client) { - Int_t i(0) ; - - //for (i=0 ; i<_vars.size() ; i++) { - RooAbsArg *var; - RooAbsArg *saveVar; - for (std::size_t j=0 ; j<_vars.size() ; j++) { - var = _vars.at(j); - saveVar = _saveVars.at(j); - - //bool valChanged = !(*var==*saveVar) ; - bool valChanged; - bool constChanged; - if (!_updateMaster) { - valChanged = !var->isIdentical(*saveVar,true) ; - constChanged = (var->isConstant() != saveVar->isConstant()) ; - _valueChanged[i] = valChanged ; - _constChanged[i] = constChanged ; - } else { - valChanged = _updateMaster->_valueChanged[i] ; - constChanged = _updateMaster->_constChanged[i] ; - } - - if ( valChanged || constChanged || _forceCalc) { - if (_verboseClient) std::cout << "RooRealMPFE::calculate(" << GetName() - << ") variable " << _vars.at(i)->GetName() << " changed" << std::endl ; - if (constChanged) { - (static_cast(saveVar))->setConstant(var->isConstant()) ; - } - saveVar->copyCache(var) ; - - // send message to server - if (dynamic_cast(var)) { - int msg = SendReal ; - double val = (static_cast(var))->getVal() ; - bool isC = var->isConstant() ; - *_pipe << msg << i << val << isC; - - if (_verboseServer) std::cout << "RooRealMPFE::calculate(" << GetName() - << ") IPC toServer> SendReal [" << i << "]=" << val << (isC?" (Constant)":"") << std::endl ; - } else if (dynamic_cast(var)) { - int msg = SendCat ; - UInt_t idx = (static_cast(var))->getCurrentIndex() ; - *_pipe << msg << i << idx; - if (_verboseServer) std::cout << "RooRealMPFE::calculate(" << GetName() - << ") IPC toServer> SendCat [" << i << "]=" << idx << std::endl ; - } - } - i++ ; - } - - int msg = hideOffset() ? Calculate : CalculateNoOffset; - *_pipe << msg; - if (_verboseServer) std::cout << "RooRealMPFE::calculate(" << GetName() - << ") IPC toServer> Calculate " << std::endl ; - - // Clear dirty state and mark that calculation request was dispatched - clearValueDirty() ; - _calcInProgress = true ; - _forceCalc = false ; - - msg = Retrieve ; - *_pipe << msg << BidirMMapPipe::flush; - if (_verboseServer) std::cout << "RooRealMPFE::evaluate(" << GetName() - << ") IPC toServer> Retrieve " << std::endl ; - _retrieveDispatched = true ; - - } else if (_state!=Inline) { - std::cout << "RooRealMPFE::calculate(" << GetName() - << ") ERROR not in Client or Inline mode" << std::endl ; - } - - -#endif // _WIN32 -} - - - - -//////////////////////////////////////////////////////////////////////////////// -/// If value needs recalculation and calculation has not been started -/// with a call to calculate() start it now. This function blocks -/// until remote process has finished calculation and returns -/// remote value - -double RooRealMPFE::getValV(const RooArgSet* /*nset*/) const -{ - - if (isValueDirty()) { - // Cache is dirty, no calculation has been started yet - calculate() ; - _value = evaluate() ; - } else if (_calcInProgress) { - // Cache is clean and calculation is in progress - _value = evaluate() ; - } else { - // Cache is clean and calculated value is in cache - } - - return _value ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Send message to server process to retrieve output value -/// If error were logged use logEvalError() on remote side -/// transfer those errors to the local eval error queue. - -double RooRealMPFE::evaluate() const -{ - // Retrieve value of arg - double return_value = 0; - if (_state==Inline) { - return_value = _arg ; - } else if (_state==Client) { -#ifndef _WIN32 - bool needflush = false; - int msg; - double value; - - // If current error logging state is not the same as remote state - // update the remote state - if (evalErrorLoggingMode() != _remoteEvalErrorLoggingState) { - msg = LogEvalError ; - RooAbsReal::ErrorLoggingMode flag = evalErrorLoggingMode() ; - *_pipe << msg << flag; - needflush = true; - _remoteEvalErrorLoggingState = evalErrorLoggingMode() ; - } - - if (!_retrieveDispatched) { - msg = Retrieve ; - *_pipe << msg; - needflush = true; - if (_verboseServer) std::cout << "RooRealMPFE::evaluate(" << GetName() - << ") IPC toServer> Retrieve " << std::endl ; - } - if (needflush) *_pipe << BidirMMapPipe::flush; - _retrieveDispatched = false ; - - - Int_t numError; - - *_pipe >> msg >> value >> _evalCarry >> numError; - - if (msg!=ReturnValue) { - std::cout << "RooRealMPFE::evaluate(" << GetName() - << ") ERROR: unexpected message from server process: " << msg << std::endl ; - return 0 ; - } - if (_verboseServer) std::cout << "RooRealMPFE::evaluate(" << GetName() - << ") IPC fromServer> ReturnValue " << value << std::endl ; - - if (_verboseServer) std::cout << "RooRealMPFE::evaluate(" << GetName() - << ") IPC fromServer> NumErrors " << numError << std::endl ; - if (numError) { - // Retrieve remote errors and feed into local error queue - char *msgbuf1 = nullptr; - char *msgbuf2 = nullptr; - char *msgbuf3 = nullptr; - RooAbsArg *ptr = nullptr; - while (true) { - *_pipe >> ptr; - if (!ptr) break; - *_pipe >> msgbuf1 >> msgbuf2 >> msgbuf3; - if (_verboseServer) std::cout << "RooRealMPFE::evaluate(" << GetName() - << ") IPC fromServer> retrieving error log Arg " << ptr << " Msg " << msgbuf1 << std::endl ; - - logEvalError(reinterpret_cast(ptr),msgbuf3,msgbuf1,msgbuf2) ; - } - std::free(msgbuf1); - std::free(msgbuf2); - std::free(msgbuf3); - } - - // Mark end of calculation in progress - _calcInProgress = false ; - return_value = value ; -#endif // _WIN32 - } - - return return_value; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Terminate remote server process and return front-end class -/// to standby mode. Calls to calculate() or evaluate() after -/// this call will automatically recreated the server process. - -void RooRealMPFE::standby() -{ -#ifndef _WIN32 - if (_state==Client) { - if (_pipe->good()) { - // Terminate server process ; - if (_verboseServer) std::cout << "RooRealMPFE::standby(" << GetName() - << ") IPC toServer> Terminate " << std::endl; - int msg = Terminate; - *_pipe << msg << BidirMMapPipe::flush; - // read handshake - msg = 0; - *_pipe >> msg; - if (Terminate != msg || 0 != _pipe->close()) { - std::cerr << "In " << __func__ << "(" << __FILE__ ", " << __LINE__ << - "): Server shutdown failed." << std::endl; - } - } else { - if (_verboseServer) { - std::cerr << "In " << __func__ << "(" << __FILE__ ", " << - __LINE__ << "): Pipe has already shut down, not sending " - "Terminate to server." << std::endl; - } - } - // Close pipes - delete _pipe; - _pipe = nullptr; - - // Revert to initialize state - _state = Initialize; - } -#endif // _WIN32 -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Control verbose messaging related to inter process communication -/// on both client and server side - -void RooRealMPFE::setVerbose(bool clientFlag, bool serverFlag) -{ -#ifndef _WIN32 - if (_state==Client) { - int msg = Verbose ; - *_pipe << msg << serverFlag; - if (_verboseServer) std::cout << "RooRealMPFE::setVerbose(" << GetName() - << ") IPC toServer> Verbose " << (serverFlag?1:0) << std::endl ; - } -#endif // _WIN32 - _verboseClient = clientFlag ; _verboseServer = serverFlag ; -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Control verbose messaging related to inter process communication -/// on both client and server side - -void RooRealMPFE::applyNLLWeightSquared(bool flag) -{ -#ifndef _WIN32 - if (_state==Client) { - int msg = ApplyNLLW2 ; - *_pipe << msg << flag; - if (_verboseServer) std::cout << "RooRealMPFE::applyNLLWeightSquared(" << GetName() - << ") IPC toServer> ApplyNLLW2 " << (flag?1:0) << std::endl ; - } -#endif // _WIN32 - doApplyNLLW2(flag) ; -} - - -//////////////////////////////////////////////////////////////////////////////// - -void RooRealMPFE::doApplyNLLW2(bool flag) -{ - RooNLLVar* nll = dynamic_cast(_arg.absArg()) ; - if (nll) { - nll->applyWeightSquared(flag) ; - } -} - - -//////////////////////////////////////////////////////////////////////////////// -/// Control verbose messaging related to inter process communication -/// on both client and server side - -void RooRealMPFE::enableOffsetting(bool flag) -{ -#ifndef _WIN32 - if (_state==Client) { - int msg = EnableOffset ; - *_pipe << msg << flag; - if (_verboseServer) std::cout << "RooRealMPFE::enableOffsetting(" << GetName() - << ") IPC toServer> EnableOffset " << (flag?1:0) << std::endl ; - } -#endif // _WIN32 - ((RooAbsReal&)_arg.arg()).enableOffsetting(flag) ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Destructor. Terminate all parallel processes still registered with -/// the sentinel - -RooMPSentinel::~RooMPSentinel() -{ - for(auto * mpfe : static_range_cast(_mpfeSet)) { - mpfe->standby() ; - } -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Register given multi-processor front-end object with the sentinel - -void RooMPSentinel::add(RooRealMPFE& mpfe) -{ - _mpfeSet.add(mpfe,true) ; -} - - - -//////////////////////////////////////////////////////////////////////////////// -/// Remove given multi-processor front-end object from the sentinel - -void RooMPSentinel::remove(RooRealMPFE& mpfe) -{ - _mpfeSet.remove(mpfe,true) ; -} - -/// \endcond diff --git a/roofit/roofitcore/src/RooRealMPFE.h b/roofit/roofitcore/src/RooRealMPFE.h deleted file mode 100644 index 2727ddfecc8eb..0000000000000 --- a/roofit/roofitcore/src/RooRealMPFE.h +++ /dev/null @@ -1,93 +0,0 @@ -/// \cond ROOFIT_INTERNAL - -/***************************************************************************** - * Project: RooFit * - * Package: RooFitCore * - * File: $Id: RooRealMPFE.h,v 1.7 2007/05/11 09:11:30 verkerke Exp $ - * Authors: * - * WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu * - * DK, David Kirkby, UC Irvine, dkirkby@uci.edu * - * * - * Copyright (c) 2000-2005, Regents of the University of California * - * and Stanford University. All rights reserved. * - * * - * Redistribution and use in source and binary forms, * - * with or without modification, are permitted according to the terms * - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * - *****************************************************************************/ -#ifndef ROO_REAL_MPFE -#define ROO_REAL_MPFE - -#include "RooAbsReal.h" -#include "RooRealProxy.h" -#include "RooListProxy.h" -#include "RooArgList.h" - -#include - -class RooArgSet ; -namespace RooFit { class BidirMMapPipe; } - -class RooRealMPFE : public RooAbsReal { -public: - // Constructors, assignment etc - RooRealMPFE(const char *name, const char *title, RooAbsReal& arg, bool calcInline=false) ; - RooRealMPFE(const RooRealMPFE& other, const char* name=nullptr); - TObject* clone(const char* newname=nullptr) const override { return new RooRealMPFE(*this,newname); } - ~RooRealMPFE() override; - - void calculate() const ; - double getValV(const RooArgSet* nset=nullptr) const override ; - void standby() ; - - void setVerbose(bool clientFlag=true, bool serverFlag=true) ; - - void applyNLLWeightSquared(bool flag) ; - - void enableOffsetting(bool flag) override ; - - void followAsSlave(RooRealMPFE& master) { _updateMaster = &master ; } - - RooAbsReal & arg() const { return *_arg; } - - protected: - - // Function evaluation - double evaluate() const override ; - friend class RooAbsTestStatistic ; - virtual double getCarry() const; - - enum State { Initialize,Client,Server,Inline } ; - State _state ; - - enum Message { SendReal=0, SendCat, Calculate, Retrieve, ReturnValue, Terminate, - Verbose, LogEvalError, ApplyNLLW2, EnableOffset, CalculateNoOffset } ; - - void initialize() ; - void initVars() ; - void serverLoop() ; - - void doApplyNLLW2(bool flag) ; - - RooRealProxy _arg ; ///< Function to calculate in parallel process - RooListProxy _vars ; ///< Variables - RooArgList _saveVars ; ///< Copy of variables - mutable bool _calcInProgress ; - bool _verboseClient ; - bool _verboseServer ; - bool _inlineMode ; - mutable bool _forceCalc ; - mutable RooAbsReal::ErrorLoggingMode _remoteEvalErrorLoggingState ; - - RooFit::BidirMMapPipe *_pipe; /// _valueChanged ; /// _constChanged ; ///setExpensiveObjectCache(_eocache); node->setWorkspace(*this); -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - if (dynamic_cast(node)) { - RooAbsOptTestStatistic *tmp = static_cast(node); - if (tmp->isSealed() && tmp->sealNotice() && strlen(tmp->sealNotice()) > 0) { - std::cout << "RooWorkspace::Streamer(" << GetName() << ") " << node->ClassName() << "::" << node->GetName() - << " : " << tmp->sealNotice() << std::endl; - } - } -#endif } for(TObject * gobj : allGenericObjects()) { diff --git a/roofit/roofitcore/src/TestStatistics/RooUnbinnedL.cxx b/roofit/roofitcore/src/TestStatistics/RooUnbinnedL.cxx index 7e8fd790e0522..0492ce63aa9f4 100644 --- a/roofit/roofitcore/src/TestStatistics/RooUnbinnedL.cxx +++ b/roofit/roofitcore/src/TestStatistics/RooUnbinnedL.cxx @@ -40,11 +40,8 @@ namespace TestStatistics { namespace { -RooAbsL::ClonePdfData clonePdfData(RooAbsPdf &pdf, RooAbsData &data, RooFit::EvalBackend evalBackend) +RooAbsL::ClonePdfData clonePdfData(RooAbsPdf &pdf, RooAbsData &data) { - if (evalBackend.value() == RooFit::EvalBackend::Value::Legacy) { - return {&pdf, &data}; - } // For the evaluation with the BatchMode, the pdf needs to be "compiled" for // a given normalization set. return {RooFit::Detail::compileForNormSet(pdf, *data.get()), &data}; @@ -54,27 +51,27 @@ RooAbsL::ClonePdfData clonePdfData(RooAbsPdf &pdf, RooAbsData &data, RooFit::Eva RooUnbinnedL::RooUnbinnedL(RooAbsPdf *pdf, RooAbsData *data, RooAbsL::Extended extended, RooFit::EvalBackend evalBackend) - : RooAbsL(clonePdfData(*pdf, *data, evalBackend), data->numEntries(), 1, extended) + : RooAbsL(clonePdfData(*pdf, *data), data->numEntries(), 1, extended) { std::unique_ptr params(pdf->getParameters(data)); paramTracker_ = std::make_unique("chtracker", "change tracker", *params, true); - if (evalBackend.value() != RooFit::EvalBackend::Value::Legacy) { - evaluator_ = std::make_unique(*pdf_, evalBackend.value() == RooFit::EvalBackend::Value::Cuda); - std::stack>{}.swap(_vectorBuffers); - auto dataSpans = - RooFit::BatchModeDataHelpers::getDataSpans(*data, "", nullptr, /*skipZeroWeights=*/true, - /*takeGlobalObservablesFromData=*/false, _vectorBuffers); - for (auto const &item : dataSpans) { - evaluator_->setInput(item.first->GetName(), item.second, false); - } + evaluator_ = std::make_unique(*pdf_, evalBackend.value() == RooFit::EvalBackend::Value::Cuda); + std::stack>{}.swap(_vectorBuffers); + // Zero-weight events must not be skipped here: the probabilities from the + // evaluator are indexed by the original event indices, aligned with the + // weights obtained from RooAbsData::getWeightBatch(). Events with zero + // weight are skipped in the summation instead. + auto dataSpans = RooFit::BatchModeDataHelpers::getDataSpans(*data, "", nullptr, /*skipZeroWeights=*/false, + /*takeGlobalObservablesFromData=*/false, _vectorBuffers); + for (auto const &item : dataSpans) { + evaluator_->setInput(item.first->GetName(), item.second, false); } } RooUnbinnedL::RooUnbinnedL(const RooUnbinnedL &other) : RooAbsL(other), apply_weight_squared(other.apply_weight_squared), - _first(other._first), lastSection_(other.lastSection_), cachedResult_(other.cachedResult_), evaluator_(other.evaluator_) @@ -101,47 +98,7 @@ namespace { using ComputeResult = std::pair, double>; -// Copy of RooNLLVar::computeScalarFunc. -ComputeResult computeScalarFunc(const RooAbsPdf *pdfClone, RooAbsData *dataClone, RooArgSet *normSet, bool weightSq, - std::size_t stepSize, std::size_t firstEvent, std::size_t lastEvent, - RooAbsPdf const *offsetPdf = nullptr) -{ - ROOT::Math::KahanSum kahanWeight; - ROOT::Math::KahanSum kahanProb; - RooNaNPacker packedNaN(0.f); - - for (auto i = firstEvent; i < lastEvent; i += stepSize) { - dataClone->get(i); - - double weight = dataClone->weight(); // FIXME - - if (0. == weight * weight) - continue; - if (weightSq) - weight = dataClone->weightSquared(); - - double logProba = pdfClone->getLogVal(normSet); - - if (offsetPdf) { - logProba -= offsetPdf->getLogVal(normSet); - } - - const double term = -weight * logProba; - - kahanWeight.Add(weight); - kahanProb.Add(term); - packedNaN.accumulate(term); - } - - if (packedNaN.getPayload() != 0.) { - // Some events with evaluation errors. Return "badness" of errors. - return {ROOT::Math::KahanSum{packedNaN.getNaNWithPayload()}, kahanWeight.Sum()}; - } - - return {kahanProb, kahanWeight.Sum()}; -} - -// Similar to computeScalarFunc, but the probabilities were already evaluated +// The probabilities were already evaluated // as a batch, and the weights are also retrieved as batches instead of looping // over RooAbsData::get(i), which loads every column of the dataset only to // then read a single weight. @@ -206,16 +163,11 @@ RooUnbinnedL::evaluatePartition(Section events, std::size_t /*components_begin*/ (cachedResult_.Sum() != 0 || cachedResult_.Carry() != 0)) return cachedResult_; - if (evaluator_) { - // Here, we have a memory allocation that should be avoided when this - // code needs to be optimized. - std::span probas = evaluator_->run(); - std::tie(result, sumWeight) = - computeBatchFunc(probas, data_.get(), apply_weight_squared, 1, events.begin(N_events_), events.end(N_events_)); - } else { - std::tie(result, sumWeight) = computeScalarFunc(pdf_.get(), data_.get(), normSet_.get(), apply_weight_squared, 1, - events.begin(N_events_), events.end(N_events_)); - } + // Here, we have a memory allocation that should be avoided when this + // code needs to be optimized. + std::span probas = evaluator_->run(); + std::tie(result, sumWeight) = + computeBatchFunc(probas, data_.get(), apply_weight_squared, 1, events.begin(N_events_), events.end(N_events_)); // include the extended maximum likelihood term, if requested if (extended_ && events.begin_fraction == 0) { @@ -228,13 +180,6 @@ RooUnbinnedL::evaluatePartition(Section events, std::size_t /*components_begin*/ result += sumWeight * log(1.0 * sim_count_); } - // At the end of the first full calculation, wire the caches. This doesn't - // need to be done in BatchMode with the RooFit driver. - if (_first && !evaluator_) { - _first = false; - pdf_->wireAllCaches(); - } - if ((RooAbsReal::evalErrorLoggingMode() == RooAbsReal::CollectErrors || RooAbsReal::evalErrorLoggingMode() == RooAbsReal::CountErrors) && numEvalErrorsBefore == RooAbsReal::numEvalErrors()) { diff --git a/roofit/roofitcore/test/CMakeLists.txt b/roofit/roofitcore/test/CMakeLists.txt index ac9d5bb73b7f0..a7839d5d40771 100644 --- a/roofit/roofitcore/test/CMakeLists.txt +++ b/roofit/roofitcore/test/CMakeLists.txt @@ -110,9 +110,6 @@ if(mathmore) endif() configure_file(stressRooFit_ref.root stressRooFit_ref.root COPYONLY) -if(roofit_legacy_eval_backend) - ROOT_ADD_TEST(test-stressroofit-legacy COMMAND stressRooFit -b legacy FAILREGEX "FAILED|Error in") -endif() ROOT_ADD_TEST(test-stressroofit-cpu COMMAND stressRooFit -b cpu FAILREGEX "FAILED|Error in") ROOT_ADD_TEST(test-stressroofit-codegen_no_grad COMMAND stressRooFit -b codegen_no_grad FAILREGEX "FAILED|Error in") if(clad) diff --git a/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx b/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx index da077eb6de785..67dee7dd01df6 100644 --- a/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx +++ b/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx @@ -63,19 +63,6 @@ ValAndError getValAndError(RooArgSet const &parsFinal, const char *name) return {var.getVal(), var.getError()}; }; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -std::vector getParamVals(RooAbsMinimizerFcn &fcn) -{ - std::vector values(fcn.getNDim()); - - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = fcn.floatableParam(i).getVal(); - } - - return values; -} -#endif - std::unique_ptr runMinimizer(RooAbsReal &nll, bool offsetting) { RooMinimizer m0{nll}; @@ -237,89 +224,6 @@ TEST(LikelihoodGradientJob, RepeatMigrad) m1.minimize("Minuit2", "migrad"); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND_ -TEST_P(LikelihoodGradientJobTest, GaussianND) -{ - // do a minimization, but now using GradMinimizer and its MP version - - unsigned int N = 4; - - RooWorkspace w = RooWorkspace(); - - std::unique_ptr nll; - std::unique_ptr values; - RooAbsPdf *pdf; - std::unique_ptr data; - std::tie(nll, pdf, data, values) = generate_ND_gaussian_pdf_nll(w, N, 1000, RooFit::EvalBackend::Legacy()); - - RooArgSet savedValues; - values->snapshot(savedValues); - - // -------- - - std::unique_ptr m0result{runMinimizer(*nll, offsetting)}; - double minNll0 = m0result->minNll(); - double edm0 = m0result->edm(); - std::vector mean0(N); - std::vector std0(N); - for (unsigned ix = 0; ix < N; ++ix) { - { - std::ostringstream os; - os << "m" << ix; - mean0[ix] = dynamic_cast(w.arg(os.str().c_str()))->getVal(); - } - { - std::ostringstream os; - os << "s" << ix; - std0[ix] = dynamic_cast(w.arg(os.str().c_str()))->getVal(); - } - } - - // -------- - - values->assign(savedValues); - - // -------- - - RFTS::RooRealL likelihood("likelihood", "likelihood", std::make_unique(pdf, data.get())); - RooMinimizer::Config cfg1; - cfg1.parallelize = NWorkers; - RooMinimizer m1(likelihood, cfg1); - - m1.setStrategy(0); - m1.setPrintLevel(-1); - m1.setOffsetting(offsetting); - - m1.minimize("Minuit2", "migrad"); - - std::unique_ptr m1result{m1.save()}; - double minNll1 = m1result->minNll(); - double edm1 = m1result->edm(); - std::vector mean1(N); - std::vector std1(N); - for (unsigned ix = 0; ix < N; ++ix) { - { - std::ostringstream os; - os << "m" << ix; - mean1[ix] = static_cast(w.arg(os.str().c_str()))->getVal(); - } - { - std::ostringstream os; - os << "s" << ix; - std1[ix] = static_cast(w.arg(os.str().c_str()))->getVal(); - } - } - - EXPECT_EQ(minNll0, minNll1); - EXPECT_EQ(edm0, edm1); - - for (unsigned ix = 0; ix < N; ++ix) { - EXPECT_EQ(mean0[ix], mean1[ix]); - EXPECT_EQ(std0[ix], std1[ix]); - } -} -#endif - INSTANTIATE_TEST_SUITE_P(NworkersSeed, LikelihoodGradientJobTest, ::testing::Combine(::testing::Values(1, 2, 3), // number of workers ::testing::Values(2, 3), // random seed @@ -582,393 +486,3 @@ TEST_P(LikelihoodGradientJobTest, Gaussian1DAlsoWithLikelihoodJob) } #undef EXPECT_NEAR_REL -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -class LikelihoodGradientJobErrorTest - : public ::testing::TestWithParam> { - void SetUp() override - { - NWorkers = std::get<0>(GetParam()); - seed = std::get<1>(GetParam()); - parallelLikelihood = std::get<2>(GetParam()); - binned = std::get<3>(GetParam()); - - RooRandom::randomGenerator()->SetSeed(seed); - - // we want to split only over components so we can test component-offsets precisely - // (event-offsets give more variation) - RFMP::Config::LikelihoodJob::defaultNEventTasks = 1; // just one events task (i.e. don't split over events) - RFMP::Config::LikelihoodJob::defaultNComponentTasks = - 1000000; // assuming components < 1000000: each component = 1 separate task - } - - void TearDown() override - { - // reset static variables to automatic - RFMP::Config::LikelihoodJob::defaultNEventTasks = RFMP::Config::LikelihoodJob::automaticNEventTasks; - RFMP::Config::LikelihoodJob::defaultNComponentTasks = RFMP::Config::LikelihoodJob::automaticNComponentTasks; - } - -protected: - std::size_t NWorkers = 0; - std::size_t seed = 0; - bool parallelLikelihood = false; - bool binned = false; -}; - -TEST_P(LikelihoodGradientJobErrorTest, ErrorHandling) -{ - // In this test, we setup a model that we know will give evaluation errors, because Minuit will try parameters - // outside of the physical range during line search. Using the error handling mechanism in RooMinimizerFcn and - // MinuitFcnGrad, Minuit should get sent out of this area again. - // Specifically, this test triggers the classic error handling mechanism (logEvalError). - - RooWorkspace w("w"); - w.factory("ArgusBG::model(m[5.2,5.3],m0[5.28,5.2,5.3],c[-2,-10,0])"); - - RooAbsPdf *pdf = w.pdf("model"); - std::unique_ptr data; - if (binned) { - data = std::unique_ptr{pdf->generateBinned(*w.var("m"), 10000)}; - } else { - data = std::unique_ptr{pdf->generate(*w.var("m"), 10000)}; - } - std::unique_ptr nll{pdf->createNLL(*data, RooFit::EvalBackend::Legacy())}; - - // if m0 were constant (i.e. setConstant(true)), the fit would converge without errors, because m0 outside of the - // physical area of the Argus distribution is what causes the errors in the line search phase of the fit - w.var("m0")->setConstant(false); - - RooArgSet values{*w.var("m"), *w.var("m0"), *w.var("c"), "values"}; - RooArgSet savedValues; - values.snapshot(savedValues); - - std::unique_ptr m0result{runMinimizer(*nll, false)}; - double minNll0 = m0result->minNll(); - double edm0 = m0result->edm(); - double m_0 = w.var("m")->getVal(); - double m0_0 = w.var("m0")->getVal(); - double c_0 = w.var("c")->getVal(); - - values.assign(savedValues); - - std::unique_ptr likelihoodAbsReal{pdf->createNLL(*data, RooFit::ModularL(true))}; - - RooMinimizer::Config cfg; - cfg.parallelize = NWorkers; - cfg.enableParallelDescent = parallelLikelihood; - // cfg.printEvalErrors = 200; - RooMinimizer m1(*likelihoodAbsReal, cfg); - m1.setStrategy(0); - m1.setPrintLevel(-1); - - m1.setVerbose(false); - - m1.minimize("Minuit2", "migrad"); - - std::unique_ptr m1result{m1.save()}; - double minNll1 = m1result->minNll(); - double edm1 = m1result->edm(); - double m_1 = w.var("m")->getVal(); - double m0_1 = w.var("m0")->getVal(); - double c_1 = w.var("c")->getVal(); - - EXPECT_EQ(minNll0, minNll1); - EXPECT_EQ(edm0, edm1); - EXPECT_EQ(m_0, m_1); - EXPECT_EQ(m0_0, m0_1); - EXPECT_EQ(c_0, c_1); -} - -// TODO: https://github.com/root-project/root/pull/12328 meenemen! - -/// Fit a simple linear function, that starts in the negative. Triggers RooNaNPacker error handling. -TEST_P(LikelihoodGradientJobErrorTest, FitSimpleLinear) -{ - RooRealVar x("x", "x", -10, 10); - RooRealVar a1("a1", "a1", 12., -5., 15.); - RooGenericPdf pdf("pdf", "a1 + x", RooArgSet(x, a1)); - std::unique_ptr data; - if (binned) { - data = std::unique_ptr{pdf.generateBinned(x, 1000)}; - } else { - data = std::unique_ptr{pdf.generate(x, 1000)}; - } - std::unique_ptr nll(pdf.createNLL(*data, RooFit::EvalBackend::Legacy())); - - RooArgSet normSet{x}; - ASSERT_FALSE(std::isnan(pdf.getVal(normSet))); - a1.setVal(-5.); - ASSERT_TRUE(std::isnan(pdf.getVal(normSet))); - - RooMinimizer minim(*nll); - minim.setPrintLevel(-1); - minim.setVerbose(false); - // minim.setPrintEvalErrors(200); - minim.migrad(); - minim.hesse(); - std::unique_ptr fitResult{minim.save()}; - auto a1Result = a1.getVal(); - - // now with multiprocess - std::unique_ptr nll_mp(pdf.createNLL(*data, RooFit::ModularL(true))); - - a1.setVal(-5.); - a1.removeError(); - ASSERT_TRUE(std::isnan(pdf.getVal(normSet))); - - RooMinimizer::Config cfg; - cfg.parallelize = NWorkers; - cfg.enableParallelDescent = parallelLikelihood; - // cfg.printEvalErrors = 200; - - RooMinimizer minim_mp(*nll_mp, cfg); - minim_mp.setPrintLevel(-1); - minim_mp.setStrategy(0); - minim_mp.setVerbose(false); - minim_mp.migrad(); - minim_mp.hesse(); - std::unique_ptr fitResult_mp{minim_mp.save()}; - auto a1Result_mp = a1.getVal(); - - EXPECT_EQ(fitResult_mp->status(), 0); - EXPECT_EQ(a1Result, a1Result_mp); - EXPECT_EQ(a1Result - a1Result_mp, 0); -} - -// TODO: add error handling tests that trigger the RooNaNPacker error handling paths (see testNaNPacker for example -// setups). In particular a fit of a simultaneous or constrained likelihood to trigger the RooSumL path which has -// additional handling of the packed NaNs that isn't tested now. - -INSTANTIATE_TEST_SUITE_P(LikelihoodGradientJob, LikelihoodGradientJobErrorTest, - ::testing::Combine(::testing::Values(1, 2, 3), // number of workers - ::testing::Values(2, 3), // random seed - ::testing::Values(false, true), // with or without LikelihoodJob - ::testing::Values(false, true)), // binned or not - [](testing::TestParamInfo const ¶mInfo) { - std::stringstream ss; - ss << std::get<0>(paramInfo.param) << "workers_seed" << std::get<1>(paramInfo.param) - << (std::get<2>(paramInfo.param) ? "AlsoWithLikelihoodJob" : "NoLikelihoodJob") - << (std::get<3>(paramInfo.param) ? "_Binned" : "_Unbinned"); - return ss.str(); - }); - -class LikelihoodGradientJobBinnedErrorTest : public ::testing::TestWithParam> { - void SetUp() override - { - do_error = std::get<0>(GetParam()); - NWorkers = std::get<1>(GetParam()); - parallelLikelihood = std::get<2>(GetParam()); - - RooRandom::randomGenerator()->SetSeed(20); - - // we want to split only over components so we can test component-offsets precisely - // (event-offsets give more variation) - RFMP::Config::LikelihoodJob::defaultNEventTasks = 1; // just one events task (i.e. don't split over events) - RFMP::Config::LikelihoodJob::defaultNComponentTasks = - 1000000; // assuming components < 1000000: each component = 1 separate task - } - - void TearDown() override - { - // reset static variables to automatic - RFMP::Config::LikelihoodJob::defaultNEventTasks = RFMP::Config::LikelihoodJob::automaticNEventTasks; - RFMP::Config::LikelihoodJob::defaultNComponentTasks = RFMP::Config::LikelihoodJob::automaticNComponentTasks; - } - -protected: - bool do_error = false; - std::size_t NWorkers = 0; - bool parallelLikelihood = false; -}; - -TEST_P(LikelihoodGradientJobBinnedErrorTest, TriggerMuLEZero) -{ - auto th_data = std::make_unique("h_data", "data", 10, 0, 10); - auto th_sig = std::make_unique("h_sig", "signal", 10, 0, 10); - auto th_bkg = std::make_unique("h_bkg", "background", 10, 0, 10); - - for (int i = 0; i < 10; i++) { - th_data->SetBinContent(i + 1, i + 1); - th_sig->SetBinContent(i + 1, i); - th_bkg->SetBinContent(i + 1, 1); - } - - if (do_error) { - // Trigger error condition by setting both sig and bkg - // to zero in bin zero, thus triggering a likelihood - // error since Poisson(N|0) is undefined - th_sig->SetBinContent(1, 0); - th_bkg->SetBinContent(1, 0); - } - - RooWorkspace w("w"); - auto x = w.factory("x[0,10]"); - w.factory("index[A,B]"); - - dynamic_cast(x)->setBins(10); - - // we have to build a simultaneous binned likelihood to trigger the "binnedL" evaluation path - - RooDataHist h_sigA("h_sigA", "h_sigA", *w.var("x"), th_sig.get()); - RooDataHist h_sigB("h_sigB", "h_sigB", *w.var("x"), th_sig.get()); - RooDataHist h_bkg("h_bkg", "h_bkg", *w.var("x"), th_bkg.get()); - - w.import(h_sigA); - w.import(h_sigB); - w.import(h_bkg); - w.factory("HistPdf::sigA(x,h_sigA)"); - w.factory("HistPdf::sigB(x,h_sigB)"); - w.factory("HistPdf::bkg(x,h_bkg)"); - - w.factory("ASUM::model_A(mu_sig[1,-1,10]*sigA,mu_bkg_A[1,-1,10]*bkg)"); - w.factory("ASUM::model_B(mu_sig*sigB,mu_bkg_B[1,-1,10]*bkg)"); - - w.pdf("model_A")->setAttribute("BinnedLikelihood"); - w.pdf("model_B")->setAttribute("BinnedLikelihood"); - - // Construct simultaneous pdf - w.factory("SIMUL::model(index[A,B],A=model_A,B=model_B)"); - - // Construct dataset - std::map th_data_2D; - th_data_2D["A"] = th_data.get(); - th_data_2D["B"] = th_data.get(); - RooDataHist h_data("h_data", "h_data", *w.var("x"), *w.cat("index"), th_data_2D); - - // store initial parameters for reuse in second fit - std::unique_ptr values(w.pdf("model")->getParameters(h_data)); - RooArgSet savedValues; - values->snapshot(savedValues); - - // legacy RooFit fit - std::unique_ptr nll(w.pdf("model")->createNLL(h_data, RooFit::EvalBackend::Legacy())); - - double nll0BeforeFit = nll->getVal(); - - std::unique_ptr m0result{runMinimizer(*nll, false)}; - double minNll0 = m0result->minNll(); - double mu_sig0 = w.var("mu_sig")->getVal(); - double mu_bkg_A0 = w.var("mu_bkg_A")->getVal(); - double mu_bkg_B0 = w.var("mu_bkg_B")->getVal(); - - values->assign(savedValues); - - std::unique_ptr likelihoodAbsReal{w.pdf("model")->createNLL(h_data, RooFit::ModularL(true))}; - - RooMinimizer::Config cfg; - cfg.parallelize = NWorkers; - cfg.enableParallelDescent = parallelLikelihood; - // cfg.printEvalErrors = 200; - RooMinimizer m1(*likelihoodAbsReal, cfg); - - double nll1BeforeFit = likelihoodAbsReal->getVal(); - - m1.setStrategy(0); - m1.setPrintLevel(-1); - - m1.setVerbose(false); - - m1.minimize("Minuit2", "migrad"); - - std::unique_ptr m1result{m1.save()}; - double minNll1 = m1result->minNll(); - double mu_sig1 = w.var("mu_sig")->getVal(); - double mu_bkg_A1 = w.var("mu_bkg_A")->getVal(); - double mu_bkg_B1 = w.var("mu_bkg_B")->getVal(); - - EXPECT_EQ(minNll0, minNll1); - if (do_error) { - EXPECT_NE(nll0BeforeFit, minNll0); - } else { - // These really should be equal, but on most platforms/builds, for some reason it - // isn't exactly. The exceptions are Apple ARM builds and some builds on x64 when - // using -march=native. - EXPECT_DOUBLE_EQ(nll0BeforeFit, minNll0); - } - EXPECT_EQ(nll0BeforeFit, nll1BeforeFit); - EXPECT_EQ(mu_sig0, mu_sig1); - EXPECT_EQ(mu_bkg_A0, mu_bkg_A1); - EXPECT_EQ(mu_bkg_B0, mu_bkg_B1); -} - -INSTANTIATE_TEST_SUITE_P(LikelihoodGradientJob, LikelihoodGradientJobBinnedErrorTest, - ::testing::Combine(::testing::Values(false, true), // trigger error or don't - ::testing::Values(1, 2, 3), // number of workers - ::testing::Values(false, true) // with or without LikelihoodJob - ), - [](testing::TestParamInfo const ¶mInfo) { - std::stringstream ss; - ss << std::get<1>(paramInfo.param) << "workers" - << (std::get<2>(paramInfo.param) ? "AlsoWithLikelihoodJob" : "NoLikelihoodJob") - << (std::get<0>(paramInfo.param) ? "ErrorTriggered" : ""); - return ss.str(); - }); - -TEST(MinuitFcnGrad, DISABLED_CompareToRooMinimizerFcn) -{ - const char *fname = "/Users/pbos/projects/roofit-ssi/benchmark_roofit/data/workspaces/HZy_split.root"; - const char *dataset_name = "combData"; - - TFile *f = TFile::Open(fname); - - RooWorkspace *w = (RooWorkspace *)f->Get("combWS"); - - // Fixes for known features, binned likelihood optimization - for (RooAbsArg *arg : w->components()) { - if (arg->IsA() == RooRealSumPdf::Class()) { - arg->setAttribute("BinnedLikelihood"); - std::cout << "Activating binned likelihood attribute for " << arg->GetName() << std::endl; - } - } - - RooAbsData *data = w->data(dataset_name); - auto mc = dynamic_cast(w->genobj("ModelConfig")); - auto global_observables = mc->GetGlobalObservables(); - auto nuisance_parameters = mc->GetNuisanceParameters(); - auto pdf = w->pdf(mc->GetPdf()->GetName()); - - std::unique_ptr nll_modularL{pdf->createNLL(*data, RooFit::Constrain(*nuisance_parameters), - RooFit::GlobalObservables(*global_observables), - RooFit::ModularL(true))}; - - std::unique_ptr nll_vanilla{pdf->createNLL(*data, RooFit::Constrain(*nuisance_parameters), - RooFit::GlobalObservables(*global_observables), - RooFit::EvalBackend::Legacy() - /*, RooFit::Offset(true)*/)}; - - double vanilla_val = nll_vanilla->getVal(); - double modular_val = nll_modularL->getVal(); - - // sanity check - EXPECT_EQ(modular_val, vanilla_val); - - // set up minimizers - RooMinimizer m_vanilla(*nll_vanilla); - // we want to split only over components so we can test component-offsets - RFMP::Config::LikelihoodJob::defaultNEventTasks = 1; // just one events task (i.e. don't split over events) - RFMP::Config::LikelihoodJob::defaultNComponentTasks = - 1000000; // assuming components < 1000000: each component = 1 separate task - RooMinimizer::Config cfg; - cfg.parallelize = 1; - cfg.enableParallelDescent = false; - cfg.enableParallelGradient = true; - RooMinimizer m_modularL(*nll_modularL, cfg); - - // now use these minimizers to build the corresponding external RooAbsMinimizerFcns - auto nll_real = dynamic_cast(nll_modularL.get()); - RFTS::MinuitFcnGrad modularL_fcn(nll_real->getRooAbsL(), &m_modularL, m_modularL.fitter()->Config().ParamsSettings(), - cfg.enableParallelDescent ? RFTS::LikelihoodMode::multiprocess - : RFTS::LikelihoodMode::serial, - RFTS::LikelihoodGradientMode::multiprocess); - RooMinimizerFcn vanilla_fcn(nll_vanilla.get(), &m_vanilla); - - EXPECT_EQ(vanilla_fcn(getParamVals(vanilla_fcn).data()), modularL_fcn(getParamVals(modularL_fcn).data())); - // let's also check with absolutely certain same parameter values, both of them - EXPECT_EQ(vanilla_fcn(getParamVals(vanilla_fcn).data()), modularL_fcn(getParamVals(vanilla_fcn).data())); - EXPECT_EQ(vanilla_fcn(getParamVals(modularL_fcn).data()), modularL_fcn(getParamVals(modularL_fcn).data())); - - // reset static variables to automatic - RFMP::Config::LikelihoodJob::defaultNEventTasks = RFMP::Config::LikelihoodJob::automaticNEventTasks; - RFMP::Config::LikelihoodJob::defaultNComponentTasks = RFMP::Config::LikelihoodJob::automaticNComponentTasks; -} -#endif diff --git a/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx b/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx index 4eae3fc591b27..22d808db99ab5 100644 --- a/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx +++ b/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx @@ -176,27 +176,6 @@ TEST_F(LikelihoodJobTest, DISABLED_UnbinnedGaussian1DTwice) EXPECT_EQ(nll0, nll1.Sum()); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_F(LikelihoodJobTest, UnbinnedGaussianND) -{ - using namespace RooFit; - unsigned int N = 4; - - std::tie(nll, pdf, data, values) = generate_ND_gaussian_pdf_nll(w, N, 1000, EvalBackend::Legacy()); - likelihood = TestStatistics::buildLikelihood(pdf, data.get()); - // dummy offsets (normally they are shared with other objects): - SharedOffset offset; - auto nll_ts = RFTS::LikelihoodWrapper::create(RFTS::LikelihoodMode::multiprocess, likelihood, clean_flags, offset); - - auto nll0 = nll->getVal(); - - nll_ts->evaluate(); - auto nll1 = nll_ts->getResult(); - - EXPECT_EQ(nll0, nll1.Sum()); -} -#endif // ROOFIT_LEGACY_EVAL_BACKEND - TEST_F(LikelihoodJobBinnedDatasetTest, UnbinnedPdf) { data = std::unique_ptr{pdf->generateBinned(*w.var("x"))}; @@ -474,60 +453,6 @@ TEST_F(LikelihoodJobSimBinnedConstrainedTest, BasicParameters) EXPECT_DOUBLE_EQ(nll0, nll1.Sum()); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_F(LikelihoodJobSimBinnedConstrainedTest, ConstrainedAndOffset) -{ - using namespace RooFit; - - // A variation to test some additional parameters (ConstrainedParameters and offsetting) - - // The reference likelihood is using the legacy evaluation backend, because - // the multiprocess test statistics classes were designed to give values - // that are bit-by-bit identical with the old test statistics based on - // RooAbsTestStatistic. - nll = std::unique_ptr{pdf->createNLL(*data, Constrain(*w.var("alpha_bkg_A")), - GlobalObservables(*w.var("alpha_bkg_obs_B")), Offset("initial"), - EvalBackend::Legacy())}; - - // -------- - - auto nll0 = nll->getVal(); - - likelihood = RFTS::NLLFactory{*pdf, *data} - .ConstrainedParameters(*w.var("alpha_bkg_A")) - .GlobalObservables(*w.var("alpha_bkg_obs_B")) - .build(); - // dummy offsets (normally they are shared with other objects): - SharedOffset offset; - auto nll_ts = RFTS::LikelihoodWrapper::create(RFTS::LikelihoodMode::multiprocess, likelihood, clean_flags, offset); - nll_ts->enableOffsetting(true); - - nll_ts->evaluate(); - // The RFTS classes used for minimization (RooAbsL and Wrapper derivatives) will return offset - // values, whereas RooNLLVar::getVal will always return the non-offset value, since that is the "actual" likelihood - // value. RooRealL will also give the non-offset value, so that can be directly compared to the RooNLLVar::getVal - // result (the nll0 vs nll2 comparison below). To compare to the raw RooAbsL/Wrapper value nll1, however, we need to - // manually add the offset. - ROOT::Math::KahanSum nll1 = nll_ts->getResult(); - ROOT::Math::KahanSum nll_ts_offset; - for (auto &offset_comp : offset.offsets()) { - nll1 += offset_comp; - nll_ts_offset += offset_comp; - } - - EXPECT_EQ(nll0, nll1.Sum()); - EXPECT_FALSE(nll_ts_offset.Sum() == 0); - - // also check against RooRealL value - RFTS::RooRealL nll_real("real_nll", "RooRealL version", likelihood); - - auto nll2 = nll_real.getVal(); - - EXPECT_EQ(nll0, nll2); - EXPECT_EQ(nll1.Sum(), nll2); -} -#endif // ROOFIT_LEGACY_EVAL_BACKEND - TEST_F(LikelihoodJobTest, BatchedUnbinnedGaussianND) { unsigned int N = 4; @@ -551,23 +476,15 @@ TEST_F(LikelihoodJobTest, BatchedUnbinnedGaussianND) class LikelihoodJobSplitStrategies : public LikelihoodJobSimBinnedConstrainedTest, public testing::WithParamInterface> {}; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_P(LikelihoodJobSplitStrategies, SimBinnedConstrainedAndOffset) -#else TEST_P(LikelihoodJobSplitStrategies, DISABLED_SimBinnedConstrainedAndOffset) -#endif { using namespace RooFit; // Based on ConstrainedAndOffset, this test tests different parallelization strategies - // The reference likelihood is using the legacy evaluation backend, because - // the multiprocess test statistics classes were designed to give values - // that are bit-by-bit identical with the old test statistics based on - // RooAbsTestStatistic. nll = std::unique_ptr{pdf->createNLL(*data, Constrain(*w.var("alpha_bkg_A")), GlobalObservables(*w.var("alpha_bkg_obs_B")), Offset("initial"), - EvalBackend::Legacy())}; + EvalBackend::Cpu())}; // -------- diff --git a/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx b/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx index fe267ccbe54a3..e16d0d0650840 100644 --- a/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx +++ b/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx @@ -16,9 +16,6 @@ #include #include #include -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#include "../../src/RooNLLVar.h" -#endif #include "RooDataHist.h" // complete type in Binned test #include "RooCategory.h" // complete type in MultiBinnedConstraint test #include @@ -99,26 +96,6 @@ TEST_F(LikelihoodSerialTest, UnbinnedGaussian1D) EXPECT_EQ(nll0, nll1.Sum()); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_F(LikelihoodSerialTest, UnbinnedGaussianND) -{ - unsigned int N = 4; - - std::tie(nll, pdf, data, values) = generate_ND_gaussian_pdf_nll(w, N, 1000, RooFit::EvalBackend::Legacy()); - likelihood = RFTS::buildLikelihood(pdf, data.get()); - // dummy offsets (normally they are shared with other objects): - SharedOffset offset; - auto nll_ts = RFTS::LikelihoodWrapper::create(RFTS::LikelihoodMode::serial, likelihood, clean_flags, offset); - - auto nll0 = nll->getVal(); - - nll_ts->evaluate(); - auto nll1 = nll_ts->getResult(); - - EXPECT_EQ(nll0, nll1.Sum()); -} -#endif // ROOFIT_LEGACY_EVAL_BACKEND - TEST_F(LikelihoodSerialBinnedDatasetTest, UnbinnedPdf) { data = std::unique_ptr{pdf->generateBinned(*w.var("x"))}; @@ -138,35 +115,6 @@ TEST_F(LikelihoodSerialBinnedDatasetTest, UnbinnedPdf) EXPECT_EQ(nll0, nll1.Sum()); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_F(LikelihoodSerialBinnedDatasetTest, BinnedManualNLL) -{ - pdf->setAttribute("BinnedLikelihood"); - data = std::unique_ptr{pdf->generateBinned(*w.var("x"))}; - - // manually create NLL, ripping all relevant parts from RooAbsPdf::createNLL, except here we also set binnedL = true - RooArgSet projDeps; - RooAbsTestStatistic::Configuration nll_config; - nll_config.verbose = false; - nll_config.cloneInputData = false; - nll_config.binnedL = true; - int extended = 2; - RooNLLVar nll_manual("nlletje", "-log(likelihood)", *pdf, *data, projDeps, extended, nll_config); - - likelihood = RFTS::buildLikelihood(pdf, data.get()); - // dummy offsets (normally they are shared with other objects): - SharedOffset offset; - auto nll_ts = RFTS::LikelihoodWrapper::create(RFTS::LikelihoodMode::serial, likelihood, clean_flags, offset); - - auto nll0 = nll_manual.getVal(); - - nll_ts->evaluate(); - auto nll1 = nll_ts->getResult(); - - EXPECT_EQ(nll0, nll1.Sum()); -} -#endif - TEST_F(LikelihoodSerialTest, SimBinned) { // Unbinned pdfs that define template histograms @@ -396,60 +344,6 @@ TEST_F(LikelihoodSerialSimBinnedConstrainedTest, BasicParameters) EXPECT_DOUBLE_EQ(nll0, nll1.Sum()); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_F(LikelihoodSerialSimBinnedConstrainedTest, ConstrainedAndOffset) -{ - using namespace RooFit; - - // A variation to test some additional parameters (ConstrainedParameters and offsetting) - - // The reference likelihood is using the legacy evaluation backend, because - // the multiprocess test statistics classes were designed to give values - // that are bit-by-bit identical with the old test statistics based on - // RooAbsTestStatistic. - nll = std::unique_ptr{pdf->createNLL(*data, Constrain(*w.var("alpha_bkg_A")), - GlobalObservables(*w.var("alpha_bkg_obs_B")), Offset("initial"), - EvalBackend::Legacy())}; - - // -------- - - auto nll0 = nll->getVal(); - - likelihood = RFTS::NLLFactory{*pdf, *data} - .ConstrainedParameters(*w.var("alpha_bkg_A")) - .GlobalObservables(*w.var("alpha_bkg_obs_B")) - .build(); - // dummy offsets (normally they are shared with other objects): - SharedOffset offset; - auto nll_ts = RFTS::LikelihoodWrapper::create(RFTS::LikelihoodMode::serial, likelihood, clean_flags, offset); - nll_ts->enableOffsetting(true); - - nll_ts->evaluate(); - // The RFTS classes used for minimization (RooAbsL and Wrapper derivatives) will return offset - // values, whereas RooNLLVar::getVal will always return the non-offset value, since that is the "actual" likelihood - // value. RooRealL will also give the non-offset value, so that can be directly compared to the RooNLLVar::getVal - // result (the nll0 vs nll2 comparison below). To compare to the raw RooAbsL/Wrapper value nll1, however, we need to - // manually add the offset. - ROOT::Math::KahanSum nll1 = nll_ts->getResult(); - ROOT::Math::KahanSum nll_ts_offset; - for (auto &offset_comp : offset.offsets()) { - nll1 += offset_comp; - nll_ts_offset += offset_comp; - } - - EXPECT_EQ(nll0, nll1.Sum()); - EXPECT_FALSE(nll_ts_offset.Sum() == 0); - - // also check against RooRealL value - RFTS::RooRealL nll_real("real_nll", "RooRealL version", likelihood); - - auto nll2 = nll_real.getVal(); - - EXPECT_EQ(nll0, nll2); - EXPECT_EQ(nll1.Sum(), nll2); -} -#endif // ROOFIT_LEGACY_EVAL_BACKEND - TEST_F(LikelihoodSerialTest, BatchedUnbinnedGaussianND) { unsigned int N = 4; diff --git a/roofit/roofitcore/test/TestStatistics/testRooAbsL.cxx b/roofit/roofitcore/test/TestStatistics/testRooAbsL.cxx index 851796f51d510..6d500ff39905f 100644 --- a/roofit/roofitcore/test/TestStatistics/testRooAbsL.cxx +++ b/roofit/roofitcore/test/TestStatistics/testRooAbsL.cxx @@ -138,7 +138,9 @@ TEST_F(RooAbsLTest, UnbinnedLikelihoodIntrospection) likelihood = RooFit::TestStatistics::buildLikelihood(pdf, data.get()); EXPECT_STREQ("RooUnbinnedL", (likelihood->GetClassName()).c_str()); - EXPECT_STREQ("RooUnbinnedL::g", (likelihood->GetInfo()).c_str()); + // The pdf is compiled for the normalization set before it is wrapped in the + // RooUnbinnedL, which changes its name from the original "g". + EXPECT_STREQ("RooUnbinnedL::g_over_g_Int[x]", (likelihood->GetInfo()).c_str()); } TEST_F(BinnedDatasetTest, BinnedLikelihoodIntrospection) diff --git a/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx b/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx index 116a163823592..e73b596a862cb 100644 --- a/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx +++ b/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx @@ -28,9 +28,6 @@ #include #include #include -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#include "../../src/RooNLLVar.h" -#endif #include #include // count_if @@ -63,229 +60,10 @@ TEST_P(RooRealL, getVal) EXPECT_DOUBLE_EQ(nominal_result, mp_result); } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -void check_NLL_type(RooAbsReal *nll, bool verbose = false) -{ - if (dynamic_cast(nll) != nullptr) { - if (verbose) { - std::cout << "the NLL object is a RooAddition*..." << std::endl; - } - bool has_rooconstraintsum = false; - for (const auto nll_component : static_cast(nll)->list()) { - if (nll_component->IsA() == RooConstraintSum::Class()) { - has_rooconstraintsum = true; - if (verbose) { - std::cout << "...containing a RooConstraintSum component: " << nll_component->GetName() << std::endl; - } - break; - } else if (nll_component->IsA() != RooNLLVar::Class() && nll_component->IsA() != RooAddition::Class()) { - std::cerr << "... containing an unexpected component class: " << nll_component->ClassName() << std::endl; - throw std::runtime_error("RooAddition* type NLL object contains unexpected component class!"); - } - } - if (!has_rooconstraintsum) { - if (verbose) { - std::cout << "...containing only RooNLLVar components." << std::endl; - } - } - } else if (dynamic_cast(nll) != nullptr) { - if (verbose) { - std::cout << "the NLL object is a RooNLLVar*" << std::endl; - } - } -} - -void count_NLL_components(RooAbsReal *nll, bool verbose = false) -{ - if (dynamic_cast(nll) != nullptr) { - if (verbose) { - std::cout << "the NLL object is a RooAddition*..." << std::endl; - } - std::size_t nll_component_count = 0; - std::unique_ptr components{nll->getComponents()}; - for (const auto &component : *components) { - if (component->IsA() == RooNLLVar::Class()) { - ++nll_component_count; - } - } - if (verbose) { - std::cout << "...containing " << nll_component_count << " RooNLLVar components." << std::endl; - } - } else if (dynamic_cast(nll) != nullptr) { - if (verbose) { - std::cout << "the NLL object is a RooNLLVar*" << std::endl; - } - } -} - -TEST_P(RooRealL, getValRooAddition) -{ - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); - - RooRandom::randomGenerator()->SetSeed(std::get<0>(GetParam())); - - RooWorkspace w; - w.factory("Gaussian::g(x[-10,10],mu[0,-3,3],sigma[1,0.01,5.0])"); - - RooRealVar *x = w.var("x"); - x->setRange("x_range", -3, 0); - x->setRange("another_range", 1, 7); - - RooAbsPdf *pdf = w.pdf("g"); - std::unique_ptr data{pdf->generate(*x, 10000)}; - - using namespace RooFit; - std::unique_ptr nll{pdf->createNLL(*data, Range("x_range,another_range"))}; - - check_NLL_type(nll.get()); - count_NLL_components(nll.get()); -} - -TEST_P(RooRealL, getValRooConstraintSumAddition) -{ - // modified from - // https://github.com/roofit-dev/rootbench/blob/43d12f33e8dac7af7d587b53a2804ddf6717e92f/root/roofit/roofit/RooFitASUM.cxx#L417 - - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); - - RooWorkspace ws; - ws.factory("Polynomial::p0(x[0, 10000])"); - ws.factory("Polynomial::p1(x, {a0[0], a1[1., 0., 2.], a2[0]}, 0)"); - - RooRealVar &x = *ws.var("x"); - RooRealVar &a1 = *ws.var("a1"); - - RooAbsPdf &p0 = *ws.pdf("p0"); - RooAbsPdf &p1 = *ws.pdf("p1"); - - x.setBins(x.getMax()); - - std::unique_ptr dh_bkg{p0.generateBinned(x, 1000000000)}; - std::unique_ptr dh_sig{p1.generateBinned(x, 100000000)}; - dh_bkg->SetName("dh_bkg"); - dh_sig->SetName("dh_sig"); - - a1.setVal(2); - std::unique_ptr dh_sig_up{p1.generateBinned(x, 1100000000)}; - dh_sig_up->SetName("dh_sig_up"); - a1.setVal(.5); - std::unique_ptr dh_sig_down{p1.generateBinned(x, 900000000)}; - dh_sig_down->SetName("dh_sig_down"); - - RooWorkspace w = RooWorkspace("w"); - w.import(x); - w.import(*dh_sig); - w.import(*dh_bkg); - w.import(*dh_sig_up); - w.import(*dh_sig_down); - w.factory("HistFunc::hf_sig(x,dh_sig)"); - w.factory("HistFunc::hf_bkg(x,dh_bkg)"); - - w.factory("ASUM::model(mu[1,0,5]*hf_sig,nu[1]*hf_bkg)"); - w.factory("Gaussian::constraint(mu,2.,1.)"); - w.factory("PROD::model2(model,constraint)"); - - RooAbsPdf *pdf = w.pdf("model2"); - - std::unique_ptr data{pdf->generateBinned(x, 1100000)}; - std::unique_ptr nll{pdf->createNLL(*data)}; - - check_NLL_type(nll.get()); - count_NLL_components(nll.get()); -} - -TEST_P(RooRealL, setVal) -{ - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); - - // calculate the NLL twice with different parameters - const bool verbose = false; - - RooRandom::randomGenerator()->SetSeed(std::get<0>(GetParam())); - RooWorkspace w; - w.factory("Gaussian::g(x[-5,5],mu[0,-3,3],sigma[1,0.01,5.0])"); - auto x = w.var("x"); - RooAbsPdf *pdf = w.pdf("g"); - std::unique_ptr data{pdf->generate(*x, 10000)}; - - // The reference likelihood is using the legacy evaluation backend, because - // the multiprocess test statistics classes were designed to give values - // that are bit-by-bit identical with the old test statistics based on - // RooAbsTestStatistic. - std::unique_ptr nll{pdf->createNLL(*data, RooFit::EvalBackend::Legacy())}; - - RooFit::TestStatistics::RooRealL nll_new("nll_new", "new style NLL", - std::make_unique(pdf, data.get())); - - // calculate first results - auto nominal_result1 = nll->getVal(); - auto mp_result1 = nll_new.getVal(); - - if (verbose) { - std::cout << "nominal_result1 = " << nominal_result1 << ", mp_result1 = " << mp_result1 << std::endl; - } - - EXPECT_EQ(nominal_result1, mp_result1); - - w.var("mu")->setVal(2); - - // calculate second results after parameter change - auto nominal_result2 = nll->getVal(); - auto mp_result2 = nll_new.getVal(); - - if (verbose) { - std::cout << "nominal_result2 = " << nominal_result2 << ", mp_result2 = " << mp_result2 << std::endl; - } - - EXPECT_EQ(nominal_result2, mp_result2); - if (HasFailure()) { - std::cout << "failed test had seed = " << std::get<0>(GetParam()) << std::endl; - } -} -#endif // ROOFIT_LEGACY_EVAL_BACKEND - INSTANTIATE_TEST_SUITE_P(NworkersModeSeed, RooRealL, ::testing::Values(2, 3)); // random seed class RealLVsMPFE : public ::testing::TestWithParam> {}; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -TEST_P(RealLVsMPFE, getVal) -{ - // Compare our MP NLL to actual RooRealMPFE results using the same strategies. - - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); - - // parameters - std::size_t seed = std::get<0>(GetParam()); - - RooRandom::randomGenerator()->SetSeed(seed); - - RooWorkspace w; - w.factory("Gaussian::g(x[-5,5],mu[0,-3,3],sigma[1,0.01,5.0])"); - auto x = w.var("x"); - RooAbsPdf *pdf = w.pdf("g"); - std::unique_ptr data{pdf->generate(*x, 10000)}; - - // The reference likelihood is using the legacy evaluation backend, because - // the multiprocess test statistics classes were designed to give values - // that are bit-by-bit identical with the old test statistics based on - // RooAbsTestStatistic. - std::unique_ptr nll_mpfe{pdf->createNLL(*data, RooFit::EvalBackend::Legacy())}; - - auto mpfe_result = nll_mpfe->getVal(); - - RooFit::TestStatistics::RooRealL nll_new("nll_new", "new style NLL", - std::make_unique(pdf, data.get())); - - auto mp_result = nll_new.getVal(); - - EXPECT_EQ(mpfe_result, mp_result); - if (HasFailure()) { - std::cout << "failed test had seed = " << std::get<0>(GetParam()) << std::endl; - } -} -#endif // ROOFIT_LEGACY_EVAL_BACKEND - TEST_P(RealLVsMPFE, minimize) { // do a minimization (e.g. like in GradMinimizer_Gaussian1D test) diff --git a/roofit/roofitcore/test/gtest_wrapper.h b/roofit/roofitcore/test/gtest_wrapper.h index f342c55575ab8..c378ed31df11b 100644 --- a/roofit/roofitcore/test/gtest_wrapper.h +++ b/roofit/roofitcore/test/gtest_wrapper.h @@ -12,12 +12,6 @@ #endif #endif -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#define ROOFIT_EVAL_BACKEND_LEGACY RooFit::EvalBackend::Legacy(), -#else -#define ROOFIT_EVAL_BACKEND_LEGACY -#endif - #ifdef ROOFIT_CUDA #define ROOFIT_EVAL_BACKEND_CUDA RooFit::EvalBackend::Cuda(), #else @@ -30,7 +24,7 @@ #define ROOFIT_EVAL_BACKEND_CODEGEN #endif -#define ROOFIT_EVAL_BACKENDS ROOFIT_EVAL_BACKEND_LEGACY ROOFIT_EVAL_BACKEND_CUDA RooFit::EvalBackend::Cpu() +#define ROOFIT_EVAL_BACKENDS ROOFIT_EVAL_BACKEND_CUDA RooFit::EvalBackend::Cpu() #define ROOFIT_EVAL_BACKENDS_WITH_CODEGEN \ ROOFIT_EVAL_BACKENDS, ROOFIT_EVAL_BACKEND_CODEGEN RooFit::EvalBackend::CodegenNoGrad() diff --git a/roofit/roofitcore/test/stressRooFit.cxx b/roofit/roofitcore/test/stressRooFit.cxx index 17bcfb842b589..2a070895d3662 100644 --- a/roofit/roofitcore/test/stressRooFit.cxx +++ b/roofit/roofitcore/test/stressRooFit.cxx @@ -241,7 +241,7 @@ int main(int argc, const char *argv[]) int dryRun = false; bool doDump = false; bool doTreeStore = false; - auto backend = RooFit::EvalBackend::Legacy(); + auto backend = RooFit::EvalBackend::Cpu(); // string refFileName = "http://root.cern.ch/files/stressRooFit_v534_ref.root" ; string refFileName = "stressRooFit_ref.root"; diff --git a/roofit/roofitcore/test/testGlobalObservables.cxx b/roofit/roofitcore/test/testGlobalObservables.cxx index c9af30685f034..4dff2730b458c 100644 --- a/roofit/roofitcore/test/testGlobalObservables.cxx +++ b/roofit/roofitcore/test/testGlobalObservables.cxx @@ -46,7 +46,7 @@ bool isNotIdentical(RooFitResult const &res1, RooFitResult const &res2) // we track the global observables separately. class GlobsTest : public testing::TestWithParam> { public: - GlobsTest() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + GlobsTest() : _evalBackend{RooFit::EvalBackend::Cpu()} {} void SetUp() override { diff --git a/roofit/roofitcore/test/testNaNPacker.cxx b/roofit/roofitcore/test/testNaNPacker.cxx index c1f307ab60277..7f4af80d8d560 100644 --- a/roofit/roofitcore/test/testNaNPacker.cxx +++ b/roofit/roofitcore/test/testNaNPacker.cxx @@ -171,7 +171,7 @@ TEST(RooNaNPacker, FitSimpleLinear) class TestForDifferentBackends : public testing::TestWithParam> { public: - TestForDifferentBackends() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + TestForDifferentBackends() : _evalBackend{RooFit::EvalBackend::Cpu()} {} private: void SetUp() override diff --git a/roofit/roofitcore/test/testRooAbsPdf.cxx b/roofit/roofitcore/test/testRooAbsPdf.cxx index f13d2afa9e316..37d5838a1bf84 100644 --- a/roofit/roofitcore/test/testRooAbsPdf.cxx +++ b/roofit/roofitcore/test/testRooAbsPdf.cxx @@ -31,7 +31,7 @@ class FitTest : public testing::TestWithParam> { public: - FitTest() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + FitTest() : _evalBackend{RooFit::EvalBackend::Cpu()} {} private: void SetUp() override @@ -345,24 +345,6 @@ TEST_P(FitTest, MultiRangeFit2D) << "Results of fitting " << model.GetName() << " to a " << data->ClassName() << " should be very similar."; } - // If the BatchMode is off, we are doing the same cross-check also with the - // chi-square fit on the RooDataHist. - if (_evalBackend.name() == EvalBackend::Legacy().name()) { - - // full range - resetValues(); - std::unique_ptr fitResultFull{ - model.fitTo(*dataHist, Range("FULL"), Save(), PrintLevel(-1), _evalBackend)}; - - // part (side band fit, but the union of the side bands is the full range) - resetValues(); - std::unique_ptr fitResultPart{ - model.fitTo(*dataHist, Range("SB1,SB2,SIG"), Save(), PrintLevel(-1), _evalBackend)}; - - EXPECT_TRUE(fitResultPart->isIdentical(*fitResultFull)) - << "Results of fitting " << model.GetName() - << " to a RooDataHist should be very similar also for chi2FitTo()."; - } } // This test will crash if the cached normalization sets are not reset @@ -467,21 +449,14 @@ TEST_P(FitTest, OutOfRangeDataThrows) // dataset's internal clone of the observable still remembers [0, 5]. x.setMax(2.5); - const bool isLegacy = _evalBackend == EvalBackend::Legacy(); - { // Normalizing over [0, 2.5] while still evaluating the entries at 3 would - // bias the fit, so the vectorizing backends throw. The legacy backend is - // not affected by this check and keeps its historical behavior. + // bias the fit, so the backends throw. RooHelpers::HijackMessageStream hijack(RooFit::ERROR, RooFit::InputArguments); auto doFit = [&]() { std::unique_ptr{gauss.fitTo(data, _evalBackend, Save(), PrintLevel(-1))}; }; - if (isLegacy) { - EXPECT_NO_THROW(doFit()); - } else { - EXPECT_THROW(doFit(), std::runtime_error); - } + EXPECT_THROW(doFit(), std::runtime_error); } // Restricting the fit with a named range is the correct approach: the diff --git a/roofit/roofitcore/test/testRooBinSamplingPdf.cxx b/roofit/roofitcore/test/testRooBinSamplingPdf.cxx index 9a297225972ac..3bc23621af1f0 100644 --- a/roofit/roofitcore/test/testRooBinSamplingPdf.cxx +++ b/roofit/roofitcore/test/testRooBinSamplingPdf.cxx @@ -147,12 +147,11 @@ TEST(RooBinSamplingPdf, AnalyticalMatchesNumeric) << "mismatch in bin " << i; } - // The results must also agree when used in a fit, for both the legacy and - // the vectorizing "cpu" evaluation backend. + // The results must also agree when used in a fit. std::unique_ptr dataH(gaus.generateBinned(x, 20000)); RooDataSet data("data", "data", x, RooFit::Import(*dataH)); - for (auto backend : {EvalBackend::Legacy(), EvalBackend::Cpu()}) { + for (auto backend : {EvalBackend::Cpu()}) { mean.setVal(0.7); sigma.setVal(1.3); std::unique_ptr nllAna{gaus.createNLL(data, IntegrateBins(1.E-3), backend)}; diff --git a/roofit/roofitcore/test/testRooMinimizer.cxx b/roofit/roofitcore/test/testRooMinimizer.cxx index 7494742178a7a..f4e956f9419f4 100644 --- a/roofit/roofitcore/test/testRooMinimizer.cxx +++ b/roofit/roofitcore/test/testRooMinimizer.cxx @@ -14,7 +14,7 @@ class EvalBackendParametrizedTest : public testing::TestWithParam> { public: - EvalBackendParametrizedTest() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + EvalBackendParametrizedTest() : _evalBackend{RooFit::EvalBackend::Cpu()} {} private: void SetUp() override diff --git a/roofit/roofitcore/test/testRooProdPdf.cxx b/roofit/roofitcore/test/testRooProdPdf.cxx index d12c7d6c08438..4ffbce68b1c2b 100644 --- a/roofit/roofitcore/test/testRooProdPdf.cxx +++ b/roofit/roofitcore/test/testRooProdPdf.cxx @@ -108,15 +108,6 @@ TEST(RooProdPdf, TestDepsAreCond) EXPECT_TRUE(result4->isIdentical(*result2)) << "alternative model fit is inconsistent!"; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - resetParameters(); - ResultPtr result1{pdf1.fitTo(*data, Save(), EvalBackend::Legacy(), PrintLevel(-1))}; - resetParameters(); - ResultPtr result3{pdf2.fitTo(*data, Save(), EvalBackend::Legacy(), PrintLevel(-1))}; - - EXPECT_TRUE(result2->isIdentical(*result1)) << "legacy fit is inconsistent!"; - EXPECT_TRUE(result4->isIdentical(*result1)) << "alternative model legacy fit is inconsistent!"; -#endif } /// This test covers a potential problem with the custom normalization ranges diff --git a/roofit/roofitcore/test/testRooSimultaneous.cxx b/roofit/roofitcore/test/testRooSimultaneous.cxx index 6792bd866fdf2..75fdaaf944b89 100644 --- a/roofit/roofitcore/test/testRooSimultaneous.cxx +++ b/roofit/roofitcore/test/testRooSimultaneous.cxx @@ -95,13 +95,6 @@ TEST(RooSimultaneous, CategoriesWithNoPdf) // We don't care about the fit result, just that it doesn't crash. using namespace RooFit; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - sim.fitTo(*ds, EvalBackend::Legacy(), PrintLevel(-1)); - m0.setVal(0.5); - m0.setError(0.0); - m1.setVal(0.5); - m1.setError(0.0); -#endif sim.fitTo(*ds, EvalBackend::Cpu(), PrintLevel(-1)); } @@ -147,9 +140,6 @@ TEST(RooSimultaneous, MultiRangeFitWithSplitRange) const char *cutRange1 = "SideBandLo_cat1,SideBandHi_cat1"; const char *cutRange2 = "SideBandLo_cat2,SideBandHi_cat2"; using RealPtr = std::unique_ptr; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - RealPtr nllSim{simPdf.createNLL(combData, Range("SideBandLo,SideBandHi"), SplitRange(), EvalBackend::Legacy())}; -#endif RealPtr nllSimBatch{simPdf.createNLL(combData, Range("SideBandLo,SideBandHi"), SplitRange(), EvalBackend::Cpu())}; // In simultaneous PDFs, the probability is normalized over the categories, @@ -165,17 +155,13 @@ TEST(RooSimultaneous, MultiRangeFitWithSplitRange) RooAddition nllSimRef{"nllSimRef", "nllSimRef", {*nll1, *nll2, RooConst(normTerm)}}; const double nllSimRefVal = nllSimRef.getVal(); -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - const double nllSimVal = nllSim->getVal(); - EXPECT_FLOAT_EQ(nllSimVal, nllSimRefVal); -#endif const double nllSimBatchVal = nllSimBatch->getVal(); - EXPECT_FLOAT_EQ(nllSimBatchVal, nllSimRefVal) << "BatchMode and old RooFit don't agree!"; + EXPECT_FLOAT_EQ(nllSimBatchVal, nllSimRefVal); } class TestStatisticTest : public testing::TestWithParam> { public: - TestStatisticTest() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + TestStatisticTest() : _evalBackend{RooFit::EvalBackend::Cpu()} {} private: void SetUp() override @@ -867,9 +853,6 @@ TEST(RooSimultaneous, ParameterIndexSwitchMode) std::unique_ptr data{refModel0.generate(x, 500)}; std::vector backends; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - backends.push_back(RooFit::EvalBackend::Legacy()); -#endif backends.push_back(RooFit::EvalBackend::Cpu()); backends.push_back(RooFit::EvalBackend::CodegenNoGrad()); @@ -945,9 +928,6 @@ TEST(RooSimultaneous, ParameterIndexTopLevelNLL) std::unique_ptr data{expo.generate(x, 500)}; std::vector backends; -#ifdef ROOFIT_LEGACY_EVAL_BACKEND - backends.push_back(RooFit::EvalBackend::Legacy()); -#endif backends.push_back(RooFit::EvalBackend::Cpu()); backends.push_back(RooFit::EvalBackend::CodegenNoGrad()); diff --git a/roofit/roofitcore/test/testTestStatistics.cxx b/roofit/roofitcore/test/testTestStatistics.cxx index b9fbddcb864f3..c14d56c6d97df 100644 --- a/roofit/roofitcore/test/testTestStatistics.cxx +++ b/roofit/roofitcore/test/testTestStatistics.cxx @@ -11,11 +11,9 @@ #include #include #include +#include #include #include -#ifdef ROOFIT_LEGACY_EVAL_BACKEND -#include "../src/RooNLLVar.h" -#endif #include #include #include @@ -66,7 +64,7 @@ std::unique_ptr generateBinnedAsimov(RooAbsPdf const &pdf, RooRealV class TestStatisticTest : public testing::TestWithParam> { public: - TestStatisticTest() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + TestStatisticTest() : _evalBackend{RooFit::EvalBackend::Cpu()} {} private: void SetUp() override @@ -357,11 +355,9 @@ TEST(RooChi2Var, IntegrateBins) << "Expect chi2/ndf at least 10% better."; } -#ifdef ROOFIT_LEGACY_EVAL_BACKEND static std::vector chi2CrossCheckBackends() { std::vector backends; - backends.push_back(RooFit::EvalBackend::Cpu()); #ifdef ROOFIT_CUDA backends.push_back(RooFit::EvalBackend::Cuda()); #endif @@ -373,8 +369,8 @@ static std::vector chi2CrossCheckBackends() return backends; } -/// Cross-check that every chi2 backend reproduces the legacy RooChi2Var for -/// every supported DataError mode. +/// Cross-check that every chi2 backend reproduces the reference "cpu" backend +/// for every supported DataError mode. TEST(RooChi2Var, ErrorTypesCrossCheck) { using namespace RooFit; @@ -408,32 +404,32 @@ TEST(RooChi2Var, ErrorTypesCrossCheck) // Chi2 value at a fixed parameter point should match to full precision. resetPars(); std::unique_ptr chi2New{gauss.createChi2(*hist, DataError(etype), backend)}; - std::unique_ptr chi2Legacy{gauss.createChi2(*hist, DataError(etype), EvalBackend::Legacy())}; - EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Legacy->getVal()); + std::unique_ptr chi2Ref{gauss.createChi2(*hist, DataError(etype), EvalBackend::Cpu())}; + EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Ref->getVal()); // Minimisation should converge to the same minimum and parameter values. resetPars(); - std::unique_ptr fitLegacy{ - gauss.chi2FitTo(*hist, DataError(etype), EvalBackend::Legacy(), Save(), PrintLevel(-1))}; + std::unique_ptr fitRef{ + gauss.chi2FitTo(*hist, DataError(etype), EvalBackend::Cpu(), Save(), PrintLevel(-1))}; resetPars(); std::unique_ptr fitNew{ gauss.chi2FitTo(*hist, DataError(etype), backend, Save(), PrintLevel(-1))}; - ASSERT_NE(fitLegacy, nullptr); + ASSERT_NE(fitRef, nullptr); ASSERT_NE(fitNew, nullptr); - EXPECT_NEAR(fitNew->minNll(), fitLegacy->minNll(), 1e-6 * std::abs(fitLegacy->minNll()) + 1e-6); + EXPECT_NEAR(fitNew->minNll(), fitRef->minNll(), 1e-6 * std::abs(fitRef->minNll()) + 1e-6); for (const char *parName : {"mean", "sigma"}) { - const double legacyVal = getVal(parName, fitLegacy->floatParsFinal()); + const double refVal = getVal(parName, fitRef->floatParsFinal()); const double newVal = getVal(parName, fitNew->floatParsFinal()); - const double legacyErr = getErr(parName, fitLegacy->floatParsFinal()); + const double refErr = getErr(parName, fitRef->floatParsFinal()); const double newErr = getErr(parName, fitNew->floatParsFinal()); - EXPECT_NEAR(newVal, legacyVal, 1e-5 * std::abs(legacyVal) + 1e-6) << "parameter " << parName; - EXPECT_NEAR(newErr, legacyErr, 1e-4 * std::abs(legacyErr) + 1e-6) << "error of " << parName; + EXPECT_NEAR(newVal, refVal, 1e-5 * std::abs(refVal) + 1e-6) << "parameter " << parName; + EXPECT_NEAR(newErr, refErr, 1e-4 * std::abs(refErr) + 1e-6) << "error of " << parName; } } } - // DataError(None) means "no errors" - legacy returns 0 for any non-empty - // bin. The other backends accept the mode and return 0 as well. + // DataError(None) means "no errors", and the chi2 is defined to be 0 for + // any non-empty bin. for (auto const &backend : chi2CrossCheckBackends()) { SCOPED_TRACE(std::string("None check, backend = ") + backend.name()); std::unique_ptr chi2{gauss.createChi2(*hist, DataError(RooAbsData::None), backend)}; @@ -446,17 +442,17 @@ TEST(RooChi2Var, ErrorTypesCrossCheck) { RooRealVar nbkg("nbkg_func", "", 200., 0., 10000.); RooFormulaVar flat("flat", "flat", "nbkg_func + 0*x", {nbkg, x}); - std::unique_ptr chi2Legacy{ - flat.createChi2(*hist, DataError(RooAbsData::Expected), EvalBackend::Legacy())}; + std::unique_ptr chi2Ref{ + flat.createChi2(*hist, DataError(RooAbsData::Expected), EvalBackend::Cpu())}; for (auto const &backend : chi2CrossCheckBackends()) { SCOPED_TRACE(std::string("Function mode, backend = ") + backend.name()); std::unique_ptr chi2New{flat.createChi2(*hist, DataError(RooAbsData::Expected), backend)}; - EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Legacy->getVal()); + EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Ref->getVal()); } } } -/// Cross-check that every backend reproduces the legacy RooChi2Var for +/// Cross-check that every backend reproduces the reference "cpu" backend for /// named-range fits (including the multi-range "low,high" case) of a plain /// Gaussian model. TEST(RooChi2Var, RangedCrossCheck) @@ -493,33 +489,33 @@ TEST(RooChi2Var, RangedCrossCheck) // Chi2 value at a fixed parameter point. resetPars(); std::unique_ptr chi2New{gauss.createChi2(*hist, Range(rangeName), backend)}; - std::unique_ptr chi2Legacy{gauss.createChi2(*hist, Range(rangeName), EvalBackend::Legacy())}; - EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Legacy->getVal()); + std::unique_ptr chi2Ref{gauss.createChi2(*hist, Range(rangeName), EvalBackend::Cpu())}; + EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Ref->getVal()); // Fit comparison. resetPars(); - std::unique_ptr fitLegacy{ - gauss.chi2FitTo(*hist, Range(rangeName), EvalBackend::Legacy(), Save(), PrintLevel(-1))}; + std::unique_ptr fitRef{ + gauss.chi2FitTo(*hist, Range(rangeName), EvalBackend::Cpu(), Save(), PrintLevel(-1))}; resetPars(); std::unique_ptr fitNew{ gauss.chi2FitTo(*hist, Range(rangeName), backend, Save(), PrintLevel(-1))}; - ASSERT_NE(fitLegacy, nullptr); + ASSERT_NE(fitRef, nullptr); ASSERT_NE(fitNew, nullptr); - EXPECT_NEAR(fitNew->minNll(), fitLegacy->minNll(), 1e-5 * std::abs(fitLegacy->minNll()) + 1e-6); + EXPECT_NEAR(fitNew->minNll(), fitRef->minNll(), 1e-5 * std::abs(fitRef->minNll()) + 1e-6); for (const char *parName : {"mean", "sigma"}) { - const double legacyVal = getVal(parName, fitLegacy->floatParsFinal()); + const double refVal = getVal(parName, fitRef->floatParsFinal()); const double newVal = getVal(parName, fitNew->floatParsFinal()); - const double legacyErr = getErr(parName, fitLegacy->floatParsFinal()); + const double refErr = getErr(parName, fitRef->floatParsFinal()); const double newErr = getErr(parName, fitNew->floatParsFinal()); - EXPECT_NEAR(newVal, legacyVal, 1e-4 * std::abs(legacyVal) + 1e-5) << "parameter " << parName; - EXPECT_NEAR(newErr, legacyErr, 1e-3 * std::abs(legacyErr) + 1e-5) << "error of " << parName; + EXPECT_NEAR(newVal, refVal, 1e-4 * std::abs(refVal) + 1e-5) << "parameter " << parName; + EXPECT_NEAR(newErr, refErr, 1e-3 * std::abs(refErr) + 1e-5) << "error of " << parName; } } } } -/// Cross-check that the evaluation backends for chi2 reproduce the legacy -/// RooChi2Var value, fit minimum and fitted errors for a simultaneous fit. +/// Cross-check that the evaluation backends for chi2 reproduce the reference +/// "cpu" backend value, fit minimum and fitted errors for a simultaneous fit. TEST(RooChi2Var, SimultaneousCrossCheck) { using namespace RooFit; @@ -560,10 +556,10 @@ TEST(RooChi2Var, SimultaneousCrossCheck) ws.var("sigmaB")->setError(0.0); }; - // Legacy baseline, computed once. + // Reference baseline, computed once. resetPars(); - std::unique_ptr fitLegacy{simPdf.chi2FitTo(combHist, EvalBackend::Legacy(), Save(), PrintLevel(-1))}; - ASSERT_NE(fitLegacy, nullptr); + std::unique_ptr fitRef{simPdf.chi2FitTo(combHist, EvalBackend::Cpu(), Save(), PrintLevel(-1))}; + ASSERT_NE(fitRef, nullptr); for (auto const &backend : chi2CrossCheckBackends()) { SCOPED_TRACE(std::string("backend = ") + backend.name()); @@ -571,60 +567,29 @@ TEST(RooChi2Var, SimultaneousCrossCheck) // Chi2 value at a fixed parameter point. resetPars(); std::unique_ptr chi2New{simPdf.createChi2(combHist, backend)}; - std::unique_ptr chi2Legacy{simPdf.createChi2(combHist, EvalBackend::Legacy())}; - EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Legacy->getVal()); + std::unique_ptr chi2Ref{simPdf.createChi2(combHist, EvalBackend::Cpu())}; + EXPECT_FLOAT_EQ(chi2New->getVal(), chi2Ref->getVal()); // Fit with the current backend, compare to the legacy baseline. resetPars(); std::unique_ptr fitNew{simPdf.chi2FitTo(combHist, backend, Save(), PrintLevel(-1))}; ASSERT_NE(fitNew, nullptr); - EXPECT_NEAR(fitNew->minNll(), fitLegacy->minNll(), 1e-6 * std::abs(fitLegacy->minNll()) + 1e-6); + EXPECT_NEAR(fitNew->minNll(), fitRef->minNll(), 1e-6 * std::abs(fitRef->minNll()) + 1e-6); for (const char *parName : {"mean", "sigmaA", "sigmaB"}) { - const double legacyVal = getVal(parName, fitLegacy->floatParsFinal()); + const double refVal = getVal(parName, fitRef->floatParsFinal()); const double newVal = getVal(parName, fitNew->floatParsFinal()); - const double legacyErr = getErr(parName, fitLegacy->floatParsFinal()); + const double refErr = getErr(parName, fitRef->floatParsFinal()); const double newErr = getErr(parName, fitNew->floatParsFinal()); - EXPECT_NEAR(newVal, legacyVal, 1e-5 * std::abs(legacyVal) + 1e-6) << "parameter " << parName; - EXPECT_NEAR(newErr, legacyErr, 1e-4 * std::abs(legacyErr) + 1e-6) << "error of " << parName; + EXPECT_NEAR(newVal, refVal, 1e-5 * std::abs(refVal) + 1e-6) << "parameter " << parName; + EXPECT_NEAR(newErr, refErr, 1e-4 * std::abs(refErr) + 1e-6) << "error of " << parName; } } } -/// Verifies that a ranged RooNLLVar has still the correct value when copied, -/// as it happens when it is plotted Covers JIRA ticket ROOT-9752. -TEST(RooNLLVar, CopyRangedNLL) -{ - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); - - RooWorkspace ws; - ws.factory("Gaussian::model(x[0, 10], mean[5, 0, 10], sigma[0.5, 0.01, 5.0])"); - - RooRealVar &x = *ws.var("x"); - RooAbsPdf &model = *ws.pdf("model"); - - x.setRange("fitrange", 0, 10); - - std::unique_ptr ds{model.generate(x, 20)}; - - // This bug is related to the implementation details of the old test - // statistics, so the EvalBackend is forced to be Legacy - using namespace RooFit; - std::unique_ptr nll{model.createNLL(*ds, EvalBackend::Legacy())}; - std::unique_ptr nllrange{model.createNLL(*ds, Range("fitrange"), EvalBackend::Legacy())}; - - auto nllClone = std::make_unique(static_cast(*nll)); - auto nllrangeClone = std::make_unique(static_cast(*nllrange)); - - EXPECT_FLOAT_EQ(nll->getVal(), nllClone->getVal()); - EXPECT_FLOAT_EQ(nll->getVal(), nllrange->getVal()); - EXPECT_FLOAT_EQ(nllrange->getVal(), nllrangeClone->getVal()); -} -#endif - class OffsetBinTest : public testing::TestWithParam> { public: - OffsetBinTest() : _evalBackend{RooFit::EvalBackend::Legacy()} {} + OffsetBinTest() : _evalBackend{RooFit::EvalBackend::Cpu()} {} private: void SetUp() override diff --git a/roofit/roostats/test/CMakeLists.txt b/roofit/roostats/test/CMakeLists.txt index e14c02d8a35e6..b4ea7b070e6a0 100644 --- a/roofit/roostats/test/CMakeLists.txt +++ b/roofit/roostats/test/CMakeLists.txt @@ -12,14 +12,8 @@ if(mathmore) endif() configure_file(stressRooStats_ref.root stressRooStats_ref.root COPYONLY) -if(roofit_legacy_eval_backend) - ROOT_ADD_TEST(test-stressroostats-legacy COMMAND stressRooStats -b legacy FAILREGEX "FAILED|Error in" LABELS longtest) -endif() ROOT_ADD_TEST(test-stressroostats-cpu COMMAND stressRooStats -b cpu FAILREGEX "FAILED|Error in" LABELS longtest) if(cuda) ROOT_ADD_TEST(test-stressroostats-cuda COMMAND stressRooStats -b cuda FAILREGEX "FAILED|Error in" LABELS longtest RESOURCE_LOCK GPU) endif() -if(roofit_legacy_eval_backend) - ROOT_ADD_TEST(test-stressroostats-legacy-minuit2 COMMAND stressRooStats -minim Minuit2 -b legacy FAILREGEX "FAILED|Error in" LABELS longtest) -endif() ROOT_ADD_TEST(test-stressroostats-cpu-minuit2 COMMAND stressRooStats -minim Minuit2 -b cpu FAILREGEX "FAILED|Error in" LABELS longtest) diff --git a/roofit/roostats/test/stressRooStats.cxx b/roofit/roostats/test/stressRooStats.cxx index 0dbd92997536c..d1d9df387f014 100644 --- a/roofit/roostats/test/stressRooStats.cxx +++ b/roofit/roostats/test/stressRooStats.cxx @@ -301,7 +301,7 @@ int main(int argc, const char *argv[]) bool dryRun = false; bool doDump = false; bool doTreeStore = false; - auto backend = RooFit::EvalBackend::Legacy(); + auto backend = RooFit::EvalBackend::Cpu(); // string refFileName = "http://root.cern/files/stressRooStats_v534_ref.root" ; string refFileName = "stressRooStats_ref.root"; diff --git a/roofit/xroofit/src/xRooNLLVar.cxx b/roofit/xroofit/src/xRooNLLVar.cxx index 4fcb5f1228e9f..fed6d845db8f8 100644 --- a/roofit/xroofit/src/xRooNLLVar.cxx +++ b/roofit/xroofit/src/xRooNLLVar.cxx @@ -26,7 +26,6 @@ This xRooNLLVar object has several special methods, e.g. for fitting and toy dat #include "RooFitResult.h" #if ROOT_VERSION_CODE < ROOT_VERSION(6, 33, 00) -#include "RooNLLVar.h" #endif #ifdef protected