Sequential-Parameter-Optimization-Cookbook.toc

\contentsline {chapter}{Preface}{1}{chapter*.2}%
\contentsline {part}{\numberline {I}Sequential Parameter Optimization Toolbox (SPOT)}{3}{part.1}%
\contentsline {chapter}{\numberline {1}Introduction to SpotOptim}{5}{chapter.1}%
\contentsline {chapter}{\numberline {2}Examples}{7}{chapter.2}%
\contentsline {section}{\numberline {2.1}Complex Constrained Optimization: The Robot Arm}{9}{section.2.1}%
\contentsline {section}{\numberline {2.2}Jupyter Notebook}{10}{section.2.2}%
\contentsline {chapter}{\numberline {3}SpotOptim API Reference}{11}{chapter.3}%
\contentsline {chapter}{\numberline {4}SpotOptim.minimize}{13}{chapter.4}%
\contentsline {section}{\numberline {4.1}API}{13}{section.4.1}%
\contentsline {section}{\numberline {4.2}Parameters}{13}{section.4.2}%
\contentsline {section}{\numberline {4.3}Returns}{14}{section.4.3}%
\contentsline {section}{\numberline {4.4}See Also}{14}{section.4.4}%
\contentsline {section}{\numberline {4.5}Notes}{14}{section.4.5}%
\contentsline {section}{\numberline {4.6}Jupyter Notebook}{14}{section.4.6}%
\contentsline {chapter}{\numberline {5}SpotOptim Internal Methods Examples}{15}{chapter.5}%
\contentsline {section}{\numberline {5.1}Setup: Import Required Packages}{15}{section.5.1}%
\contentsline {section}{\numberline {5.2}1. Main Optimization Method: \texttt {optimize()}}{15}{section.5.2}%
\contentsline {subsection}{\numberline {5.2.1}Changing the Optimizer}{17}{subsection.5.2.1}%
\contentsline {section}{\numberline {5.3}2. Initial Design Methods}{18}{section.5.3}%
\contentsline {subsection}{\numberline {5.3.1}2.1 \texttt {get\_initial\_design()}}{18}{subsection.5.3.1}%
\contentsline {subsection}{\numberline {5.3.2}2.2 \texttt {curate\_initial\_design()}}{19}{subsection.5.3.2}%
\contentsline {subsection}{\numberline {5.3.3}2.3 \texttt {rm\_initial\_design\_NA\_values()}}{20}{subsection.5.3.3}%
\contentsline {subsection}{\numberline {5.3.4}2.4 \texttt {check\_size\_initial\_design()}}{21}{subsection.5.3.4}%
\contentsline {subsection}{\numberline {5.3.5}2.5 \texttt {get\_best\_xy\_initial\_design()}}{22}{subsection.5.3.5}%
\contentsline {section}{\numberline {5.4}3. Surrogate Model Methods}{22}{section.5.4}%
\contentsline {subsection}{\numberline {5.4.1}3.1 \texttt {fit\_surrogate()}}{22}{subsection.5.4.1}%
\contentsline {subsection}{\numberline {5.4.2}3.2 \texttt {\_predict\_with\_uncertainty()}}{23}{subsection.5.4.2}%
\contentsline {subsection}{\numberline {5.4.3}3.3 \texttt {\_acquisition\_function()}}{24}{subsection.5.4.3}%
\contentsline {subsection}{\numberline {5.4.4}3.4 \texttt {suggest\_next\_infill\_point()}}{25}{subsection.5.4.4}%
\contentsline {section}{\numberline {5.5}4. OCBA Method (Optimal Computing Budget Allocation)}{25}{section.5.5}%
\contentsline {subsection}{\numberline {5.5.1}4.1 \texttt {apply\_ocba()}}{25}{subsection.5.5.1}%
\contentsline {section}{\numberline {5.6}5. NaN/Inf Handling Methods}{26}{section.5.6}%
\contentsline {subsection}{\numberline {5.6.1}5.1 \texttt {apply\_penalty\_NA()}}{26}{subsection.5.6.1}%
\contentsline {subsection}{\numberline {5.6.2}5.2 \texttt {remove\_nan()}}{27}{subsection.5.6.2}%
\contentsline {subsection}{\numberline {5.6.3}5.3 \texttt {\_handle\_NA\_new\_points()}}{28}{subsection.5.6.3}%
\contentsline {section}{\numberline {5.7}6. Main Loop Update Methods}{29}{section.5.7}%
\contentsline {subsection}{\numberline {5.7.1}6.1 \texttt {\_update\_best\_main\_loop()}}{29}{subsection.5.7.1}%
\contentsline {section}{\numberline {5.8}7. Termination Method}{30}{section.5.8}%
\contentsline {subsection}{\numberline {5.8.1}7.1 \texttt {determine\_termination()}}{30}{subsection.5.8.1}%
\contentsline {section}{\numberline {5.9}8. Utility Methods}{31}{section.5.9}%
\contentsline {subsection}{\numberline {5.9.1}8.1 \texttt {select\_new()}}{31}{subsection.5.9.1}%
\contentsline {subsection}{\numberline {5.9.2}8.2 \texttt {repair\_non\_numeric()}}{32}{subsection.5.9.2}%
\contentsline {subsection}{\numberline {5.9.3}8.3 \texttt {map\_to\_factor\_values()}}{33}{subsection.5.9.3}%
\contentsline {section}{\numberline {5.10}9. Integration Examples}{33}{section.5.10}%
\contentsline {subsection}{\numberline {5.10.1}9.1 Optimization with Custom Initial Design}{33}{subsection.5.10.1}%
\contentsline {subsection}{\numberline {5.10.2}9.2 Optimization with Starting Point}{34}{subsection.5.10.2}%
\contentsline {subsection}{\numberline {5.10.3}9.3 Optimization with Integer Variables}{35}{subsection.5.10.3}%
\contentsline {subsection}{\numberline {5.10.4}9.4 Noisy Function Optimization with OCBA}{37}{subsection.5.10.4}%
\contentsline {subsection}{\numberline {5.10.5}9.5 Optimization with NaN Handling}{38}{subsection.5.10.5}%
\contentsline {section}{\numberline {5.11}Summary}{39}{section.5.11}%
\contentsline {section}{\numberline {5.12}Jupyter Notebook}{40}{section.5.12}%
\contentsline {chapter}{\numberline {6}Optimizing the Aircraft Wing Weight Example}{41}{chapter.6}%
\contentsline {section}{\numberline {6.1}The AWWE Objective Function}{41}{section.6.1}%
\contentsline {section}{\numberline {6.2}Baseline Configuration}{42}{section.6.2}%
\contentsline {section}{\numberline {6.3}Optimization Setup}{43}{section.6.3}%
\contentsline {section}{\numberline {6.4}Method 1: SpotOptim (Surrogate Model Based Optimization)}{43}{section.6.4}%
\contentsline {section}{\numberline {6.5}Design Table}{44}{section.6.5}%
\contentsline {section}{\numberline {6.6}Run optimization}{44}{section.6.6}%
\contentsline {section}{\numberline {6.7}Result Table}{46}{section.6.7}%
\contentsline {section}{\numberline {6.8}Progress of the Optimization}{46}{section.6.8}%
\contentsline {section}{\numberline {6.9}Contour Plots of Most Important Hyperparameters}{47}{section.6.9}%
\contentsline {section}{\numberline {6.10}Method 2: Nelder-Mead Simplex}{50}{section.6.10}%
\contentsline {section}{\numberline {6.11}Method 3: BFGS (Quasi-Newton)}{51}{section.6.11}%
\contentsline {section}{\numberline {6.12}Comparison of Results}{52}{section.6.12}%
\contentsline {section}{\numberline {6.13}Visualization: Convergence Plots}{53}{section.6.13}%
\contentsline {subsection}{\numberline {6.13.1}SpotOptim Convergence}{53}{subsection.6.13.1}%
\contentsline {section}{\numberline {6.14}Optimal Parameter Values}{54}{section.6.14}%
\contentsline {section}{\numberline {6.15}Analysis of Optimal Solutions}{56}{section.6.15}%
\contentsline {section}{\numberline {6.16}Key Insights from Optimal Solutions}{57}{section.6.16}%
\contentsline {section}{\numberline {6.17}Method Efficiency Comparison}{58}{section.6.17}%
\contentsline {section}{\numberline {6.18}Visualization: 2D Slices of Optimal Solutions}{59}{section.6.18}%
\contentsline {section}{\numberline {6.19}Conclusion}{61}{section.6.19}%
\contentsline {section}{\numberline {6.20}Jupyter Notebook}{62}{section.6.20}%
\contentsline {chapter}{\numberline {7}Benchmarking SpotOptim with Sklearn Kriging (Matern Kernel) on 6D Rosenbrock and 10D Michalewicz Functions}{63}{chapter.7}%
\contentsline {section}{\numberline {7.1}SpotOptim with Sklearn Kriging in 6 Dimensions: Rosenbrock Function}{63}{section.7.1}%
\contentsline {subsection}{\numberline {7.1.1}Define the 6D Rosenbrock Function}{63}{subsection.7.1.1}%
\contentsline {subsection}{\numberline {7.1.2}Set up SpotOptim Parameters}{64}{subsection.7.1.2}%
\contentsline {subsection}{\numberline {7.1.3}Sklearn Gaussian Process Regressor as Surrogate}{64}{subsection.7.1.3}%
\contentsline {subsection}{\numberline {7.1.4}Visualize Optimization Progress}{67}{subsection.7.1.4}%
\contentsline {subsection}{\numberline {7.1.5}Evaluation of Multiple Repeats}{68}{subsection.7.1.5}%
\contentsline {section}{\numberline {7.2}SpotOptim with Sklearn Kriging in 10 Dimensions: Michalewicz Function}{69}{section.7.2}%
\contentsline {subsection}{\numberline {7.2.1}Define the 10D Michalewicz Function}{69}{subsection.7.2.1}%
\contentsline {subsection}{\numberline {7.2.2}Set up SpotOptim Parameters}{70}{subsection.7.2.2}%
\contentsline {subsection}{\numberline {7.2.3}Sklearn Gaussian Process Regressor as Surrogate}{70}{subsection.7.2.3}%
\contentsline {subsection}{\numberline {7.2.4}Visualize Optimization Progress}{77}{subsection.7.2.4}%
\contentsline {subsection}{\numberline {7.2.5}Evaluation of Multiple Repeats}{78}{subsection.7.2.5}%
\contentsline {section}{\numberline {7.3}Comparison: SpotOptim vs SpotPython}{79}{section.7.3}%
\contentsline {section}{\numberline {7.4}Summary}{80}{section.7.4}%
\contentsline {section}{\numberline {7.5}Jupyter Notebook}{80}{section.7.5}%
\contentsline {chapter}{\numberline {8}Success Rate Tracking in SpotOptim}{81}{chapter.8}%
\contentsline {section}{\numberline {8.1}What is Success Rate?}{81}{section.8.1}%
\contentsline {section}{\numberline {8.2}First Example}{81}{section.8.2}%
\contentsline {section}{\numberline {8.3}Second Example}{82}{section.8.3}%
\contentsline {section}{\numberline {8.4}Accessing Success Rate}{83}{section.8.4}%
\contentsline {section}{\numberline {8.5}Interpreting Success Rate}{84}{section.8.5}%
\contentsline {subsection}{\numberline {8.5.1}High Success Rate (\textgreater {} 0.5)}{84}{subsection.8.5.1}%
\contentsline {subsection}{\numberline {8.5.2}Medium Success Rate (0.2 - 0.5)}{84}{subsection.8.5.2}%
\contentsline {subsection}{\numberline {8.5.3}Low Success Rate (\textless {} 0.2)}{85}{subsection.8.5.3}%
\contentsline {section}{\numberline {8.6}TensorBoard Visualization}{85}{section.8.6}%
\contentsline {section}{\numberline {8.7}Example: Comparing Multiple Runs}{85}{section.8.7}%
\contentsline {section}{\numberline {8.8}Success Rate with Noisy Functions}{87}{section.8.8}%
\contentsline {section}{\numberline {8.9}Advanced: Custom Window Size}{88}{section.8.9}%
\contentsline {section}{\numberline {8.10}Best Practices}{89}{section.8.10}%
\contentsline {subsection}{\numberline {8.10.1}1. Monitor During Long Runs}{89}{subsection.8.10.1}%
\contentsline {subsection}{\numberline {8.10.2}2. Combine with TensorBoard}{89}{subsection.8.10.2}%
\contentsline {subsection}{\numberline {8.10.3}3. Use as Stopping Criterion}{89}{subsection.8.10.3}%
\contentsline {subsection}{\numberline {8.10.4}4. Compare Different Strategies}{90}{subsection.8.10.4}%
\contentsline {section}{\numberline {8.11}Technical Details}{90}{section.8.11}%
\contentsline {subsection}{\numberline {8.11.1}How Success is Counted}{90}{subsection.8.11.1}%
\contentsline {subsection}{\numberline {8.11.2}Rolling Window Calculation}{90}{subsection.8.11.2}%
\contentsline {subsection}{\numberline {8.11.3}Update Frequency}{90}{subsection.8.11.3}%
\contentsline {section}{\numberline {8.12}Summary}{91}{section.8.12}%
\contentsline {section}{\numberline {8.13}Jupyter Notebook}{91}{section.8.13}%
\contentsline {chapter}{\numberline {9}Parallelization}{93}{chapter.9}%
\contentsline {section}{\numberline {9.1}Overview}{93}{section.9.1}%
\contentsline {section}{\numberline {9.2}How it Workstest/}{93}{section.9.2}%
\contentsline {section}{\numberline {9.3}Benchmark Example}{94}{section.9.3}%
\contentsline {subsection}{\numberline {9.3.1}Benchmark Script}{94}{subsection.9.3.1}%
\contentsline {subsection}{\numberline {9.3.2}Results}{97}{subsection.9.3.2}%
\contentsline {section}{\numberline {9.4}Best Practices}{97}{section.9.4}%
\contentsline {section}{\numberline {9.5}Jupyter Notebook}{98}{section.9.5}%
\contentsline {chapter}{\numberline {10}Restart Mechanism in SpotOptim}{99}{chapter.10}%
\contentsline {section}{\numberline {10.1}Key Concepts}{99}{section.10.1}%
\contentsline {subsection}{\numberline {10.1.1}Parameters}{99}{subsection.10.1.1}%
\contentsline {section}{\numberline {10.2}How it Works}{99}{section.10.2}%
\contentsline {section}{\numberline {10.3}Example: Triggering Restarts}{100}{section.10.3}%
\contentsline {subsection}{\numberline {10.3.1}Analyzing Restart Results}{102}{subsection.10.3.1}%
\contentsline {section}{\numberline {10.4}Example: Effect of \texttt {restart\_inject\_best}}{102}{section.10.4}%
\contentsline {section}{\numberline {10.5}When to Use Restarts?}{105}{section.10.5}%
\contentsline {part}{\numberline {II}Variables and Hyperparameters}{107}{part.2}%
\contentsline {chapter}{\numberline {11}Variable Type (var\_type) Implementation in SpotOptim}{109}{chapter.11}%
\contentsline {section}{\numberline {11.1}Overview}{109}{section.11.1}%
\contentsline {section}{\numberline {11.2}Supported Variable Types}{109}{section.11.2}%
\contentsline {subsection}{\numberline {11.2.1}1. \textbf {`float'}}{109}{subsection.11.2.1}%
\contentsline {subsection}{\numberline {11.2.2}2. \textbf {`int'}}{109}{subsection.11.2.2}%
\contentsline {subsection}{\numberline {11.2.3}3. \textbf {`factor'}}{109}{subsection.11.2.3}%
\contentsline {section}{\numberline {11.3}Implementation Details}{110}{section.11.3}%
\contentsline {subsection}{\numberline {11.3.1}Where \texttt {var\_type} is Used}{110}{subsection.11.3.1}%
\contentsline {subsection}{\numberline {11.3.2}Core Method: \texttt {repair\_non\_numeric()}}{110}{subsection.11.3.2}%
\contentsline {section}{\numberline {11.4}Example Usage}{111}{section.11.4}%
\contentsline {subsection}{\numberline {11.4.1}Example 1: All Float Variables (Default)}{111}{subsection.11.4.1}%
\contentsline {subsection}{\numberline {11.4.2}Example 2: Pure Integer Optimization}{111}{subsection.11.4.2}%
\contentsline {subsection}{\numberline {11.4.3}Example 3: Categorical (Factor) Variables}{112}{subsection.11.4.3}%
\contentsline {subsection}{\numberline {11.4.4}Example 4: Mixed Variable Types}{113}{subsection.11.4.4}%
\contentsline {section}{\numberline {11.5}Key Findings}{114}{section.11.5}%
\contentsline {section}{\numberline {11.6}Recommendations}{114}{section.11.6}%
\contentsline {section}{\numberline {11.7}Future Enhancements (Optional)}{114}{section.11.7}%
\contentsline {section}{\numberline {11.8}Jupyter Notebook}{115}{section.11.8}%
\contentsline {chapter}{\numberline {12}Factor Variables for Categorical Hyperparameters}{117}{chapter.12}%
\contentsline {section}{\numberline {12.1}Overview}{117}{section.12.1}%
\contentsline {section}{\numberline {12.2}Quick Start}{117}{section.12.2}%
\contentsline {subsection}{\numberline {12.2.1}Basic Factor Variable Usage}{117}{subsection.12.2.1}%
\contentsline {subsection}{\numberline {12.2.2}Neural Network Activation Function Optimization}{119}{subsection.12.2.2}%
\contentsline {section}{\numberline {12.3}Mixed Variable Types}{121}{section.12.3}%
\contentsline {subsection}{\numberline {12.3.1}Combining Factor, Integer, and Continuous Variables}{121}{subsection.12.3.1}%
\contentsline {section}{\numberline {12.4}Multiple Factor Variables}{124}{section.12.4}%
\contentsline {subsection}{\numberline {12.4.1}Optimizing Both Activation and Optimizer}{124}{subsection.12.4.1}%
\contentsline {section}{\numberline {12.5}Advanced Usage}{126}{section.12.5}%
\contentsline {subsection}{\numberline {12.5.1}Custom Categorical Choices}{126}{subsection.12.5.1}%
\contentsline {subsection}{\numberline {12.5.2}Viewing All Evaluated Configurations}{127}{subsection.12.5.2}%
\contentsline {section}{\numberline {12.6}How It Works}{130}{section.12.6}%
\contentsline {subsection}{\numberline {12.6.1}Internal Mechanism}{130}{subsection.12.6.1}%
\contentsline {subsection}{\numberline {12.6.2}Variable Type Auto-Detection}{131}{subsection.12.6.2}%
\contentsline {section}{\numberline {12.7}Complete Example: Full Workflow}{132}{section.12.7}%
\contentsline {section}{\numberline {12.8}Best Practices}{140}{section.12.8}%
\contentsline {subsection}{\numberline {12.8.1}Do's}{140}{subsection.12.8.1}%
\contentsline {subsection}{\numberline {12.8.2}Don'ts}{141}{subsection.12.8.2}%
\contentsline {section}{\numberline {12.9}Troubleshooting}{142}{section.12.9}%
\contentsline {subsection}{\numberline {12.9.1}Common Issues}{142}{subsection.12.9.1}%
\contentsline {section}{\numberline {12.10}Summary}{143}{section.12.10}%
\contentsline {section}{\numberline {12.11}Jupyter Notebook}{143}{section.12.11}%
\contentsline {chapter}{\numberline {13}Variable Transformations for Search Space Scaling}{145}{chapter.13}%
\contentsline {section}{\numberline {13.1}Overview}{145}{section.13.1}%
\contentsline {section}{\numberline {13.2}Why Use Transformations?}{145}{section.13.2}%
\contentsline {subsection}{\numberline {13.2.1}Problem: Poorly Scaled Search Spaces}{145}{subsection.13.2.1}%
\contentsline {subsection}{\numberline {13.2.2}Solution: Logarithmic and Other Transformations}{146}{subsection.13.2.2}%
\contentsline {section}{\numberline {13.3}Quick Start}{146}{section.13.3}%
\contentsline {subsection}{\numberline {13.3.1}Basic Log-Scale Transformation}{146}{subsection.13.3.1}%
\contentsline {section}{\numberline {13.4}Supported Transformations}{147}{section.13.4}%
\contentsline {subsection}{\numberline {13.4.1}Transformation Guidelines}{148}{subsection.13.4.1}%
\contentsline {section}{\numberline {13.5}Detailed Examples}{149}{section.13.5}%
\contentsline {subsection}{\numberline {13.5.1}Example 1: Neural Network Hyperparameter Tuning}{149}{subsection.13.5.1}%
\contentsline {subsection}{\numberline {13.5.2}Example 2: Physics-Informed Neural Networks (PINNs)}{152}{subsection.13.5.2}%
\contentsline {subsection}{\numberline {13.5.3}Example 3: Mixing Transformations}{153}{subsection.13.5.3}%
\contentsline {section}{\numberline {13.6}Viewing Transformations in Tables}{154}{section.13.6}%
\contentsline {subsection}{\numberline {13.6.1}Design Table (Before Optimization)}{154}{subsection.13.6.1}%
\contentsline {subsection}{\numberline {13.6.2}Results Table (After Optimization)}{155}{subsection.13.6.2}%
\contentsline {section}{\numberline {13.7}Internal Architecture}{155}{section.13.7}%
\contentsline {subsection}{\numberline {13.7.1}Flow Diagram}{156}{subsection.13.7.1}%
\contentsline {subsection}{\numberline {13.7.2}Key Components}{156}{subsection.13.7.2}%
\contentsline {section}{\numberline {13.8}Best Practices}{157}{section.13.8}%
\contentsline {subsection}{\numberline {13.8.1}1. Choose Appropriate Transformations}{157}{subsection.13.8.1}%
\contentsline {subsection}{\numberline {13.8.2}2. Match Transformation to Range}{157}{subsection.13.8.2}%
\contentsline {subsection}{\numberline {13.8.3}3. Validate Transformation Choice}{157}{subsection.13.8.3}%
\contentsline {subsection}{\numberline {13.8.4}4. Combine with Variable Types}{158}{subsection.13.8.4}%
\contentsline {section}{\numberline {13.9}Troubleshooting}{158}{section.13.9}%
\contentsline {subsection}{\numberline {13.9.1}Issue: Values Out of Bounds}{158}{subsection.13.9.1}%
\contentsline {subsection}{\numberline {13.9.2}Issue: Poor Optimization Performance}{158}{subsection.13.9.2}%
\contentsline {subsection}{\numberline {13.9.3}Issue: Transformation Not Applied}{159}{subsection.13.9.3}%
\contentsline {section}{\numberline {13.10}Comparison: Manual vs Automatic Transformations}{159}{section.13.10}%
\contentsline {subsection}{\numberline {13.10.1}Manual Approach (Old Way)}{159}{subsection.13.10.1}%
\contentsline {subsection}{\numberline {13.10.2}Automatic Approach (New Way)}{160}{subsection.13.10.2}%
\contentsline {section}{\numberline {13.11}Summary}{161}{section.13.11}%
\contentsline {section}{\numberline {13.12}Jupyter Notebook}{161}{section.13.12}%
\contentsline {part}{\numberline {III}Visualization}{163}{part.3}%
\contentsline {chapter}{\numberline {14}Surrogate Model Visualization}{165}{chapter.14}%
\contentsline {section}{\numberline {14.1}Overview}{165}{section.14.1}%
\contentsline {section}{\numberline {14.2}Features}{165}{section.14.2}%
\contentsline {section}{\numberline {14.3}Usage}{165}{section.14.3}%
\contentsline {subsection}{\numberline {14.3.1}Basic Usage}{165}{subsection.14.3.1}%
\contentsline {subsection}{\numberline {14.3.2}With Custom Parameters}{166}{subsection.14.3.2}%
\contentsline {subsection}{\numberline {14.3.3}Higher-Dimensional Problems}{167}{subsection.14.3.3}%
\contentsline {section}{\numberline {14.4}Plot Interpretation}{169}{section.14.4}%
\contentsline {subsection}{\numberline {14.4.1}Top Left: Prediction Surface}{169}{subsection.14.4.1}%
\contentsline {subsection}{\numberline {14.4.2}Top Right: Prediction Uncertainty Surface}{169}{subsection.14.4.2}%
\contentsline {subsection}{\numberline {14.4.3}Bottom Left: Prediction Contour with Points}{170}{subsection.14.4.3}%
\contentsline {subsection}{\numberline {14.4.4}Bottom Right: Uncertainty Contour with Points}{170}{subsection.14.4.4}%
\contentsline {section}{\numberline {14.5}Parameters}{170}{section.14.5}%
\contentsline {subsection}{\numberline {14.5.1}Dimension Selection}{170}{subsection.14.5.1}%
\contentsline {subsection}{\numberline {14.5.2}Appearance}{170}{subsection.14.5.2}%
\contentsline {subsection}{\numberline {14.5.3}Grid and Resolution}{170}{subsection.14.5.3}%
\contentsline {subsection}{\numberline {14.5.4}Color Scaling}{170}{subsection.14.5.4}%
\contentsline {subsection}{\numberline {14.5.5}Display}{171}{subsection.14.5.5}%
\contentsline {section}{\numberline {14.6}Examples}{171}{section.14.6}%
\contentsline {subsection}{\numberline {14.6.1}Example 1: 2D Rosenbrock Function}{171}{subsection.14.6.1}%
\contentsline {subsection}{\numberline {14.6.2}Example 2: Using Kriging Surrogate}{172}{subsection.14.6.2}%
\contentsline {subsection}{\numberline {14.6.3}Example 3: Comparing Different Dimension Pairs}{173}{subsection.14.6.3}%
\contentsline {section}{\numberline {14.7}Tips and Best Practices}{176}{section.14.7}%
\contentsline {section}{\numberline {14.8}Comparison with spotpython's plotkd()}{177}{section.14.8}%
\contentsline {subsection}{\numberline {14.8.1}Similarities}{177}{subsection.14.8.1}%
\contentsline {subsection}{\numberline {14.8.2}Differences}{177}{subsection.14.8.2}%
\contentsline {section}{\numberline {14.9}Error Handling}{177}{section.14.9}%
\contentsline {section}{\numberline {14.10}Jupyter Notebook}{177}{section.14.10}%
\contentsline {chapter}{\numberline {15}TensorBoard Logging in SpotOptim}{179}{chapter.15}%
\contentsline {section}{\numberline {15.1}Quick Start}{179}{section.15.1}%
\contentsline {subsection}{\numberline {15.1.1}Enable TensorBoard Logging}{179}{subsection.15.1.1}%
\contentsline {subsection}{\numberline {15.1.2}View Logs in TensorBoard}{180}{subsection.15.1.2}%
\contentsline {subsection}{\numberline {15.1.3}Cleaning Old Logs}{180}{subsection.15.1.3}%
\contentsline {subsection}{\numberline {15.1.4}Use Cases}{181}{subsection.15.1.4}%
\contentsline {subsection}{\numberline {15.1.5}Custom Log Directory}{181}{subsection.15.1.5}%
\contentsline {subsection}{\numberline {15.1.6}What Gets Logged}{181}{subsection.15.1.6}%
\contentsline {subsubsection}{\numberline {15.1.6.1}Scalar Metrics}{181}{subsubsection.15.1.6.1}%
\contentsline {subsubsection}{\numberline {15.1.6.2}Hyperparameters}{182}{subsubsection.15.1.6.2}%
\contentsline {subsection}{\numberline {15.1.7}Examples}{182}{subsection.15.1.7}%
\contentsline {section}{\numberline {15.2}TensorBoard Features}{184}{section.15.2}%
\contentsline {subsection}{\numberline {15.2.1}SCALARS Tab}{184}{subsection.15.2.1}%
\contentsline {subsection}{\numberline {15.2.2}HPARAMS Tab}{185}{subsection.15.2.2}%
\contentsline {subsection}{\numberline {15.2.3}Text Tab}{185}{subsection.15.2.3}%
\contentsline {section}{\numberline {15.3}TensorBoard Log Cleaning Feature in SpotOptim}{185}{section.15.3}%
\contentsline {subsection}{\numberline {15.3.1}Basic Usage}{186}{subsection.15.3.1}%
\contentsline {subsection}{\numberline {15.3.2}Use Cases}{187}{subsection.15.3.2}%
\contentsline {subsection}{\numberline {15.3.3}Implementation Details}{187}{subsection.15.3.3}%
\contentsline {subsubsection}{\numberline {15.3.3.1}Cleaning Method}{187}{subsubsection.15.3.3.1}%
\contentsline {subsection}{\numberline {15.3.4}Execution Flow}{187}{subsection.15.3.4}%
\contentsline {section}{\numberline {15.4}Safety Features}{188}{section.15.4}%
\contentsline {section}{\numberline {15.5}Tensorboard Demo Scripts}{188}{section.15.5}%
\contentsline {section}{\numberline {15.6}Troubleshooting}{189}{section.15.6}%
\contentsline {section}{\numberline {15.7}Related Parameters}{189}{section.15.7}%
\contentsline {section}{\numberline {15.8}Performance Notes}{189}{section.15.8}%
\contentsline {section}{\numberline {15.9}Jupyter Notebook}{190}{section.15.9}%
\contentsline {part}{\numberline {IV}Saving and Loading Models}{191}{part.4}%
\contentsline {chapter}{\numberline {16}Save and Load in SpotOptim}{193}{chapter.16}%
\contentsline {section}{\numberline {16.1}Key Concepts}{193}{section.16.1}%
\contentsline {subsection}{\numberline {16.1.1}Experiments vs Results}{193}{subsection.16.1.1}%
\contentsline {section}{\numberline {16.2}Quick Start}{194}{section.16.2}%
\contentsline {section}{\numberline {16.3}Distributed Workflow}{194}{section.16.3}%
\contentsline {subsection}{\numberline {16.3.1}Step 1: Define Experiment Locally}{195}{subsection.16.3.1}%
\contentsline {subsection}{\numberline {16.3.2}Step 2: Execute on Remote Machine}{195}{subsection.16.3.2}%
\contentsline {subsection}{\numberline {16.3.3}Step 3: Analyze Results Locally}{196}{subsection.16.3.3}%
\contentsline {section}{\numberline {16.4}Advanced Usage}{198}{section.16.4}%
\contentsline {subsection}{\numberline {16.4.1}Custom Filenames and Paths}{198}{subsection.16.4.1}%
\contentsline {subsection}{\numberline {16.4.2}Overwrite Protection}{198}{subsection.16.4.2}%
\contentsline {subsection}{\numberline {16.4.3}Loading and Continuing Optimization}{199}{subsection.16.4.3}%
\contentsline {section}{\numberline {16.5}Working with Noisy Functions}{200}{section.16.5}%
\contentsline {section}{\numberline {16.6}Working with Different Variable Types}{201}{section.16.6}%
\contentsline {section}{\numberline {16.7}Best Practices}{203}{section.16.7}%
\contentsline {subsection}{\numberline {16.7.1}1. Re-attaching the Objective Function (Optional)}{203}{subsection.16.7.1}%
\contentsline {subsection}{\numberline {16.7.2}2. Use Meaningful Prefixes}{203}{subsection.16.7.2}%
\contentsline {subsection}{\numberline {16.7.3}3. Save Experiments Before Remote Execution}{203}{subsection.16.7.3}%
\contentsline {subsection}{\numberline {16.7.4}4. Version Your Experiments}{204}{subsection.16.7.4}%
\contentsline {subsection}{\numberline {16.7.5}5. Handle File Paths Robustly}{204}{subsection.16.7.5}%
\contentsline {section}{\numberline {16.8}Complete Example: Multi-Machine Workflow}{204}{section.16.8}%
\contentsline {subsection}{\numberline {16.8.1}Local Machine (Setup)}{205}{subsection.16.8.1}%
\contentsline {subsection}{\numberline {16.8.2}Remote Machine (Execution)}{206}{subsection.16.8.2}%
\contentsline {subsection}{\numberline {16.8.3}Local Machine (Analysis)}{208}{subsection.16.8.3}%
\contentsline {section}{\numberline {16.9}Technical Details}{210}{section.16.9}%
\contentsline {subsection}{\numberline {16.9.1}Serialization Method}{210}{subsection.16.9.1}%
\contentsline {subsection}{\numberline {16.9.2}Component Reinitialization}{211}{subsection.16.9.2}%
\contentsline {subsection}{\numberline {16.9.3}Excluded Components}{211}{subsection.16.9.3}%
\contentsline {subsection}{\numberline {16.9.4}File Format}{211}{subsection.16.9.4}%
\contentsline {section}{\numberline {16.10}Troubleshooting}{211}{section.16.10}%
\contentsline {subsection}{\numberline {16.10.1}Issue: ``AttributeError: `SpotOptim' object has no attribute `fun'\,''}{211}{subsection.16.10.1}%
\contentsline {subsection}{\numberline {16.10.2}Issue: ``FileNotFoundError: Experiment file not found''}{212}{subsection.16.10.2}%
\contentsline {subsection}{\numberline {16.10.3}Issue: ``FileExistsError: File already exists''}{212}{subsection.16.10.3}%
\contentsline {subsection}{\numberline {16.10.4}Issue: Results differ after loading}{212}{subsection.16.10.4}%
\contentsline {section}{\numberline {16.11}Jupyter Notebook}{213}{section.16.11}%
\contentsline {chapter}{\numberline {17}Reproducibility in SpotOptim}{215}{chapter.17}%
\contentsline {section}{\numberline {17.1}Introduction}{215}{section.17.1}%
\contentsline {section}{\numberline {17.2}Basic Usage}{215}{section.17.2}%
\contentsline {subsection}{\numberline {17.2.1}Making Optimization Reproducible}{215}{subsection.17.2.1}%
\contentsline {subsection}{\numberline {17.2.2}Running Independent Experiments}{216}{subsection.17.2.2}%
\contentsline {section}{\numberline {17.3}Practical Examples}{217}{section.17.3}%
\contentsline {subsection}{\numberline {17.3.1}Example 1: Comparing Different Configurations}{217}{subsection.17.3.1}%
\contentsline {subsection}{\numberline {17.3.2}Example 2: Reproducible Research Experiment}{218}{subsection.17.3.2}%
\contentsline {subsection}{\numberline {17.3.3}Example 3: Multiple Independent Runs}{220}{subsection.17.3.3}%
\contentsline {subsection}{\numberline {17.3.4}Example 4: Reproducible Initial Design}{222}{subsection.17.3.4}%
\contentsline {subsection}{\numberline {17.3.5}Example 5: Custom Initial Design with Seed}{223}{subsection.17.3.5}%
\contentsline {section}{\numberline {17.4}Advanced Topics}{224}{section.17.4}%
\contentsline {subsection}{\numberline {17.4.1}Seed and Noisy Functions}{224}{subsection.17.4.1}%
\contentsline {subsection}{\numberline {17.4.2}Different Seeds for Different Exploration}{225}{subsection.17.4.2}%
\contentsline {section}{\numberline {17.5}Best Practices}{226}{section.17.5}%
\contentsline {subsection}{\numberline {17.5.1}1. Always Use Seeds for Production Code}{226}{subsection.17.5.1}%
\contentsline {subsection}{\numberline {17.5.2}2. Document Your Seeds}{226}{subsection.17.5.2}%
\contentsline {subsection}{\numberline {17.5.3}3. Use Different Seeds for Different Experiments}{226}{subsection.17.5.3}%
\contentsline {subsection}{\numberline {17.5.4}4. Test Robustness Across Multiple Seeds}{227}{subsection.17.5.4}%
\contentsline {section}{\numberline {17.6}What the Seed Controls}{227}{section.17.6}%
\contentsline {section}{\numberline {17.7}Common Questions}{227}{section.17.7}%
\contentsline {section}{\numberline {17.8}Summary}{228}{section.17.8}%
\contentsline {section}{\numberline {17.9}Jupyter Notebook}{228}{section.17.9}%
\contentsline {part}{\numberline {V}Surrogate Handling}{229}{part.5}%
\contentsline {chapter}{\numberline {18}Surrogate Model Selection in SpotOptim}{231}{chapter.18}%
\contentsline {section}{\numberline {18.1}Introduction}{231}{section.18.1}%
\contentsline {subsection}{\numberline {18.1.1}Acquisition Functions and Uncertainty}{232}{subsection.18.1.1}%
\contentsline {section}{\numberline {18.2}Setup and Imports}{232}{section.18.2}%
\contentsline {section}{\numberline {18.3}The AWWE Objective Function}{233}{section.18.3}%
\contentsline {section}{\numberline {18.4}1. Default Surrogate: Gaussian Process with Matern Kernel}{233}{section.18.4}%
\contentsline {subsection}{\numberline {18.4.1}Visualization: Default Surrogate}{234}{subsection.18.4.1}%
\contentsline {section}{\numberline {18.5}2. Gaussian Process with RBF (Radial Basis Function) Kernel}{238}{section.18.5}%
\contentsline {subsection}{\numberline {18.5.1}Visualization: RBF Kernel}{240}{subsection.18.5.1}%
\contentsline {section}{\numberline {18.6}3. Gaussian Process with Matern nu=1.5 Kernel}{243}{section.18.6}%
\contentsline {subsection}{\numberline {18.6.1}Visualization: Matern nu=1.5 Kernel}{245}{subsection.18.6.1}%
\contentsline {section}{\numberline {18.7}4. Gaussian Process with Rational Quadratic Kernel}{248}{section.18.7}%
\contentsline {subsection}{\numberline {18.7.1}Visualization: Rational Quadratic Kernel}{250}{subsection.18.7.1}%
\contentsline {section}{\numberline {18.8}5. SpotOptim Kriging Model}{253}{section.18.8}%
\contentsline {subsection}{\numberline {18.8.1}Visualization: Kriging Model}{255}{subsection.18.8.1}%
\contentsline {section}{\numberline {18.9}6. Random Forest Regressor}{258}{section.18.9}%
\contentsline {subsection}{\numberline {18.9.1}Visualization: Random Forest}{260}{subsection.18.9.1}%
\contentsline {section}{\numberline {18.10}7. XGBoost Regressor}{263}{section.18.10}%
\contentsline {section}{\numberline {18.11}8. Support Vector Regression (SVR)}{269}{section.18.11}%
\contentsline {subsection}{\numberline {18.11.1}Visualization: SVR}{270}{subsection.18.11.1}%
\contentsline {section}{\numberline {18.12}9. Gradient Boosting Regressor}{274}{section.18.12}%
\contentsline {subsection}{\numberline {18.12.1}Visualization: Gradient Boosting}{275}{subsection.18.12.1}%
\contentsline {section}{\numberline {18.13}Comprehensive Comparison}{279}{section.18.13}%
\contentsline {subsection}{\numberline {18.13.1}Visualization: Performance Comparison}{280}{subsection.18.13.1}%
\contentsline {section}{\numberline {18.14}Convergence Comparison}{282}{section.18.14}%
\contentsline {section}{\numberline {18.15}Key Insights and Recommendations}{283}{section.18.15}%
\contentsline {section}{\numberline {18.16}Summary Statistics}{285}{section.18.16}%
\contentsline {section}{\numberline {18.17}Conclusion}{287}{section.18.17}%
\contentsline {section}{\numberline {18.18}Jupyter Notebook}{287}{section.18.18}%
\contentsline {chapter}{\numberline {19}Acquisition Failure Handling in SpotOptim}{289}{chapter.19}%
\contentsline {section}{\numberline {19.1}What is Acquisition Failure?}{289}{section.19.1}%
\contentsline {section}{\numberline {19.2}Fallback Strategies}{289}{section.19.2}%
\contentsline {subsection}{\numberline {19.2.1}Random Space-Filling Design (Default)}{289}{subsection.19.2.1}%
\contentsline {section}{\numberline {19.3}How It Works}{291}{section.19.3}%
\contentsline {section}{\numberline {19.4}Advanced Usage: Setting Tolerance}{291}{section.19.4}%
\contentsline {section}{\numberline {19.5}Best Practices}{292}{section.19.5}%
\contentsline {subsection}{\numberline {19.5.1}1. Monitor Fallback Activations}{292}{subsection.19.5.1}%
\contentsline {subsection}{\numberline {19.5.2}2. Adjust Tolerance Based on Problem Scale}{293}{subsection.19.5.2}%
\contentsline {section}{\numberline {19.6}Technical Details}{293}{section.19.6}%
\contentsline {subsection}{\numberline {19.6.1}Random Strategy Implementation}{293}{subsection.19.6.1}%
\contentsline {section}{\numberline {19.7}Summary}{294}{section.19.7}%
\contentsline {section}{\numberline {19.8}Jupyter Notebook}{294}{section.19.8}%
\contentsline {chapter}{\numberline {20}Point Selection Implementation in SpotOptim}{295}{chapter.20}%
\contentsline {section}{\numberline {20.1}Overview}{295}{section.20.1}%
\contentsline {section}{\numberline {20.2}Implementation Details}{295}{section.20.2}%
\contentsline {subsection}{\numberline {20.2.1}Parameters}{295}{subsection.20.2.1}%
\contentsline {subsection}{\numberline {20.2.2}Methods}{295}{subsection.20.2.2}%
\contentsline {section}{\numberline {20.3}Key Differences from spotpython}{296}{section.20.3}%
\contentsline {section}{\numberline {20.4}Example Usage}{296}{section.20.4}%
\contentsline {section}{\numberline {20.5}Benefits}{298}{section.20.5}%
\contentsline {section}{\numberline {20.6}Comparison with spotpython}{298}{section.20.6}%
\contentsline {section}{\numberline {20.7}Jupyter Notebook}{298}{section.20.7}%
\contentsline {part}{\numberline {VI}Optimization on the Surrogate}{299}{part.6}%
\contentsline {chapter}{\numberline {21}Optimization on the Surrogate}{301}{chapter.21}%
\contentsline {section}{\numberline {21.1}Key Arguments}{301}{section.21.1}%
\contentsline {section}{\numberline {21.2}Examples}{302}{section.21.2}%
\contentsline {subsection}{\numberline {21.2.1}1. Default Configuration (Differential Evolution)}{302}{subsection.21.2.1}%
\contentsline {subsection}{\numberline {21.2.2}2. Customizing Differential Evolution}{302}{subsection.21.2.2}%
\contentsline {subsection}{\numberline {21.2.3}3. Using Gradient-Based Optimization (L-BFGS-B)}{303}{subsection.21.2.3}%
\contentsline {subsection}{\numberline {21.2.4}4. Using Gradient-Free Optimization (Nelder-Mead)}{304}{subsection.21.2.4}%
\contentsline {subsection}{\numberline {21.2.5}5. Returning Multiple Candidates}{305}{subsection.21.2.5}%
\contentsline {part}{\numberline {VII}Kriging}{307}{part.7}%
\contentsline {chapter}{\numberline {22}Kriging Surrogate Integration}{309}{chapter.22}%
\contentsline {section}{\numberline {22.1}Overview}{309}{section.22.1}%
\contentsline {section}{\numberline {22.2}Module Structure}{309}{section.22.2}%
\contentsline {section}{\numberline {22.3}Kriging Class (\texttt {src/spotoptim/surrogate/kriging.py})}{309}{section.22.3}%
\contentsline {section}{\numberline {22.4}Integration with SpotOptim}{310}{section.22.4}%
\contentsline {section}{\numberline {22.5}Documentation}{310}{section.22.5}%
\contentsline {section}{\numberline {22.6}Usage Examples}{311}{section.22.6}%
\contentsline {subsection}{\numberline {22.6.1}Basic Usage}{311}{subsection.22.6.1}%
\contentsline {subsection}{\numberline {22.6.2}Custom Parameters}{311}{subsection.22.6.2}%
\contentsline {subsection}{\numberline {22.6.3}Prediction with Uncertainty}{312}{subsection.22.6.3}%
\contentsline {section}{\numberline {22.7}Technical Details}{313}{section.22.7}%
\contentsline {subsection}{\numberline {22.7.1}Kriging vs GaussianProcessRegressor}{313}{subsection.22.7.1}%
\contentsline {subsection}{\numberline {22.7.2}Algorithm}{313}{subsection.22.7.2}%
\contentsline {subsection}{\numberline {22.7.3}Key Arguments Passed from SpotOptim}{314}{subsection.22.7.3}%
\contentsline {section}{\numberline {22.8}Benefits}{314}{section.22.8}%
\contentsline {section}{\numberline {22.9}Future Enhancements}{314}{section.22.9}%
\contentsline {section}{\numberline {22.10}Jupyter Notebook}{315}{section.22.10}%
\contentsline {chapter}{\numberline {23}Kriging Surrogate Models in SpotOptim}{317}{chapter.23}%
\contentsline {section}{\numberline {23.1}Introduction}{317}{section.23.1}%
\contentsline {subsection}{\numberline {23.1.1}What is Kriging?}{317}{subsection.23.1.1}%
\contentsline {subsection}{\numberline {23.1.2}Why Use Kriging in SpotOptim?}{318}{subsection.23.1.2}%
\contentsline {section}{\numberline {23.2}Basic Usage}{318}{section.23.2}%
\contentsline {subsection}{\numberline {23.2.1}Creating a Simple Kriging Model}{318}{subsection.23.2.1}%
\contentsline {subsection}{\numberline {23.2.2}Default vs Custom Surrogate}{319}{subsection.23.2.2}%
\contentsline {section}{\numberline {23.3}Understanding Kriging Parameters}{320}{section.23.3}%
\contentsline {subsection}{\numberline {23.3.1}Method: Interpolation vs Regression}{320}{subsection.23.3.1}%
\contentsline {subsection}{\numberline {23.3.2}Noise Parameter and Regularization}{322}{subsection.23.3.2}%
\contentsline {subsection}{\numberline {23.3.3}Correlation Length Parameters (Theta)}{323}{subsection.23.3.3}%
\contentsline {subsection}{\numberline {23.3.4}Isotropic vs Anisotropic Correlation}{324}{subsection.23.3.4}%
\contentsline {section}{\numberline {23.4}Advanced Features}{325}{section.23.4}%
\contentsline {subsection}{\numberline {23.4.1}Mixed Variable Types}{325}{subsection.23.4.1}%
\contentsline {subsection}{\numberline {23.4.2}Customizing the Distance Metric for Factors}{328}{subsection.23.4.2}%
\contentsline {subsection}{\numberline {23.4.3}Handling High-Dimensional Problems}{329}{subsection.23.4.3}%
\contentsline {subsection}{\numberline {23.4.4}Uncertainty Quantification}{330}{subsection.23.4.4}%
\contentsline {section}{\numberline {23.5}Practical Examples}{332}{section.23.5}%
\contentsline {subsection}{\numberline {23.5.1}Example 1: Optimizing the Rastrigin Function}{332}{subsection.23.5.1}%
\contentsline {subsection}{\numberline {23.5.2}Example 2: Robust Optimization with Noise}{334}{subsection.23.5.2}%
\contentsline {subsection}{\numberline {23.5.3}Example 3: Real-World Machine Learning Hyperparameter Tuning}{335}{subsection.23.5.3}%
\contentsline {subsection}{\numberline {23.5.4}Example 4: Comparing Kriging Methods}{338}{subsection.23.5.4}%
\contentsline {subsection}{\numberline {23.5.5}Example 5: Sensitivity to Theta Bounds}{339}{subsection.23.5.5}%
\contentsline {section}{\numberline {23.6}Comparing Surrogates}{341}{section.23.6}%
\contentsline {subsection}{\numberline {23.6.1}Kriging vs Gaussian Process vs Random Forest}{341}{subsection.23.6.1}%
\contentsline {section}{\numberline {23.7}Best Practices}{342}{section.23.7}%
\contentsline {subsection}{\numberline {23.7.1}1. Choosing the Right Method}{342}{subsection.23.7.1}%
\contentsline {subsection}{\numberline {23.7.2}2. Setting Model Complexity}{343}{subsection.23.7.2}%
\contentsline {subsection}{\numberline {23.7.3}3. Handling Different Variable Types}{343}{subsection.23.7.3}%
\contentsline {subsection}{\numberline {23.7.4}4. Reproducibility}{344}{subsection.23.7.4}%
\contentsline {subsection}{\numberline {23.7.5}5. Monitoring Surrogate Quality}{344}{subsection.23.7.5}%
\contentsline {section}{\numberline {23.8}Common Issues and Solutions}{345}{section.23.8}%
\contentsline {subsection}{\numberline {23.8.1}Issue 1: Slow Fitting for Large Datasets}{345}{subsection.23.8.1}%
\contentsline {subsection}{\numberline {23.8.2}Issue 2: Poor Predictions for Categorical Variables}{345}{subsection.23.8.2}%
\contentsline {subsection}{\numberline {23.8.3}Issue 3: Numerical Instability}{346}{subsection.23.8.3}%
\contentsline {subsection}{\numberline {23.8.4}Issue 4: Overfitting to Noisy Data}{346}{subsection.23.8.4}%
\contentsline {section}{\numberline {23.9}Summary}{346}{section.23.9}%
\contentsline {subsection}{\numberline {23.9.1}Key Takeaways}{346}{subsection.23.9.1}%
\contentsline {subsection}{\numberline {23.9.2}Quick Reference}{347}{subsection.23.9.2}%
\contentsline {subsection}{\numberline {23.9.3}Parameter Cheat Sheet}{347}{subsection.23.9.3}%
\contentsline {section}{\numberline {23.10}Jupyter Notebook}{348}{section.23.10}%
\contentsline {chapter}{\numberline {24}Kriging in SpotOptim: The Forrester Implementation}{349}{chapter.24}%
\contentsline {chapter}{\numberline {25}Introduction}{351}{chapter.25}%
\contentsline {chapter}{\numberline {26}1. The Core Kriging Model (Section 2.4)}{353}{chapter.26}%
\contentsline {section}{\numberline {26.1}Correlation Function}{353}{section.26.1}%
\contentsline {section}{\numberline {26.2}Training: Concentrated Log-Likelihood}{353}{section.26.2}%
\contentsline {section}{\numberline {26.3}Prediction}{354}{section.26.3}%
\contentsline {chapter}{\numberline {27}2. Handling Noisy Data}{355}{chapter.27}%
\contentsline {section}{\numberline {27.1}A. Interpolation (\texttt {method="interpolation"})}{355}{section.27.1}%
\contentsline {section}{\numberline {27.2}B. Regression (\texttt {method="regression"})}{355}{section.27.2}%
\contentsline {section}{\numberline {27.3}C. Re-interpolation (\texttt {method="reinterpolation"})}{356}{section.27.3}%
\contentsline {chapter}{\numberline {28}3. Examples}{357}{chapter.28}%
\contentsline {section}{\numberline {28.1}Setup}{357}{section.28.1}%
\contentsline {section}{\numberline {28.2}Comparisons}{357}{section.28.2}%
\contentsline {section}{\numberline {28.3}Visualization}{358}{section.28.3}%
\contentsline {subsection}{\numberline {28.3.1}Interpretation}{359}{subsection.28.3.1}%
\contentsline {chapter}{\numberline {29}4. Infill Criteria}{361}{chapter.29}%
\contentsline {chapter}{\numberline {30}References}{363}{chapter.30}%
\contentsline {section}{\numberline {30.1}Jupyter Notebook}{363}{section.30.1}%
\contentsline {chapter}{\numberline {31}Nystroem Kernel Approximation}{365}{chapter.31}%
\contentsline {section}{\numberline {31.1}Introduction}{365}{section.31.1}%
\contentsline {section}{\numberline {31.2}The Nystroem Method}{365}{section.31.2}%
\contentsline {subsection}{\numberline {31.2.1}Moderate \(n\), Large \(k\) Use Case}{366}{subsection.31.2.1}%
\contentsline {section}{\numberline {31.3}Implementation in SpotOptim}{366}{section.31.3}%
\contentsline {subsection}{\numberline {31.3.1}Components}{366}{subsection.31.3.1}%
\contentsline {subsection}{\numberline {31.3.2}Example Usage}{366}{subsection.31.3.2}%
\contentsline {section}{\numberline {31.4}Handling Mixed Variables}{368}{section.31.4}%
\contentsline {subsection}{\numberline {31.4.1}Using \texttt {SpotOptimKernel}}{368}{subsection.31.4.1}%
\contentsline {section}{\numberline {31.5}Jupyter Notebook}{369}{section.31.5}%
\contentsline {part}{\numberline {VIII}Multiobjective Optimization}{371}{part.8}%
\contentsline {chapter}{\numberline {32}Multi-Objective Optimization Support in SpotOptim}{373}{chapter.32}%
\contentsline {section}{\numberline {32.1}Overview}{374}{section.32.1}%
\contentsline {section}{\numberline {32.2}Common Scalarization Strategies}{378}{section.32.2}%
\contentsline {subsection}{\numberline {32.2.1}1. Weighted Sum}{378}{subsection.32.2.1}%
\contentsline {subsection}{\numberline {32.2.2}2. Weighted Sum with Normalization}{378}{subsection.32.2.2}%
\contentsline {subsection}{\numberline {32.2.3}3. Min-Max (Chebyshev)}{378}{subsection.32.2.3}%
\contentsline {subsection}{\numberline {32.2.4}4. Target Achievement}{379}{subsection.32.2.4}%
\contentsline {subsection}{\numberline {32.2.5}5. Product}{379}{subsection.32.2.5}%
\contentsline {section}{\numberline {32.3}Integration with Other Features}{379}{section.32.3}%
\contentsline {section}{\numberline {32.4}Implementation Details}{379}{section.32.4}%
\contentsline {subsection}{\numberline {32.4.1}Automatic Detection}{379}{subsection.32.4.1}%
\contentsline {subsection}{\numberline {32.4.2}Data Flow}{380}{subsection.32.4.2}%
\contentsline {subsection}{\numberline {32.4.3}Backward Compatibility}{380}{subsection.32.4.3}%
\contentsline {section}{\numberline {32.5}Limitations and Notes}{380}{section.32.5}%
\contentsline {subsection}{\numberline {32.5.1}What This Is}{380}{subsection.32.5.1}%
\contentsline {subsection}{\numberline {32.5.2}What This Is Not}{380}{subsection.32.5.2}%
\contentsline {subsection}{\numberline {32.5.3}For True Multi-Objective Optimization}{381}{subsection.32.5.3}%
\contentsline {section}{\numberline {32.6}Benchmark Multi-Objective Test Functions}{381}{section.32.6}%
\contentsline {subsection}{\numberline {32.6.1}Function Overview}{381}{subsection.32.6.1}%
\contentsline {subsection}{\numberline {32.6.2}Example 6: ZDT1 - Convex Pareto Front}{382}{subsection.32.6.2}%
\contentsline {subsection}{\numberline {32.6.3}Example 7: ZDT2 - Non-Convex Pareto Front}{384}{subsection.32.6.3}%
\contentsline {subsection}{\numberline {32.6.4}Example 8: ZDT3 - Disconnected Pareto Front}{386}{subsection.32.6.4}%
\contentsline {subsection}{\numberline {32.6.5}Example 9: ZDT4 - Multimodal with Many Local Fronts}{387}{subsection.32.6.5}%
\contentsline {subsection}{\numberline {32.6.6}Example 10: ZDT6 - Non-Uniform Density}{389}{subsection.32.6.6}%
\contentsline {subsection}{\numberline {32.6.7}Example 11: DTLZ2 - Scalable Spherical Pareto Front}{391}{subsection.32.6.7}%
\contentsline {subsection}{\numberline {32.6.8}Example 12: Schaffer N1 - Simple Bi-Objective}{393}{subsection.32.6.8}%
\contentsline {subsection}{\numberline {32.6.9}Example 13: Fonseca-Fleming - Symmetric Concave Front}{395}{subsection.32.6.9}%
\contentsline {section}{\numberline {32.7}Example 14: Kursawe - Non-Convex Disconnected Front}{397}{section.32.7}%
\contentsline {subsection}{\numberline {32.7.1}Comparison: Different Scalarization Strategies on ZDT1}{398}{subsection.32.7.1}%
\contentsline {section}{\numberline {32.8}A Simple Visualization Example}{400}{section.32.8}%
\contentsline {section}{\numberline {32.9}Jupyter Notebook}{407}{section.32.9}%
\contentsline {part}{\numberline {IX}Hyperparameter Tuning}{409}{part.9}%
\contentsline {chapter}{\numberline {33}The TorchObjective Class for Hyperparameter Tuning}{411}{chapter.33}%
\contentsline {section}{\numberline {33.1}Overview}{411}{section.33.1}%
\contentsline {section}{\numberline {33.2}Workflow Steps}{411}{section.33.2}%
\contentsline {section}{\numberline {33.3}Step-by-Step Example}{412}{section.33.3}%
\contentsline {subsection}{\numberline {33.3.1}1. Define the Model}{412}{subsection.33.3.1}%
\contentsline {subsection}{\numberline {33.3.2}2. Prepare the Data}{412}{subsection.33.3.2}%
\contentsline {subsection}{\numberline {33.3.3}3. Define Hyperparameters}{413}{subsection.33.3.3}%
\contentsline {subsection}{\numberline {33.3.4}4. Create Experiment Control}{414}{subsection.33.3.4}%
\contentsline {subsection}{\numberline {33.3.5}5. Initialize TorchObjective}{415}{subsection.33.3.5}%
\contentsline {subsection}{\numberline {33.3.6}6. Run Optimization}{415}{subsection.33.3.6}%
\contentsline {section}{\numberline {33.4}Advanced Features}{416}{section.33.4}%
\contentsline {subsection}{\numberline {33.4.1}Tuning Batch Size}{416}{subsection.33.4.1}%
\contentsline {subsection}{\numberline {33.4.2}Tuning Architecture}{416}{subsection.33.4.2}%
\contentsline {subsection}{\numberline {33.4.3}Log-Scale Transformations}{418}{subsection.33.4.3}%
\contentsline {section}{\numberline {33.5}Custom Models}{418}{section.33.5}%
\contentsline {chapter}{\numberline {34}The MLP Class}{421}{chapter.34}%
\contentsline {section}{\numberline {34.1}Overview}{421}{section.34.1}%
\contentsline {section}{\numberline {34.2}Basic Usage}{421}{section.34.2}%
\contentsline {subsection}{\numberline {34.2.1}Initialization}{421}{subsection.34.2.1}%
\contentsline {subsection}{\numberline {34.2.2}Implicit Architecture (Width \& Depth)}{422}{subsection.34.2.2}%
\contentsline {section}{\numberline {34.3}Configuration Options}{423}{section.34.3}%
\contentsline {section}{\numberline {34.4}Optimizer Integration}{423}{section.34.4}%
\contentsline {section}{\numberline {34.5}Hyperparameter Tuning}{424}{section.34.5}%
\contentsline {subsection}{\numberline {34.5.1}Example: Tuning with SpotOptim}{425}{subsection.34.5.1}%
\contentsline {chapter}{\numberline {35}Physics-Informed Neural Networks (PINNs) Demo 1}{427}{chapter.35}%
\contentsline {chapter}{\numberline {36}Overview}{429}{chapter.36}%
\contentsline {section}{\numberline {36.1}The Differential Equation}{429}{section.36.1}%
\contentsline {chapter}{\numberline {37}Setup}{431}{chapter.37}%
\contentsline {chapter}{\numberline {38}The Neural Network}{433}{chapter.38}%
\contentsline {chapter}{\numberline {39}Generate Training Data}{435}{chapter.39}%
\contentsline {chapter}{\numberline {40}Collocation Points}{439}{chapter.40}%
\contentsline {chapter}{\numberline {41}PINN Training}{441}{chapter.41}%
\contentsline {section}{\numberline {41.1}Loss Components Explained}{441}{section.41.1}%
\contentsline {subsection}{\numberline {41.1.1}Neural Network Prediction on Data Points}{441}{subsection.41.1.1}%
\contentsline {subsection}{\numberline {41.1.2}Data Loss (Loss1)}{441}{subsection.41.1.2}%
\contentsline {subsection}{\numberline {41.1.3}Neural Network Prediction on Collocation Points}{441}{subsection.41.1.3}%
\contentsline {subsection}{\numberline {41.1.4}Computing Derivatives for the ODE}{441}{subsection.41.1.4}%
\contentsline {subsection}{\numberline {41.1.5}Physics Loss (Loss2)}{442}{subsection.41.1.5}%
\contentsline {subsection}{\numberline {41.1.6}Total Loss}{442}{subsection.41.1.6}%
\contentsline {section}{\numberline {41.2}Training Loop}{442}{section.41.2}%
\contentsline {chapter}{\numberline {42}Results Visualization}{445}{chapter.42}%
\contentsline {section}{\numberline {42.1}Training Progress}{445}{section.42.1}%
\contentsline {section}{\numberline {42.2}Solution Evolution During Training}{446}{section.42.2}%
\contentsline {section}{\numberline {42.3}Final Solution Comparison}{448}{section.42.3}%
\contentsline {section}{\numberline {42.4}Error Analysis}{450}{section.42.4}%
\contentsline {chapter}{\numberline {43}Summary}{453}{chapter.43}%
\contentsline {section}{\numberline {43.1}Key Advantages of PINNs}{453}{section.43.1}%
\contentsline {section}{\numberline {43.2}Using SpotOptim for Hyperparameter Optimization}{453}{section.43.2}%
\contentsline {section}{\numberline {43.3}Jupyter Notebook}{454}{section.43.3}%
\contentsline {chapter}{\numberline {44}Hyperparameter Tuning for Physics-Informed Neural Networks}{455}{chapter.44}%
\contentsline {chapter}{\numberline {45}Overview}{457}{chapter.45}%
\contentsline {section}{\numberline {45.1}Key Features}{457}{section.45.1}%
\contentsline {subsection}{\numberline {45.1.1}1. PyTorch Dataset and DataLoader}{457}{subsection.45.1.1}%
\contentsline {subsection}{\numberline {45.1.2}2. Automatic Transformation Handling}{458}{subsection.45.1.2}%
\contentsline {chapter}{\numberline {46}The Problem}{459}{chapter.46}%
\contentsline {chapter}{\numberline {47}Setup}{461}{chapter.47}%
\contentsline {chapter}{\numberline {48}Data Generation}{463}{chapter.48}%
\contentsline {section}{\numberline {48.1}Custom Dataset Classes}{463}{section.48.1}%
\contentsline {chapter}{\numberline {49}Define the PINN Training Function}{467}{chapter.49}%
\contentsline {chapter}{\numberline {50}Hyperparameter Optimization with SpotOptim}{471}{chapter.50}%
\contentsline {section}{\numberline {50.1}Define the Objective Function}{471}{section.50.1}%
\contentsline {section}{\numberline {50.2}Run the Optimization}{473}{section.50.2}%
\contentsline {chapter}{\numberline {51}Results Analysis}{477}{chapter.51}%
\contentsline {section}{\numberline {51.1}Best Configuration}{477}{section.51.1}%
\contentsline {subsection}{\numberline {51.1.1}Results Table with Importance Scores}{478}{subsection.51.1.1}%
\contentsline {section}{\numberline {51.2}Optimization History}{478}{section.51.2}%
\contentsline {section}{\numberline {51.3}Surrogate Visualization}{479}{section.51.3}%
\contentsline {section}{\numberline {51.4}Parameter Distribution Analysis}{481}{section.51.4}%
\contentsline {chapter}{\numberline {52}Train Final Model with Best Hyperparameters}{483}{chapter.52}%
\contentsline {chapter}{\numberline {53}Evaluate Final Model}{487}{chapter.53}%
\contentsline {chapter}{\numberline {54}Visualize Final Solution}{489}{chapter.54}%
\contentsline {chapter}{\numberline {55}Comparison with Baseline}{491}{chapter.55}%
\contentsline {chapter}{\numberline {56}Hyperparameter Sensitivity Analysis}{495}{chapter.56}%
\contentsline {section}{\numberline {56.1}Sensitivity Analysis (Spearman Correlation)}{496}{section.56.1}%
\contentsline {chapter}{\numberline {57}Summary}{497}{chapter.57}%
\contentsline {section}{\numberline {57.1}Key Findings}{497}{section.57.1}%
\contentsline {section}{\numberline {57.2}Recommendations}{498}{section.57.2}%
\contentsline {section}{\numberline {57.3}Using These Results}{499}{section.57.3}%
\contentsline {section}{\numberline {57.4}Using var\_trans for Your Hyperparameter Optimization}{500}{section.57.4}%
\contentsline {section}{\numberline {57.5}Future Directions}{501}{section.57.5}%
\contentsline {section}{\numberline {57.6}Jupyter Notebook}{501}{section.57.6}%
\contentsline {chapter}{\numberline {58}Learning Rate Mapping for Unified Optimizer Interface}{503}{chapter.58}%
\contentsline {section}{\numberline {58.1}Overview}{503}{section.58.1}%
\contentsline {section}{\numberline {58.2}Quick Start}{503}{section.58.2}%
\contentsline {subsection}{\numberline {58.2.1}Basic Usage}{503}{subsection.58.2.1}%
\contentsline {subsection}{\numberline {58.2.2}Scaling Learning Rates}{504}{subsection.58.2.2}%
\contentsline {subsection}{\numberline {58.2.3}Integration with LinearRegressor}{505}{subsection.58.2.3}%
\contentsline {section}{\numberline {58.3}Function Reference}{505}{section.58.3}%
\contentsline {subsection}{\numberline {58.3.1}\texttt {map\_lr(lr\_unified,\ optimizer\_name,\ use\_default\_scale=True)}}{505}{subsection.58.3.1}%
\contentsline {section}{\numberline {58.4}Supported Optimizers}{506}{section.58.4}%
\contentsline {section}{\numberline {58.5}Use Cases}{507}{section.58.5}%
\contentsline {subsection}{\numberline {58.5.1}Comparing Different Optimizers}{507}{subsection.58.5.1}%
\contentsline {subsection}{\numberline {58.5.2}Hyperparameter Optimization}{508}{subsection.58.5.2}%
\contentsline {subsection}{\numberline {58.5.3}Hyperparameter Optimization with SpotOptim}{510}{subsection.58.5.3}%
\contentsline {subsection}{\numberline {58.5.4}Log-Scale Hyperparameter Search}{512}{subsection.58.5.4}%
\contentsline {subsection}{\numberline {58.5.5}Custom Learning Rate Schedules}{513}{subsection.58.5.5}%
\contentsline {subsection}{\numberline {58.5.6}Direct Usage Without LinearRegressor}{514}{subsection.58.5.6}%
\contentsline {section}{\numberline {58.6}Best Practices}{515}{section.58.6}%
\contentsline {subsection}{\numberline {58.6.1}Choosing Unified Learning Rate}{515}{subsection.58.6.1}%
\contentsline {subsection}{\numberline {58.6.2}Optimizer Selection Guidelines}{515}{subsection.58.6.2}%
\contentsline {subsection}{\numberline {58.6.3}Common Patterns}{516}{subsection.58.6.3}%
\contentsline {section}{\numberline {58.7}Troubleshooting}{516}{section.58.7}%
\contentsline {subsection}{\numberline {58.7.1}Issue: Training is unstable (loss explodes)}{516}{subsection.58.7.1}%
\contentsline {subsection}{\numberline {58.7.2}Issue: Training is too slow (loss decreases very slowly)}{517}{subsection.58.7.2}%
\contentsline {subsection}{\numberline {58.7.3}Issue: Different results across optimizer runs}{517}{subsection.58.7.3}%
\contentsline {subsection}{\numberline {58.7.4}Issue: Want to use raw learning rate without mapping}{517}{subsection.58.7.4}%
\contentsline {subsection}{\numberline {58.7.5}Issue: Optimizer not supported}{517}{subsection.58.7.5}%
\contentsline {section}{\numberline {58.8}Technical Details}{517}{section.58.8}%
\contentsline {subsection}{\numberline {58.8.1}How It Works}{517}{subsection.58.8.1}%
\contentsline {subsection}{\numberline {58.8.2}Design Rationale}{518}{subsection.58.8.2}%
\contentsline {subsection}{\numberline {58.8.3}Default Learning Rates}{518}{subsection.58.8.3}%
\contentsline {section}{\numberline {58.9}Examples}{519}{section.58.9}%
\contentsline {subsection}{\numberline {58.9.1}Complete Example: Optimizer Comparison Study}{519}{subsection.58.9.1}%
\contentsline {section}{\numberline {58.10}Jupyter Notebook}{522}{section.58.10}%
\contentsline {chapter}{\numberline {59}SpotOptim Scaling}{523}{chapter.59}%
\contentsline {chapter}{\numberline {60}TorchStandardScaler in SpotOptim}{525}{chapter.60}%
\contentsline {section}{\numberline {60.1}Introduction}{525}{section.60.1}%
\contentsline {section}{\numberline {60.2}Using TorchStandardScaler Directly}{525}{section.60.2}%
\contentsline {section}{\numberline {60.3}Integrating with TorchObjective}{526}{section.60.3}%
\contentsline {subsection}{\numberline {60.3.1}Example: Scaling Effect}{526}{subsection.60.3.1}%
\contentsline {subsection}{\numberline {60.3.2}Explanation}{528}{subsection.60.3.2}%
\contentsline {part}{\numberline {X}Data Sets}{531}{part.10}%
\contentsline {chapter}{\numberline {61}Diabetes Dataset Utilities}{533}{chapter.61}%
\contentsline {section}{\numberline {61.1}Overview}{533}{section.61.1}%
\contentsline {section}{\numberline {61.2}Quick Start}{533}{section.61.2}%
\contentsline {subsection}{\numberline {61.2.1}Basic Usage}{533}{subsection.61.2.1}%
\contentsline {subsection}{\numberline {61.2.2}Training a Model}{534}{subsection.61.2.2}%
\contentsline {section}{\numberline {61.3}Function Reference}{536}{section.61.3}%
\contentsline {subsection}{\numberline {61.3.1}get\_diabetes\_dataloaders()}{536}{subsection.61.3.1}%
\contentsline {section}{\numberline {61.4}DiabetesDataset Class}{537}{section.61.4}%
\contentsline {subsection}{\numberline {61.4.1}Manual Dataset Creation}{538}{subsection.61.4.1}%
\contentsline {section}{\numberline {61.5}Advanced Usage}{539}{section.61.5}%
\contentsline {subsection}{\numberline {61.5.1}Custom Transforms}{539}{subsection.61.5.1}%
\contentsline {subsection}{\numberline {61.5.2}Different Train/Test Splits}{540}{subsection.61.5.2}%
\contentsline {subsection}{\numberline {61.5.3}Without Feature Scaling}{540}{subsection.61.5.3}%
\contentsline {subsection}{\numberline {61.5.4}Larger Batch Sizes}{541}{subsection.61.5.4}%
\contentsline {subsection}{\numberline {61.5.5}GPU Training with Pin Memory}{541}{subsection.61.5.5}%
\contentsline {section}{\numberline {61.6}Complete Training Example}{542}{section.61.6}%
\contentsline {section}{\numberline {61.7}Integration with SpotOptim}{545}{section.61.7}%
\contentsline {section}{\numberline {61.8}Best Practices}{547}{section.61.8}%
\contentsline {subsection}{\numberline {61.8.1}1. Always Use Feature Scaling}{547}{subsection.61.8.1}%
\contentsline {subsection}{\numberline {61.8.2}2. Set Random Seeds for Reproducibility}{548}{subsection.61.8.2}%
\contentsline {subsection}{\numberline {61.8.3}3. Don't Shuffle Test Data}{548}{subsection.61.8.3}%
\contentsline {subsection}{\numberline {61.8.4}4. Choose Appropriate Batch Size}{548}{subsection.61.8.4}%
\contentsline {subsection}{\numberline {61.8.5}5. Save the Scaler for Production}{549}{subsection.61.8.5}%
\contentsline {section}{\numberline {61.9}Troubleshooting}{549}{section.61.9}%
\contentsline {subsection}{\numberline {61.9.1}Issue: Out of Memory}{549}{subsection.61.9.1}%
\contentsline {subsection}{\numberline {61.9.2}Issue: Different Data Ranges}{550}{subsection.61.9.2}%
\contentsline {subsection}{\numberline {61.9.3}Issue: Non-Reproducible Results}{550}{subsection.61.9.3}%
\contentsline {subsection}{\numberline {61.9.4}Issue: Slow Data Loading}{550}{subsection.61.9.4}%
\contentsline {section}{\numberline {61.10}Summary}{551}{section.61.10}%
\contentsline {section}{\numberline {61.11}Jupyter Notebook}{551}{section.61.11}%
\contentsline {part}{\numberline {XI}Details About SPOT}{553}{part.11}%
\contentsline {chapter}{\numberline {62}SpotOptim Step-by-Step Optimization Process}{555}{chapter.62}%
\contentsline {chapter}{\numberline {63}Introduction}{557}{chapter.63}%
\contentsline {chapter}{\numberline {64}Setup and Test Functions}{559}{chapter.64}%
\contentsline {chapter}{\numberline {65}Standard Optimization Workflow}{561}{chapter.65}%
\contentsline {section}{\numberline {65.1}Phase: Initialization and Setup}{561}{section.65.1}%
\contentsline {section}{\numberline {65.2}Phase: Initial Design Generation}{562}{section.65.2}%
\contentsline {subsection}{\numberline {65.2.1}Method: \texttt {get\_initial\_design()}}{562}{subsection.65.2.1}%
\contentsline {section}{\numberline {65.3}Phase 3: Initial Design Curation}{565}{section.65.3}%
\contentsline {subsection}{\numberline {65.3.1}Method: \texttt {curate\_initial\_design()}}{565}{subsection.65.3.1}%
\contentsline {section}{\numberline {65.4}Phase 4: Initial Design Evaluation}{565}{section.65.4}%
\contentsline {subsection}{\numberline {65.4.1}Method: \texttt {evaluate\_function()}}{565}{subsection.65.4.1}%
\contentsline {section}{\numberline {65.5}Phase 5: Handling Failed Evaluations}{566}{section.65.5}%
\contentsline {subsection}{\numberline {65.5.1}Method: \texttt {rm\_initial\_design\_NA\_values(X0,y0)}}{566}{subsection.65.5.1}%
\contentsline {section}{\numberline {65.6}Phase 6: Validation Check}{567}{section.65.6}%
\contentsline {subsection}{\numberline {65.6.1}Method: \texttt {check\_size\_initial\_design(y0,\ n\_evaluated)}}{567}{subsection.65.6.1}%
\contentsline {section}{\numberline {65.7}Phase 7: Storage Initialization}{567}{section.65.7}%
\contentsline {subsection}{\numberline {65.7.1}Method: \texttt {init\_storage()}}{567}{subsection.65.7.1}%
\contentsline {section}{\numberline {65.8}Phase: Statistics Update}{568}{section.65.8}%
\contentsline {subsection}{\numberline {65.8.1}Method: \texttt {update\_stats()}}{568}{subsection.65.8.1}%
\contentsline {section}{\numberline {65.9}Phase: Log initial Design to TensorBoard}{568}{section.65.9}%
\contentsline {subsection}{\numberline {65.9.1}Method: \texttt {\_log\_initial\_design\_tensorboard()}}{568}{subsection.65.9.1}%
\contentsline {section}{\numberline {65.10}Phase: Initial Best Point}{569}{section.65.10}%
\contentsline {subsection}{\numberline {65.10.1}Method: \texttt {get\_best\_xy\_initial\_design()}}{569}{subsection.65.10.1}%
\contentsline {section}{\numberline {65.11}Illustration of the Initial Design Phase Results}{569}{section.65.11}%
\contentsline {chapter}{\numberline {66}Sequential Optimization Loop}{571}{chapter.66}%
\contentsline {section}{\numberline {66.1}Step: Surrogate Model Fitting}{571}{section.66.1}%
\contentsline {subsection}{\numberline {66.1.1}Method: \texttt {fit\_scheduler()}}{571}{subsection.66.1.1}%
\contentsline {subsection}{\numberline {66.1.2}Method: \texttt {fit\_surrogate()}}{571}{subsection.66.1.2}%
\contentsline {subsection}{\numberline {66.1.3}Method: \texttt {fit\_selection\_dispatcher()}}{572}{subsection.66.1.3}%
\contentsline {subsection}{\numberline {66.1.4}Step: Apply OCBA}{573}{subsection.66.1.4}%
\contentsline {subsection}{\numberline {66.1.5}Method: \texttt {apply\_ocba()}}{573}{subsection.66.1.5}%
\contentsline {section}{\numberline {66.2}Step: Predict with Uncertainty}{574}{section.66.2}%
\contentsline {subsection}{\numberline {66.2.1}Method: \texttt {\_predict\_with\_uncertainty()}}{574}{subsection.66.2.1}%
\contentsline {section}{\numberline {66.3}Step: Next Point Suggestion}{574}{section.66.3}%
\contentsline {subsection}{\numberline {66.3.1}Method: \texttt {suggest\_next\_infill\_point()}}{574}{subsection.66.3.1}%
\contentsline {section}{\numberline {66.4}Step: Acquisition Function Evaluation}{575}{section.66.4}%
\contentsline {subsection}{\numberline {66.4.1}Method: \texttt {\_acquisition\_function()}}{575}{subsection.66.4.1}%
\contentsline {section}{\numberline {66.5}Step: Update Repeats for Infill Points}{576}{section.66.5}%
\contentsline {subsection}{\numberline {66.5.1}Method: \texttt {update\_repeats\_infill\_points()}}{576}{subsection.66.5.1}%
\contentsline {section}{\numberline {66.6}Append OCBA Points to Infill Points}{577}{section.66.6}%
\contentsline {section}{\numberline {66.7}Step: Evaluation of New Points}{577}{section.66.7}%
\contentsline {subsection}{\numberline {66.7.1}Method: \texttt {evaluate\_function()} (again)}{577}{subsection.66.7.1}%
\contentsline {section}{\numberline {66.8}Step: Handle Failed Evaluations (Sequential)}{577}{section.66.8}%
\contentsline {subsection}{\numberline {66.8.1}Method: \texttt {\_handle\_NA\_new\_points()}}{577}{subsection.66.8.1}%
\contentsline {section}{\numberline {66.9}Step: Update Success Rate}{578}{section.66.9}%
\contentsline {subsection}{\numberline {66.9.1}Method: \texttt {update\_success\_rate(y0)}}{578}{subsection.66.9.1}%
\contentsline {section}{\numberline {66.10}Step: Update Storage}{579}{section.66.10}%
\contentsline {subsection}{\numberline {66.10.1}Internal updates}{579}{subsection.66.10.1}%
\contentsline {section}{\numberline {66.11}Update Statistics}{579}{section.66.11}%
\contentsline {subsection}{\numberline {66.11.1}What happens in \texttt {update\_stats()}:}{579}{subsection.66.11.1}%
\contentsline {section}{\numberline {66.12}Step: Update Best Solution}{580}{section.66.12}%
\contentsline {subsection}{\numberline {66.12.1}Method: \texttt {\_update\_best\_main\_loop()}}{580}{subsection.66.12.1}%
\contentsline {chapter}{\numberline {67}Complete Optimization Example}{581}{chapter.67}%
\contentsline {chapter}{\numberline {68}Noisy Functions with Repeats}{583}{chapter.68}%
\contentsline {section}{\numberline {68.1}Configuration for Noisy Functions}{583}{section.68.1}%
\contentsline {section}{\numberline {68.2}Noisy Optimization Workflow Differences}{584}{section.68.2}%
\contentsline {subsection}{\numberline {68.2.1}Modified Initial Design}{584}{subsection.68.2.1}%
\contentsline {subsection}{\numberline {68.2.2}Update Statistics Method}{585}{subsection.68.2.2}%
\contentsline {chapter}{\numberline {69}Optimal Computing Budget Allocation (OCBA)}{587}{chapter.69}%
\contentsline {section}{\numberline {69.1}OCBA Configuration}{587}{section.69.1}%
\contentsline {section}{\numberline {69.2}OCBA Method}{588}{section.69.2}%
\contentsline {subsection}{\numberline {69.2.1}Method: \texttt {apply\_ocba()}}{588}{subsection.69.2.1}%
\contentsline {chapter}{\numberline {70}Handling Function Evaluation Failures}{591}{chapter.70}%
\contentsline {section}{\numberline {70.1}Example with Failures}{591}{section.70.1}%
\contentsline {section}{\numberline {70.2}Failure Handling in Initial Design}{592}{section.70.2}%
\contentsline {subsection}{\numberline {70.2.1}Method: \texttt {rm\_initial\_design\_NA\_values()}}{592}{subsection.70.2.1}%
\contentsline {section}{\numberline {70.3}Failure Handling in Sequential Phase}{593}{section.70.3}%
\contentsline {subsection}{\numberline {70.3.1}Method: \texttt {\_handle\_NA\_new\_points()}}{593}{subsection.70.3.1}%
\contentsline {section}{\numberline {70.4}Penalty Application}{593}{section.70.4}%
\contentsline {subsection}{\numberline {70.4.1}Method: \texttt {apply\_penalty\_NA()}}{593}{subsection.70.4.1}%
\contentsline {chapter}{\numberline {71}Complete Method Summary}{595}{chapter.71}%
\contentsline {section}{\numberline {71.1}Methods Called During \texttt {optimize()}}{595}{section.71.1}%
\contentsline {subsection}{\numberline {71.1.1}Preparation Phase (inside \texttt {optimize\_sequential\_run()} via \texttt {\_initialize\_run()})}{595}{subsection.71.1.1}%
\contentsline {subsection}{\numberline {71.1.2}Sequential Optimization Loop (each iteration inside \texttt {\_run\_sequential\_loop()})}{595}{subsection.71.1.2}%
\contentsline {subsection}{\numberline {71.1.3}Finalization}{596}{subsection.71.1.3}%
\contentsline {section}{\numberline {71.2}Helper Methods Used}{596}{section.71.2}%
\contentsline {chapter}{\numberline {72}Termination Conditions}{597}{chapter.72}%
\contentsline {section}{\numberline {72.1}Method: \texttt {determine\_termination()}}{597}{section.72.1}%
\contentsline {chapter}{\numberline {73}Performance Comparison on Noisy Functions}{599}{chapter.73}%
\contentsline {chapter}{\numberline {74}Summary}{603}{chapter.74}%
\contentsline {section}{\numberline {74.1}Complete Workflow Diagram}{603}{section.74.1}%
\contentsline {section}{\numberline {74.2}Key Concepts}{605}{section.74.2}%
\contentsline {subsection}{\numberline {74.2.1}1. Initial Design}{605}{subsection.74.2.1}%
\contentsline {subsection}{\numberline {74.2.2}2. Surrogate Model}{605}{subsection.74.2.2}%
\contentsline {subsection}{\numberline {74.2.3}3. Acquisition Function}{605}{subsection.74.2.3}%
\contentsline {subsection}{\numberline {74.2.4}4. Noise Handling}{606}{subsection.74.2.4}%
\contentsline {subsection}{\numberline {74.2.5}5. Failure Handling}{606}{subsection.74.2.5}%
\contentsline {section}{\numberline {74.3}Best Practices}{606}{section.74.3}%
\contentsline {section}{\numberline {74.4}Conclusion}{607}{section.74.4}%
\contentsline {section}{\numberline {74.5}Jupyter Notebook}{607}{section.74.5}%
\contentsline {chapter}{\numberline {75}SpotOptim Execution Flow}{609}{chapter.75}%
\contentsline {chapter}{\numberline {76}SpotOptim Execution Flow}{611}{chapter.76}%
\contentsline {section}{\numberline {76.1}Example Without Restarts}{611}{section.76.1}%
\contentsline {subsection}{\numberline {76.1.1}\texttt {SpotOptim.\_\_init\_\_}}{611}{subsection.76.1.1}%
\contentsline {subsection}{\numberline {76.1.2}\texttt {SpotOptim.optimize}}{612}{subsection.76.1.2}%
\contentsline {subsection}{\numberline {76.1.3}\texttt {SpotOptim.\_optimize\_single\_run}}{612}{subsection.76.1.3}%
\contentsline {subsection}{\numberline {76.1.4}\texttt {SpotOptim.get\_initial\_design}}{612}{subsection.76.1.4}%
\contentsline {subsection}{\numberline {76.1.5}\texttt {SpotOptim.curate\_initial\_design}}{612}{subsection.76.1.5}%
\contentsline {subsection}{\numberline {76.1.6}\texttt {SpotOptim.evaluate\_function} (Initial Design)}{613}{subsection.76.1.6}%
\contentsline {subsection}{\numberline {76.1.7}\texttt {SpotOptim.init\_storage}}{613}{subsection.76.1.7}%
\contentsline {subsection}{\numberline {76.1.8}Optimization Loop Start}{613}{subsection.76.1.8}%
\contentsline {subsection}{\numberline {76.1.9}\texttt {SpotOptim.\_fit\_scheduler}}{613}{subsection.76.1.9}%
\contentsline {subsection}{\numberline {76.1.10}\texttt {SpotOptim.suggest\_next\_infill\_point}}{613}{subsection.76.1.10}%
\contentsline {subsection}{\numberline {76.1.11}\texttt {SpotOptim.\_try\_optimizer\_candidates}}{614}{subsection.76.1.11}%
\contentsline {subsection}{\numberline {76.1.12}\texttt {SpotOptim.optimize\_acquisition\_func}}{614}{subsection.76.1.12}%
\contentsline {subsection}{\numberline {76.1.13}\texttt {SpotOptim.evaluate\_function} (Loop)}{614}{subsection.76.1.13}%
\contentsline {subsection}{\numberline {76.1.14}\texttt {SpotOptim.update\_storage}}{614}{subsection.76.1.14}%
\contentsline {subsection}{\numberline {76.1.15}\texttt {SpotOptim.\_update\_best\_main\_loop}}{615}{subsection.76.1.15}%
\contentsline {subsection}{\numberline {76.1.16}Termination}{615}{subsection.76.1.16}%
\contentsline {section}{\numberline {76.2}Example With Restarts}{615}{section.76.2}%
\contentsline {subsection}{\numberline {76.2.1}\texttt {SpotOptim.optimize} (Global Loop)}{615}{subsection.76.2.1}%
\contentsline {subsection}{\numberline {76.2.2}\texttt {SpotOptim.\_optimize\_single\_run} (First Run)}{616}{subsection.76.2.2}%
\contentsline {subsection}{\numberline {76.2.3}Restart Injection (Between Runs)}{616}{subsection.76.2.3}%
\contentsline {subsection}{\numberline {76.2.4}\texttt {SpotOptim.\_optimize\_single\_run} (Subsequent Runs)}{616}{subsection.76.2.4}%
\contentsline {subsection}{\numberline {76.2.5}Termination}{617}{subsection.76.2.5}%
\contentsline {section}{\numberline {76.3}Description Based on Chunks}{617}{section.76.3}%
\contentsline {subsection}{\numberline {76.3.1}Configuration \& Helpers}{617}{subsection.76.3.1}%
\contentsline {subsubsection}{\numberline {76.3.1.1}\texttt {set\_seed}}{617}{subsubsection.76.3.1.1}%
\contentsline {subsubsection}{\numberline {76.3.1.2}\texttt {detect\_var\_type}}{617}{subsubsection.76.3.1.2}%
\contentsline {subsubsection}{\numberline {76.3.1.3}\texttt {modify\_bounds\_based\_on\_var\_type}}{617}{subsubsection.76.3.1.3}%
\contentsline {subsubsection}{\numberline {76.3.1.4}\texttt {process\_factor\_bounds}}{618}{subsubsection.76.3.1.4}%
\contentsline {subsubsection}{\numberline {76.3.1.5}\texttt {repair\_non\_numeric}}{618}{subsubsection.76.3.1.5}%
\contentsline {subsection}{\numberline {76.3.2}Dimension Reduction}{618}{subsection.76.3.2}%
\contentsline {subsubsection}{\numberline {76.3.2.1}\texttt {setup\_dimension\_reduction}}{618}{subsubsection.76.3.2.1}%
\contentsline {subsubsection}{\numberline {76.3.2.2}\texttt {to\_red\_dim}}{618}{subsubsection.76.3.2.2}%
\contentsline {subsubsection}{\numberline {76.3.2.3}\texttt {to\_all\_dim}}{619}{subsubsection.76.3.2.3}%
\contentsline {subsection}{\numberline {76.3.3}Variable Transformation}{619}{subsection.76.3.3}%
\contentsline {subsubsection}{\numberline {76.3.3.1}\texttt {transform\_value} / \texttt {inverse\_transform\_value}}{619}{subsubsection.76.3.3.1}%
\contentsline {subsubsection}{\numberline {76.3.3.2}\texttt {transform\_X} / \texttt {inverse\_transform\_X}}{619}{subsubsection.76.3.3.2}%
\contentsline {subsection}{\numberline {76.3.4}Initial Design}{619}{subsection.76.3.4}%
\contentsline {subsubsection}{\numberline {76.3.4.1}\texttt {get\_initial\_design}}{619}{subsubsection.76.3.4.1}%
\contentsline {subsubsection}{\numberline {76.3.4.2}\texttt {generate\_initial\_design}}{620}{subsubsection.76.3.4.2}%
\contentsline {subsubsection}{\numberline {76.3.4.3}\texttt {curate\_initial\_design}}{620}{subsubsection.76.3.4.3}%
\contentsline {subsection}{\numberline {76.3.5}Surrogate \& Acquisition}{620}{subsection.76.3.5}%
\contentsline {subsubsection}{\numberline {76.3.5.1}\texttt {\_fit\_surrogate}}{620}{subsubsection.76.3.5.1}%
\contentsline {subsubsection}{\numberline {76.3.5.2}\texttt {\_predict\_with\_uncertainty}}{620}{subsubsection.76.3.5.2}%
\contentsline {subsubsection}{\numberline {76.3.5.3}\texttt {suggest\_next\_infill\_point}}{620}{subsubsection.76.3.5.3}%
\contentsline {subsection}{\numberline {76.3.6}Optimization Loop}{620}{subsection.76.3.6}%
\contentsline {subsubsection}{\numberline {76.3.6.1}\texttt {optimize}}{621}{subsubsection.76.3.6.1}%
\contentsline {subsubsection}{\numberline {76.3.6.2}\texttt {\_optimize\_single\_run}}{621}{subsubsection.76.3.6.2}%
\contentsline {subsection}{\numberline {76.3.7}Results \& Analysis}{621}{subsection.76.3.7}%
\contentsline {subsubsection}{\numberline {76.3.7.1}\texttt {save\_result} / \texttt {load\_result}}{621}{subsubsection.76.3.7.1}%
\contentsline {subsubsection}{\numberline {76.3.7.2}\texttt {save\_experiment} / \texttt {load\_experiment}}{621}{subsubsection.76.3.7.2}%
\contentsline {subsubsection}{\numberline {76.3.7.3}\texttt {get\_results\_table}}{621}{subsubsection.76.3.7.3}%
\contentsline {subsection}{\numberline {76.3.8}Plotting}{622}{subsection.76.3.8}%
\contentsline {subsubsection}{\numberline {76.3.8.1}\texttt {plot\_progress}}{622}{subsubsection.76.3.8.1}%
\contentsline {subsubsection}{\numberline {76.3.8.2}\texttt {plot\_surrogate}}{622}{subsubsection.76.3.8.2}%
\contentsline {subsubsection}{\numberline {76.3.8.3}\texttt {plot\_important\_hyperparameter\_contour}}{622}{subsubsection.76.3.8.3}%
\contentsline {subsection}{\numberline {76.3.9}TensorBoard Integration}{622}{subsection.76.3.9}%
\contentsline {subsubsection}{\numberline {76.3.9.1}\texttt {\_init\_tensorboard}}{622}{subsubsection.76.3.9.1}%
\contentsline {subsubsection}{\numberline {76.3.9.2}\texttt {\_write\_tensorboard\_scalars}}{622}{subsubsection.76.3.9.2}%
\contentsline {subsubsection}{\numberline {76.3.9.3}\texttt {\_write\_tensorboard\_hparams}}{623}{subsubsection.76.3.9.3}%
\contentsline {section}{\numberline {76.4}Description of the Infill Point Acquisition Steps}{623}{section.76.4}%
\contentsline {subsection}{\numberline {76.4.1}1. Optimize Acquisition Function}{623}{subsection.76.4.1}%
\contentsline {subsection}{\numberline {76.4.2}2. Candidate Validation and Selection}{623}{subsection.76.4.2}%
\contentsline {subsection}{\numberline {76.4.3}3. Failure Handling and Fallback Strategies}{624}{subsection.76.4.3}%
\contentsline {subsection}{\numberline {76.4.4}Key Arguments and Configuration}{624}{subsection.76.4.4}%
\contentsline {subsubsection}{\numberline {76.4.4.1}\texttt {tolerance\_x}}{624}{subsubsection.76.4.4.1}%
\contentsline {subsubsection}{\numberline {76.4.4.2}\texttt {min\_tol\_metric}}{625}{subsubsection.76.4.4.2}%
\contentsline {subsubsection}{\numberline {76.4.4.3}\texttt {acquisition\_failure\_strategy}}{625}{subsubsection.76.4.4.3}%
\contentsline {subsubsection}{\numberline {76.4.4.4}\texttt {acquisition\_optimizer}}{626}{subsubsection.76.4.4.4}%
\contentsline {subsubsection}{\numberline {76.4.4.5}\texttt {acquisition\_fun\_return\_size}}{626}{subsubsection.76.4.4.5}%
\contentsline {subsection}{\numberline {76.4.5}Example Usage}{626}{subsection.76.4.5}%
\contentsline {section}{\numberline {76.5}Using the Surrogate Model}{627}{section.76.5}%
\contentsline {section}{\numberline {76.6}Comparing the SpotOptim With the Default Surrogate Model and MLP Surrogate}{628}{section.76.6}%
\contentsline {section}{\numberline {76.7}Multi-Surrogate Switching in SpotOptim}{631}{section.76.7}%
\contentsline {subsection}{\numberline {76.7.1}Comparison: Default (GP) vs Multi-Surrogate (GP + MLP) on Robot Arm}{631}{subsection.76.7.1}%
\contentsline {part}{\numberline {XII}Optimization}{663}{part.12}%
\contentsline {chapter}{\numberline {77}Aircraft Wing Weight Example}{665}{chapter.77}%
\contentsline {section}{\numberline {77.1}AWWE Equation}{665}{section.77.1}%
\contentsline {section}{\numberline {77.2}AWWE Parameters and Equations (Part 1)}{665}{section.77.2}%
\contentsline {section}{\numberline {77.3}Goals: Understanding and Optimization}{666}{section.77.3}%
\contentsline {section}{\numberline {77.4}Properties of the Python ``Solver''}{667}{section.77.4}%
\contentsline {section}{\numberline {77.5}Plot 1: Load Factor (\(N_z\)) and Aspect Ratio (\(A\))}{668}{section.77.5}%
\contentsline {section}{\numberline {77.6}Plot 2: Taper Ratio and Fuel Weight}{671}{section.77.6}%
\contentsline {section}{\numberline {77.7}The Big Picture: Combining all Variables}{672}{section.77.7}%
\contentsline {section}{\numberline {77.8}AWWE Landscape}{675}{section.77.8}%
\contentsline {section}{\numberline {77.9}Summary of the First Experiments}{676}{section.77.9}%
\contentsline {section}{\numberline {77.10}Exercise}{676}{section.77.10}%
\contentsline {subsection}{\numberline {77.10.1}Adding Paint Weight}{676}{subsection.77.10.1}%
\contentsline {section}{\numberline {77.11}Jupyter Notebook}{677}{section.77.11}%
\contentsline {part}{\numberline {XIII}Numerical Methods}{679}{part.13}%
\contentsline {chapter}{\numberline {78}Simulation and Surrogate Modeling}{681}{chapter.78}%
\contentsline {section}{\numberline {78.1}Surrogates}{681}{section.78.1}%
\contentsline {subsection}{\numberline {78.1.1}Costs of Simulation}{682}{subsection.78.1.1}%
\contentsline {subsection}{\numberline {78.1.2}Mathematical Models and Meta-Models}{682}{subsection.78.1.2}%
\contentsline {subsection}{\numberline {78.1.3}Surrogates = Trained Meta-models}{682}{subsection.78.1.3}%
\contentsline {subsection}{\numberline {78.1.4}Computer Experiments}{682}{subsection.78.1.4}%
\contentsline {subsection}{\numberline {78.1.5}Limits of Mathematical Modeling}{683}{subsection.78.1.5}%
\contentsline {subsection}{\numberline {78.1.6}Why Computer Simulations are Necessary}{683}{subsection.78.1.6}%
\contentsline {subsection}{\numberline {78.1.7}Simulation Requirements}{683}{subsection.78.1.7}%
\contentsline {section}{\numberline {78.2}Applications of Surrogate Models}{683}{section.78.2}%
\contentsline {section}{\numberline {78.3}DACE and RSM}{684}{section.78.3}%
\contentsline {subsection}{\numberline {78.3.1}Noise Handling in RSM and DACE}{685}{subsection.78.3.1}%
\contentsline {section}{\numberline {78.4}Updating a Surrogate Model}{685}{section.78.4}%
\contentsline {chapter}{\numberline {79}Sampling Plans}{687}{chapter.79}%
\contentsline {section}{\numberline {79.1}Ideas and Concepts}{687}{section.79.1}%
\contentsline {subsection}{\numberline {79.1.1}The `Curse of Dimensionality' and How to Avoid It}{688}{subsection.79.1.1}%
\contentsline {subsection}{\numberline {79.1.2}Physical versus Computational Experiments}{689}{subsection.79.1.2}%
\contentsline {subsection}{\numberline {79.1.3}Designing Preliminary Experiments (Screening)}{690}{subsection.79.1.3}%
\contentsline {subsubsection}{\numberline {79.1.3.1}Estimating the Distribution of Elementary Effects}{690}{subsubsection.79.1.3.1}%
\contentsline {subsection}{\numberline {79.1.4}Special Considerations When Deploying Screening Algorithms}{699}{subsection.79.1.4}%
\contentsline {section}{\numberline {79.2}Analyzing Variable Importance in Aircraft Wing Weight}{699}{section.79.2}%
\contentsline {section}{\numberline {79.3}Designing a Sampling Plan}{702}{section.79.3}%
\contentsline {subsection}{\numberline {79.3.1}Stratification}{702}{subsection.79.3.1}%
\contentsline {subsection}{\numberline {79.3.2}Latin Squares and Random Latin Hypercubes}{707}{subsection.79.3.2}%
\contentsline {subsection}{\numberline {79.3.3}Space-filling Designs: Maximin Plans}{710}{subsection.79.3.3}%
\contentsline {subsubsection}{\numberline {79.3.3.1}The Function \texttt {jd}}{712}{subsubsection.79.3.3.1}%
\contentsline {subsection}{\numberline {79.3.4}Memory Management}{715}{subsection.79.3.4}%
\contentsline {subsubsection}{\numberline {79.3.4.1}The Function \texttt {mm}}{715}{subsubsection.79.3.4.1}%
\contentsline {subsubsection}{\numberline {79.3.4.2}The Function \texttt {mmphi}}{718}{subsubsection.79.3.4.2}%
\contentsline {subsubsection}{\numberline {79.3.4.3}The Function \texttt {mmsort}}{720}{subsubsection.79.3.4.3}%
\contentsline {subsubsection}{\numberline {79.3.4.4}The Function \texttt {phisort}}{722}{subsubsection.79.3.4.4}%
\contentsline {subsection}{\numberline {79.3.5}Optimizing the Morris-Mitchell Criterion \(\Phi _q\)}{723}{subsection.79.3.5}%
\contentsline {subsubsection}{\numberline {79.3.5.1}The Function \texttt {perturb()}}{724}{subsubsection.79.3.5.1}%
\contentsline {subsection}{\numberline {79.3.6}Evolutionary Operation}{725}{subsection.79.3.6}%
\contentsline {subsection}{\numberline {79.3.7}Putting it all Together}{726}{subsection.79.3.7}%
\contentsline {subsubsection}{\numberline {79.3.7.1}The Function \texttt {mmlhs}}{726}{subsubsection.79.3.7.1}%
\contentsline {subsubsection}{\numberline {79.3.7.2}The Function \texttt {bestlh}}{729}{subsubsection.79.3.7.2}%
\contentsline {section}{\numberline {79.4}Experimental Analysis of the Morris-Mitchell Criterion}{733}{section.79.4}%
\contentsline {subsection}{\numberline {79.4.1}Evaluation of Sampling Designs}{733}{subsection.79.4.1}%
\contentsline {subsection}{\numberline {79.4.2}Demonstrate the Impact of mmphi Parameters}{736}{subsection.79.4.2}%
\contentsline {subsection}{\numberline {79.4.3}Morris-Mitchell Criterion: Impact of Adding Points}{736}{subsection.79.4.3}%
\contentsline {section}{\numberline {79.5}A Sample-Size Invariant Version of the Morris-Mitchell Criterion}{741}{section.79.5}%
\contentsline {subsection}{\numberline {79.5.1}Comparison of \texttt {mmphi()} and \texttt {mmphi\_intensive()}}{741}{subsection.79.5.1}%
\contentsline {subsection}{\numberline {79.5.2}Plotting the Two Morris-Mitchell Criteria for Different Sample Sizes}{742}{subsection.79.5.2}%
\contentsline {section}{\numberline {79.6}Jupyter Notebook}{745}{section.79.6}%
\contentsline {chapter}{\numberline {80}Constructing a Surrogate}{747}{chapter.80}%
\contentsline {section}{\numberline {80.1}Stage One: Preparing the Data and Choosing a Modelling Approach}{747}{section.80.1}%
\contentsline {section}{\numberline {80.2}Stage Two: Parameter Estimation and Training}{749}{section.80.2}%
\contentsline {section}{\numberline {80.3}Stage Three: Model Testing}{751}{section.80.3}%
\contentsline {subsubsection}{\numberline {80.3.0.1}The Tea and Sugar Analogy}{752}{subsubsection.80.3.0.1}%
\contentsline {section}{\numberline {80.4}Jupyter Notebook}{753}{section.80.4}%
\contentsline {chapter}{\numberline {81}Response Surface Methods}{755}{chapter.81}%
\contentsline {section}{\numberline {81.1}What is RSM?}{755}{section.81.1}%
\contentsline {subsection}{\numberline {81.1.1}Visualization: Problems in Practice}{758}{subsection.81.1.1}%
\contentsline {subsection}{\numberline {81.1.2}RSM: Strategies}{758}{subsection.81.1.2}%
\contentsline {subsection}{\numberline {81.1.3}RSM: Noise in the Empirical Model}{759}{subsection.81.1.3}%
\contentsline {subsection}{\numberline {81.1.4}RSM: Natural and Coded Variables}{759}{subsection.81.1.4}%
\contentsline {subsection}{\numberline {81.1.5}RSM Low-order Polynomials}{760}{subsection.81.1.5}%
\contentsline {section}{\numberline {81.2}First-Order Models (Main Effects Model)}{760}{section.81.2}%
\contentsline {subsection}{\numberline {81.2.1}First-Order Model Properties}{761}{subsection.81.2.1}%
\contentsline {subsection}{\numberline {81.2.2}First-order Model with Interactions in python}{761}{subsection.81.2.2}%
\contentsline {subsection}{\numberline {81.2.3}Observations: First-Order Model with Interactions}{763}{subsection.81.2.3}%
\contentsline {section}{\numberline {81.3}Second-Order Models}{763}{section.81.3}%
\contentsline {subsection}{\numberline {81.3.1}Second-Order Models: Properties}{764}{subsection.81.3.1}%
\contentsline {subsection}{\numberline {81.3.2}Example: Stationary Ridge}{764}{subsection.81.3.2}%
\contentsline {subsection}{\numberline {81.3.3}Observations: Second-Order Model (Ridge)}{765}{subsection.81.3.3}%
\contentsline {subsection}{\numberline {81.3.4}Example: Rising Ridge}{766}{subsection.81.3.4}%
\contentsline {subsection}{\numberline {81.3.5}Summary: Rising Ridge}{767}{subsection.81.3.5}%
\contentsline {subsection}{\numberline {81.3.6}Falling Ridge}{767}{subsection.81.3.6}%
\contentsline {subsection}{\numberline {81.3.7}Saddle Point}{767}{subsection.81.3.7}%
\contentsline {subsection}{\numberline {81.3.8}Interpretation: Saddle Points}{769}{subsection.81.3.8}%
\contentsline {subsection}{\numberline {81.3.9}Summary: Ridge Analysis}{769}{subsection.81.3.9}%
\contentsline {section}{\numberline {81.4}General RSM Models}{769}{section.81.4}%
\contentsline {subsection}{\numberline {81.4.1}Ordinary Least Squares}{769}{subsection.81.4.1}%
\contentsline {section}{\numberline {81.5}General Linear Regression}{770}{section.81.5}%
\contentsline {section}{\numberline {81.6}Designs}{771}{section.81.6}%
\contentsline {subsection}{\numberline {81.6.1}Different Designs}{772}{subsection.81.6.1}%
\contentsline {section}{\numberline {81.7}RSM Experimentation}{772}{section.81.7}%
\contentsline {subsection}{\numberline {81.7.1}First Step}{772}{subsection.81.7.1}%
\contentsline {subsection}{\numberline {81.7.2}Second Step}{772}{subsection.81.7.2}%
\contentsline {subsection}{\numberline {81.7.3}Third Step}{772}{subsection.81.7.3}%
\contentsline {section}{\numberline {81.8}RSM: Review and General Considerations}{773}{section.81.8}%
\contentsline {subsection}{\numberline {81.8.1}Historical Considerations about RSM}{774}{subsection.81.8.1}%
\contentsline {subsection}{\numberline {81.8.2}Status Quo}{774}{subsection.81.8.2}%
\contentsline {subsection}{\numberline {81.8.3}The Role of Statistics}{774}{subsection.81.8.3}%
\contentsline {subsection}{\numberline {81.8.4}New RSM is needed: DACE}{774}{subsection.81.8.4}%
\contentsline {section}{\numberline {81.9}Exercises}{775}{section.81.9}%
\contentsline {section}{\numberline {81.10}Jupyter Notebook}{775}{section.81.10}%
\contentsline {chapter}{\numberline {82}Polynomial Models}{777}{chapter.82}%
\contentsline {section}{\numberline {82.1}Fitting a Polynomial}{777}{section.82.1}%
\contentsline {section}{\numberline {82.2}Polynomial Fitting in Python}{778}{section.82.2}%
\contentsline {subsection}{\numberline {82.2.1}Fitting the Polynomial}{778}{subsection.82.2.1}%
\contentsline {subsection}{\numberline {82.2.2}Explaining the \(k\)-fold Cross-Validation}{779}{subsection.82.2.2}%
\contentsline {subsection}{\numberline {82.2.3}Making Predictions}{780}{subsection.82.2.3}%
\contentsline {subsection}{\numberline {82.2.4}Plotting the Results}{781}{subsection.82.2.4}%
\contentsline {section}{\numberline {82.3}Example One: Aerofoil Drag}{782}{section.82.3}%
\contentsline {section}{\numberline {82.4}Example Two: A Multimodal Test Case}{784}{section.82.4}%
\contentsline {section}{\numberline {82.5}Extending to Multivariable Polynomial Models}{785}{section.82.5}%
\contentsline {section}{\numberline {82.6}Jupyter Notebook}{787}{section.82.6}%
\contentsline {chapter}{\numberline {83}Radial Basis Function Models}{789}{chapter.83}%
\contentsline {section}{\numberline {83.1}Radial Basis Function Models}{789}{section.83.1}%
\contentsline {subsection}{\numberline {83.1.1}Fitting Noise-Free Data}{790}{subsection.83.1.1}%
\contentsline {subsubsection}{\numberline {83.1.1.1}Selecting Basis Functions: From Fixed to Parametric Forms}{791}{subsubsection.83.1.1.1}%
\contentsline {subsubsection}{\numberline {83.1.1.2}The Interpolation Condition: Elegant Solutions Through Linear Systems}{792}{subsubsection.83.1.1.2}%
\contentsline {subsection}{\numberline {83.1.2}Numerical Stability Through Positive Definite Matrices}{794}{subsection.83.1.2}%
\contentsline {subsection}{\numberline {83.1.3}Ill-Conditioning}{795}{subsection.83.1.3}%
\contentsline {subsection}{\numberline {83.1.4}Parameter Optimization: A Two-Level Approach}{796}{subsection.83.1.4}%
\contentsline {subsubsection}{\numberline {83.1.4.1}Inner Level (\(\vec {w}\))}{796}{subsubsection.83.1.4.1}%
\contentsline {subsubsection}{\numberline {83.1.4.2}Outer Level (\(\sigma \))}{797}{subsubsection.83.1.4.2}%
\contentsline {section}{\numberline {83.2}Python Implementation of the RBF Model}{799}{section.83.2}%
\contentsline {subsection}{\numberline {83.2.1}The Rbf Class}{800}{subsection.83.2.1}%
\contentsline {section}{\numberline {83.3}RBF Example: The One-Dimensional \texttt {sin} Function}{806}{section.83.3}%
\contentsline {section}{\numberline {83.4}RBF Example: The Two-Diemnsional \texttt {dome} Function}{808}{section.83.4}%
\contentsline {subsection}{\numberline {83.4.1}The Connection Between RBF Models and Neural Networks}{811}{subsection.83.4.1}%
\contentsline {section}{\numberline {83.5}Radial Basis Function Models for Noisy Data}{812}{section.83.5}%
\contentsline {subsection}{\numberline {83.5.1}Ridge Regularization Approach}{812}{subsection.83.5.1}%
\contentsline {subsection}{\numberline {83.5.2}Reduced Basis Approach}{812}{subsection.83.5.2}%
\contentsline {section}{\numberline {83.6}Jupyter Notebook}{813}{section.83.6}%
\contentsline {chapter}{\numberline {84}Kriging (Gaussian Process Regression)}{815}{chapter.84}%
\contentsline {section}{\numberline {84.1}From Gaussian RBF to Kriging Basis Functions}{815}{section.84.1}%
\contentsline {section}{\numberline {84.2}Building the Kriging Model}{816}{section.84.2}%
\contentsline {section}{\numberline {84.3}MLE to estimate \(\theta \) and \(p\)}{823}{section.84.3}%
\contentsline {subsection}{\numberline {84.3.1}The Log-Likelihood}{823}{subsection.84.3.1}%
\contentsline {subsection}{\numberline {84.3.2}Differentiation with Respect to \(\mu \)}{825}{subsection.84.3.2}%
\contentsline {subsection}{\numberline {84.3.3}Differentiation with Respect to \(\sigma \)}{827}{subsection.84.3.3}%
\contentsline {subsection}{\numberline {84.3.4}Results of the Optimizations}{827}{subsection.84.3.4}%
\contentsline {subsection}{\numberline {84.3.5}The Concentrated Log-Likelihood Function}{828}{subsection.84.3.5}%
\contentsline {subsection}{\numberline {84.3.6}Optimizing the Parameters \(\vec {\theta }\) and \(\vec {p}\)}{828}{subsection.84.3.6}%
\contentsline {subsection}{\numberline {84.3.7}Correlation and Covariance Matrices Revisited}{828}{subsection.84.3.7}%
\contentsline {section}{\numberline {84.4}Implementing an MLE of the Model Parameters}{829}{section.84.4}%
\contentsline {section}{\numberline {84.5}Kriging Prediction}{830}{section.84.5}%
\contentsline {section}{\numberline {84.6}Kriging Example: Sinusoid Function}{831}{section.84.6}%
\contentsline {subsection}{\numberline {84.6.1}Calculating the Correlation Matrix \(\Psi \)}{831}{subsection.84.6.1}%
\contentsline {subsection}{\numberline {84.6.2}Computing the \(\Psi \) Matrix}{833}{subsection.84.6.2}%
\contentsline {subsection}{\numberline {84.6.3}Selecting the New Locations}{834}{subsection.84.6.3}%
\contentsline {subsection}{\numberline {84.6.4}Computing the \(\psi \) Vector}{834}{subsection.84.6.4}%
\contentsline {subsection}{\numberline {84.6.5}Predicting at New Locations}{835}{subsection.84.6.5}%
\contentsline {subsection}{\numberline {84.6.6}Visualization}{835}{subsection.84.6.6}%
\contentsline {subsection}{\numberline {84.6.7}The Complete Python Code for the Example}{836}{subsection.84.6.7}%
\contentsline {section}{\numberline {84.7}Jupyter Notebook}{842}{section.84.7}%
\contentsline {chapter}{\numberline {85}Matrices}{843}{chapter.85}%
\contentsline {section}{\numberline {85.1}Derivatives of Quadratic Forms}{843}{section.85.1}%
\contentsline {section}{\numberline {85.2}The Condition Number}{844}{section.85.2}%
\contentsline {section}{\numberline {85.3}The Moore-Penrose Pseudoinverse}{845}{section.85.3}%
\contentsline {subsection}{\numberline {85.3.1}Definitions}{845}{subsection.85.3.1}%
\contentsline {subsection}{\numberline {85.3.2}Implementation in Python}{845}{subsection.85.3.2}%
\contentsline {section}{\numberline {85.4}Strictly Positive Definite Kernels}{846}{section.85.4}%
\contentsline {subsection}{\numberline {85.4.1}Definition}{846}{subsection.85.4.1}%
\contentsline {subsection}{\numberline {85.4.2}Connection to Positive Definite Matrices}{846}{subsection.85.4.2}%
\contentsline {subsection}{\numberline {85.4.3}Connection to RBF Models}{847}{subsection.85.4.3}%
\contentsline {section}{\numberline {85.5}Cholesky Decomposition and Positive Definite Matrices}{847}{section.85.5}%
\contentsline {subsection}{\numberline {85.5.1}Example of Cholesky Decomposition}{850}{subsection.85.5.1}%
\contentsline {subsection}{\numberline {85.5.2}Inverse Matrix Using Cholesky Decomposition}{851}{subsection.85.5.2}%
\contentsline {section}{\numberline {85.6}Nyström Approximation}{852}{section.85.6}%
\contentsline {subsection}{\numberline {85.6.1}What's the Big Idea?}{852}{subsection.85.6.1}%
\contentsline {subsection}{\numberline {85.6.2}How Does It Work?}{852}{subsection.85.6.2}%
\contentsline {subsection}{\numberline {85.6.3}Example}{853}{subsection.85.6.3}%
\contentsline {subsection}{\numberline {85.6.4}Why Is This Useful?}{853}{subsection.85.6.4}%
\contentsline {subsection}{\numberline {85.6.5}Applying the Nyström Approximation: How Nyström Approximation Helps Kriging}{854}{subsection.85.6.5}%
\contentsline {subsection}{\numberline {85.6.6}Example: Predicting Temperature with Nyström-Kriging}{854}{subsection.85.6.6}%
\contentsline {subsubsection}{\numberline {85.6.6.1}Step 1: Nyström Approximation of the Covariance Matrix}{855}{subsubsection.85.6.6.1}%
\contentsline {subsubsection}{\numberline {85.6.6.2}Step 2: Modeling and Prediction}{855}{subsubsection.85.6.6.2}%
\contentsline {subsection}{\numberline {85.6.7}Details: Woodbury Matrix Identity for Avoiding the Big Inversion}{855}{subsection.85.6.7}%
\contentsline {subsection}{\numberline {85.6.8}The Example: Step-by-Step}{856}{subsection.85.6.8}%
\contentsline {subsubsection}{\numberline {85.6.8.1}Method 1: The Slow Way (Forming the full matrix)}{857}{subsubsection.85.6.8.1}%
\contentsline {subsubsection}{\numberline {85.6.8.2}Method 2: The Fast Way (Using Woodbury Identity)}{857}{subsubsection.85.6.8.2}%
\contentsline {section}{\numberline {85.7}Extending spotpython's Kriging Surrogate with Nyström Approximation for Enhanced Scalability}{858}{section.85.7}%
\contentsline {subsection}{\numberline {85.7.1}Introduction: Overcoming the Scalability Challenge in Kriging for Sequential Optimization}{858}{subsection.85.7.1}%
\contentsline {subsection}{\numberline {85.7.2}Report Objectives and Structure}{859}{subsection.85.7.2}%
\contentsline {section}{\numberline {85.8}Theoretical Foundations: The Nyström--Kriging Framework}{859}{section.85.8}%
\contentsline {subsection}{\numberline {85.8.1}A Primer on Kriging (Gaussian Process Regression)}{859}{subsection.85.8.1}%
\contentsline {subsection}{\numberline {85.8.2}The Nyström Method for Low-Rank Kernel Approximation}{859}{subsection.85.8.2}%
\contentsline {subsection}{\numberline {85.8.3}Justification for Landmark Selection}{860}{subsection.85.8.3}%
\contentsline {section}{\numberline {85.9}Implementation: A Scalable Kriging Class for spotpython}{860}{section.85.9}%
\contentsline {subsection}{\numberline {85.9.1}Updated kriging.py with Nyström Approximation (excerpt)}{860}{subsection.85.9.1}%
\contentsline {section}{\numberline {85.10}Implementation Details}{862}{section.85.10}%
\contentsline {subsection}{\numberline {85.10.1}Architectural Enhancements to \texttt {init}}{862}{subsection.85.10.1}%
\contentsline {subsection}{\numberline {85.10.2}The \texttt {fit()} Method: A Dual-Pathway Approach}{863}{subsection.85.10.2}%
\contentsline {subsection}{\numberline {85.10.3}The \texttt {predict()} Method: Conditional Prediction Logic}{863}{subsection.85.10.3}%
\contentsline {subsection}{\numberline {85.10.4}Critical Detail: Preserving Mixed Variable Type Functionality}{863}{subsection.85.10.4}%
\contentsline {subsection}{\numberline {85.10.5}Seamless Integration into the Nyström Workflow}{864}{subsection.85.10.5}%
\contentsline {section}{\numberline {85.11}Jupyter Notebook}{864}{section.85.11}%
\contentsline {chapter}{\numberline {86}Infill Criteria}{865}{chapter.86}%
\contentsline {section}{\numberline {86.1}Balancing Exploitation and Exploration}{866}{section.86.1}%
\contentsline {section}{\numberline {86.2}Expected Improvement (EI)}{866}{section.86.2}%
\contentsline {section}{\numberline {86.3}Expected Improvement in the Hyperparameter Tuning Cookbook (Python Implementation)}{867}{section.86.3}%
\contentsline {section}{\numberline {86.4}Jupyter Notebook}{868}{section.86.4}%
\contentsline {chapter}{\numberline {87}Remote Optimization Challenge: A Benchmarking Study}{869}{chapter.87}%
\contentsline {section}{\numberline {87.1}Benchmarking Best Practices}{869}{section.87.1}%
\contentsline {section}{\numberline {87.2}Experimental Setup}{869}{section.87.2}%
\contentsline {subsection}{\numberline {87.2.1}The Remote Objective Function}{869}{subsection.87.2.1}%
\contentsline {subsection}{\numberline {87.2.2}Algorithms}{870}{subsection.87.2.2}%
\contentsline {subsection}{\numberline {87.2.3}Budget and Settings}{870}{subsection.87.2.3}%
\contentsline {section}{\numberline {87.3}Comparison of the Difficulty}{870}{section.87.3}%
\contentsline {section}{\numberline {87.4}Statistical Power Analysis}{873}{section.87.4}%
\contentsline {subsection}{\numberline {87.4.1}1. Estimating Noise (\(\sigma \))}{873}{subsection.87.4.1}%
\contentsline {subsection}{\numberline {87.4.2}2. Determining Sample Size (\(n\))}{873}{subsection.87.4.2}%
\contentsline {section}{\numberline {87.5}Experimental Comparison}{874}{section.87.5}%
\contentsline {section}{\numberline {87.6}Convergence Analysis (Noisy)}{877}{section.87.6}%
\contentsline {section}{\numberline {87.7}Comparative Analysis}{879}{section.87.7}%
\contentsline {section}{\numberline {87.8}Conclusion}{881}{section.87.8}%
\contentsline {section}{\numberline {87.9}Landscape Analysis of \texttt {robot\_arm\_hard}}{881}{section.87.9}%
\contentsline {section}{\numberline {87.10}References}{883}{section.87.10}%
\contentsline {section}{\numberline {87.11}Jupyter Notebook}{884}{section.87.11}%
\contentsline {part}{\numberline {XIV}Data-Driven Modeling and Optimization}{885}{part.14}%
\contentsline {chapter}{\numberline {88}Basic Statistics and Data Analysis}{887}{chapter.88}%
\contentsline {section}{\numberline {88.1}Exploratory Data Analysis}{887}{section.88.1}%
\contentsline {subsection}{\numberline {88.1.1}Histograms}{887}{subsection.88.1.1}%
\contentsline {subsection}{\numberline {88.1.2}Boxplots}{890}{subsection.88.1.2}%
\contentsline {section}{\numberline {88.2}Probability Distributions}{890}{section.88.2}%
\contentsline {subsection}{\numberline {88.2.1}Sampling from a Distribution}{891}{subsection.88.2.1}%
\contentsline {section}{\numberline {88.3}Discrete Distributions}{891}{section.88.3}%
\contentsline {subsection}{\numberline {88.3.1}Bernoulli Distribution}{891}{subsection.88.3.1}%
\contentsline {subsection}{\numberline {88.3.2}Binomial Distribution}{892}{subsection.88.3.2}%
\contentsline {section}{\numberline {88.4}Continuous Distributions}{893}{section.88.4}%
\contentsline {subsection}{\numberline {88.4.1}Distribution functions: PDFs and CDFs}{893}{subsection.88.4.1}%
\contentsline {section}{\numberline {88.5}Background: Expectation, Mean, Standard Deviation}{894}{section.88.5}%
\contentsline {section}{\numberline {88.6}Distributions and Random Numbers in Python}{896}{section.88.6}%
\contentsline {subsection}{\numberline {88.6.1}Calculation of the Standard Deviation with Python}{896}{subsection.88.6.1}%
\contentsline {subsection}{\numberline {88.6.2}The Argument ``axis''}{897}{subsection.88.6.2}%
\contentsline {section}{\numberline {88.7}Expectation (Continuous)}{898}{section.88.7}%
\contentsline {section}{\numberline {88.8}Variance and Standard Deviation (Continuous)}{898}{section.88.8}%
\contentsline {subsection}{\numberline {88.8.1}The Uniform Distribution}{899}{subsection.88.8.1}%
\contentsline {section}{\numberline {88.9}The Uniform Distribution}{900}{section.88.9}%
\contentsline {subsection}{\numberline {88.9.1}The Normal Distribution}{901}{subsection.88.9.1}%
\contentsline {subsection}{\numberline {88.9.2}Visualization of the Standard Deviation}{902}{subsection.88.9.2}%
\contentsline {subsection}{\numberline {88.9.3}Realizations of a Normal Distribution}{903}{subsection.88.9.3}%
\contentsline {section}{\numberline {88.10}The Normal Distribution}{904}{section.88.10}%
\contentsline {section}{\numberline {88.11}The Exponential Distribution}{909}{section.88.11}%
\contentsline {subsection}{\numberline {88.11.1}Standardization of Random Variables}{909}{subsection.88.11.1}%
\contentsline {section}{\numberline {88.12}Covariance and Correlation}{909}{section.88.12}%
\contentsline {subsection}{\numberline {88.12.1}The Multivariate Normal Distribution}{909}{subsection.88.12.1}%
\contentsline {section}{\numberline {88.13}Covariance}{912}{section.88.13}%
\contentsline {section}{\numberline {88.14}Correlation}{914}{section.88.14}%
\contentsline {subsection}{\numberline {88.14.1}Definitions}{914}{subsection.88.14.1}%
\contentsline {subsection}{\numberline {88.14.2}Computations}{916}{subsection.88.14.2}%
\contentsline {subsection}{\numberline {88.14.3}The Outer-product and the \texttt {np.outer} Function}{917}{subsection.88.14.3}%
\contentsline {subsection}{\numberline {88.14.4}Correlation and Independence}{920}{subsection.88.14.4}%
\contentsline {subsection}{\numberline {88.14.5}Pearson's Correlation}{922}{subsection.88.14.5}%
\contentsline {subsection}{\numberline {88.14.6}Interpreting the Correlation: Correlation Squared}{922}{subsection.88.14.6}%
\contentsline {subsection}{\numberline {88.14.7}Partial Correlation}{923}{subsection.88.14.7}%
\contentsline {section}{\numberline {88.15}Hypothesis Testing and the Null-Hypothesis}{926}{section.88.15}%
\contentsline {subsection}{\numberline {88.15.1}Alternative Hypotheses, Main Ideas}{926}{subsection.88.15.1}%
\contentsline {subsection}{\numberline {88.15.2}p-values: What they are and how to interpret them}{926}{subsection.88.15.2}%
\contentsline {subsubsection}{\numberline {88.15.2.1}How to calculate p-values}{926}{subsubsection.88.15.2.1}%
\contentsline {subsubsection}{\numberline {88.15.2.2}p-hacking: What it is and how to avoid it}{926}{subsubsection.88.15.2.2}%
\contentsline {section}{\numberline {88.16}Statistical Power}{926}{section.88.16}%
\contentsline {subsubsection}{\numberline {88.16.0.1}Power Analysis}{926}{subsubsection.88.16.0.1}%
\contentsline {section}{\numberline {88.17}The Central Limit Theorem}{926}{section.88.17}%
\contentsline {section}{\numberline {88.18}Maximum Likelihood}{926}{section.88.18}%
\contentsline {section}{\numberline {88.19}Maximum Likelihood Estimation: Multivariate Normal Distribution}{928}{section.88.19}%
\contentsline {subsection}{\numberline {88.19.1}The Joint Probability Density Function of the Multivariate Normal Distribution}{928}{subsection.88.19.1}%
\contentsline {subsection}{\numberline {88.19.2}The Log-Likelihood Function}{929}{subsection.88.19.2}%
\contentsline {section}{\numberline {88.20}Cross-Validation}{929}{section.88.20}%
\contentsline {subsubsection}{\numberline {88.20.0.1}Bias and Variance}{930}{subsubsection.88.20.0.1}%
\contentsline {section}{\numberline {88.21}Mutual Information}{930}{section.88.21}%
\contentsline {section}{\numberline {88.22}t-SNE}{931}{section.88.22}%
\contentsline {chapter}{\numberline {89}Hypothesis Testing}{933}{chapter.89}%
\contentsline {section}{\numberline {89.1}Paired-Sample Analysis}{933}{section.89.1}%
\contentsline {subsection}{\numberline {89.1.1}Directional Hypotheses for a Paired Experiment}{934}{subsection.89.1.1}%
\contentsline {subsubsection}{\numberline {89.1.1.1}Defining the Parameter of Interest and the Alternative Hypothesis}{934}{subsubsection.89.1.1.1}%
\contentsline {subsubsection}{\numberline {89.1.1.2}Establishing the Null Hypothesis}{935}{subsubsection.89.1.1.2}%
\contentsline {subsubsection}{\numberline {89.1.1.3}The Critical Justification for a One-Tailed Test}{935}{subsubsection.89.1.1.3}%
\contentsline {subsubsection}{\numberline {89.1.1.4}One-tailed Test Justification Example}{935}{subsubsection.89.1.1.4}%
\contentsline {subsection}{\numberline {89.1.2}The Paired Samples t-Test: A Technical Examination}{936}{subsection.89.1.2}%
\contentsline {subsubsection}{\numberline {89.1.2.1}3.1 Core Assumptions of the Paired t-Test}{936}{subsubsection.89.1.2.1}%
\contentsline {subsubsection}{\numberline {89.1.2.2}A Nuanced Approach to Assumption Checking}{937}{subsubsection.89.1.2.2}%
\contentsline {subsubsection}{\numberline {89.1.2.3}Calculating the Test Statistic}{937}{subsubsection.89.1.2.3}%
\contentsline {subsubsection}{\numberline {89.1.2.4}The Decision Framework}{938}{subsubsection.89.1.2.4}%
\contentsline {subsection}{\numberline {89.1.3}A Priori Sample Size Determination for Desired Power}{938}{subsection.89.1.3}%
\contentsline {subsubsection}{\numberline {89.1.3.1}The Four Pillars of Power Analysis}{939}{subsubsection.89.1.3.1}%
\contentsline {subsection}{\numberline {89.1.4}Effect Size (\(\Delta \))}{939}{subsection.89.1.4}%
\contentsline {subsubsection}{\numberline {89.1.4.1}Practical Challenges: Estimating \(\sigma _d\) and \(\Delta \)}{940}{subsubsection.89.1.4.1}%
\contentsline {subsubsection}{\numberline {89.1.4.2}Formulating the Hypotheses}{941}{subsubsection.89.1.4.2}%
\contentsline {subsubsection}{\numberline {89.1.4.3}A Priori Sample Size Calculation}{942}{subsubsection.89.1.4.3}%
\contentsline {subsubsection}{\numberline {89.1.4.4}Data Collection and Presentation (Hypothetical)}{942}{subsubsection.89.1.4.4}%
\contentsline {subsubsection}{\numberline {89.1.4.5}Checking the Assumptions}{943}{subsubsection.89.1.4.5}%
\contentsline {subsubsection}{\numberline {89.1.4.6}Calculating the Test Statistic}{943}{subsubsection.89.1.4.6}%
\contentsline {subsubsection}{\numberline {89.1.4.7}Determining the p-value}{943}{subsubsection.89.1.4.7}%
\contentsline {subsubsection}{\numberline {89.1.4.8}Drawing a Conclusion}{943}{subsubsection.89.1.4.8}%
\contentsline {section}{\numberline {89.2}Non-inferiority Test}{944}{section.89.2}%
\contentsline {chapter}{\numberline {90}Addressing Multicollinearity: Principle Component Analysis (PCA) and Factor Analysis (FA)}{947}{chapter.90}%
\contentsline {section}{\numberline {90.1}Introduction}{947}{section.90.1}%
\contentsline {section}{\numberline {90.2}The Car-Sales Data Set}{948}{section.90.2}%
\contentsline {subsection}{\numberline {90.2.1}The Target Variable}{949}{subsection.90.2.1}%
\contentsline {subsection}{\numberline {90.2.2}The Features}{949}{subsection.90.2.2}%
\contentsline {subsubsection}{\numberline {90.2.2.1}Numerical Features}{949}{subsubsection.90.2.2.1}%
\contentsline {subsubsection}{\numberline {90.2.2.2}Categorical Features}{950}{subsubsection.90.2.2.2}%
\contentsline {subsection}{\numberline {90.2.3}Combining Non-categorical and Categorical (encoded) Data}{951}{subsection.90.2.3}%
\contentsline {section}{\numberline {90.3}Fit the Linear Regression Model}{951}{section.90.3}%
\contentsline {subsection}{\numberline {90.3.1}Model Summary and Interpretation}{952}{subsection.90.3.1}%
\contentsline {subsubsection}{\numberline {90.3.1.1}Model Summary (ANOVA Table)}{952}{subsubsection.90.3.1.1}%
\contentsline {section}{\numberline {90.4}Collinearity Diagnostics}{953}{section.90.4}%
\contentsline {subsection}{\numberline {90.4.1}The Coefficient Table}{953}{subsection.90.4.1}%
\contentsline {subsection}{\numberline {90.4.2}Eigenvalues and Condition Indices}{954}{subsection.90.4.2}%
\contentsline {subsubsection}{\numberline {90.4.2.1}Eigenvalues}{954}{subsubsection.90.4.2.1}%
\contentsline {subsubsection}{\numberline {90.4.2.2}Condition Indices}{954}{subsubsection.90.4.2.2}%
\contentsline {subsection}{\numberline {90.4.3}Kayser-Meyer-Olkin (KMO) Measure}{955}{subsection.90.4.3}%
\contentsline {section}{\numberline {90.5}Addressing Multicollinearity with Principal Component Analysis (PCA)}{956}{section.90.5}%
\contentsline {subsection}{\numberline {90.5.1}Introduction to PCA}{956}{subsection.90.5.1}%
\contentsline {subsection}{\numberline {90.5.2}Application of PCA in Regression Problems:}{956}{subsection.90.5.2}%
\contentsline {subsection}{\numberline {90.5.3}Scree Plot}{957}{subsection.90.5.3}%
\contentsline {subsection}{\numberline {90.5.4}Loading Scores (for PCA)}{957}{subsection.90.5.4}%
\contentsline {subsection}{\numberline {90.5.5}PCA for Car Sales Example}{958}{subsection.90.5.5}%
\contentsline {subsubsection}{\numberline {90.5.5.1}Computing the Principal Components}{958}{subsubsection.90.5.5.1}%
\contentsline {subsubsection}{\numberline {90.5.5.2}Loading Scores for PCA (10 Components)}{960}{subsubsection.90.5.5.2}%
\contentsline {subsection}{\numberline {90.5.6}Creating the Regression Model with Principal Components}{962}{subsection.90.5.6}%
\contentsline {subsection}{\numberline {90.5.7}Collinearity Diagnostics for PCA Regression Model}{963}{subsection.90.5.7}%
\contentsline {subsection}{\numberline {90.5.8}PCA: Creating the Regression Model with three Principle Components only}{965}{subsection.90.5.8}%
\contentsline {section}{\numberline {90.6}Addressing Multicollinearity and Latent Structure with Factor Analysis (FA)}{966}{section.90.6}%
\contentsline {subsection}{\numberline {90.6.1}Introduction to Factor Analysis}{966}{subsection.90.6.1}%
\contentsline {subsection}{\numberline {90.6.2}Determining the Number of Factors for Factor Analysis}{966}{subsection.90.6.2}%
\contentsline {subsection}{\numberline {90.6.3}Scree Plot for Factor Analysis}{967}{subsection.90.6.3}%
\contentsline {subsection}{\numberline {90.6.4}Factor Loadings}{968}{subsection.90.6.4}%
\contentsline {subsection}{\numberline {90.6.5}Factor Scores}{970}{subsection.90.6.5}%
\contentsline {subsection}{\numberline {90.6.6}Creating the Regression Model with Extracted Factors (from FA)}{971}{subsection.90.6.6}%
\contentsline {subsection}{\numberline {90.6.7}Factor Analysis: Creating the Regression Model with three Extracted Factors only}{974}{subsection.90.6.7}%
\contentsline {subsubsection}{\numberline {90.6.7.1}Setting Up the Regression Model with Reduced Factors}{974}{subsubsection.90.6.7.1}%
\contentsline {subsubsection}{\numberline {90.6.7.2}Comparison of Model Performance of the Reduced FA Model and the Full OLS Model}{975}{subsubsection.90.6.7.2}%
\contentsline {section}{\numberline {90.7}Summary: Comparing OLS, PCA, and Factor Analysis Models}{976}{section.90.7}%
\contentsline {subsection}{\numberline {90.7.1}Interpretation of the Regression Models}{976}{subsection.90.7.1}%
\contentsline {subsection}{\numberline {90.7.2}Differences Compared to the Standard OLS Model}{976}{subsection.90.7.2}%
\contentsline {subsection}{\numberline {90.7.3}Key Differences Between Loading Scores (PCA) and Factor Loadings (FA)}{977}{subsection.90.7.3}%
\contentsline {subsection}{\numberline {90.7.4}Advantages of Using PCA and FA}{978}{subsection.90.7.4}%
\contentsline {subsection}{\numberline {90.7.5}Disadvantages of Using PCA and FA}{979}{subsection.90.7.5}%
\contentsline {subsection}{\numberline {90.7.6}When to Use Which Method}{979}{subsection.90.7.6}%
\contentsline {section}{\numberline {90.8}Using Principal Components / Factors in Other Models}{979}{section.90.8}%
\contentsline {subsection}{\numberline {90.8.1}Random Forest Regressor with the Full Dataset}{980}{subsection.90.8.1}%
\contentsline {subsection}{\numberline {90.8.2}Random Forest Regressor with PCA Components}{980}{subsection.90.8.2}%
\contentsline {subsection}{\numberline {90.8.3}Random Forest Regressor with Extracted Factors (from FA)}{981}{subsection.90.8.3}%
\contentsline {subsection}{\numberline {90.8.4}Comparison of the Random Forest Models}{982}{subsection.90.8.4}%
\contentsline {section}{\numberline {90.9}Videos: Principal Component Analysis (PCA)}{983}{section.90.9}%
\contentsline {section}{\numberline {90.10}Jupyter Notebook}{984}{section.90.10}%
\contentsline {chapter}{\numberline {91}Regression}{985}{chapter.91}%
\contentsline {section}{\numberline {91.1}Supervised and Unsupervised Learning}{985}{section.91.1}%
\contentsline {subsection}{\numberline {91.1.1}Starting point}{985}{subsection.91.1.1}%
\contentsline {subsection}{\numberline {91.1.2}Philosophy}{985}{subsection.91.1.2}%
\contentsline {subsection}{\numberline {91.1.3}Supervised Learning}{985}{subsection.91.1.3}%
\contentsline {subsection}{\numberline {91.1.4}Unsupervised Learning}{986}{subsection.91.1.4}%
\contentsline {paragraph}{\numberline {91.1.4.0.1}Statistical Learning}{986}{paragraph.91.1.4.0.1}%
\contentsline {section}{\numberline {91.2}Linear Regression}{987}{section.91.2}%
\contentsline {subsection}{\numberline {91.2.1}The main ideas of fitting a line to data (The main ideas of least squares and linear regression.)}{987}{subsection.91.2.1}%
\contentsline {subsubsection}{\numberline {91.2.1.1}Linear Regression}{987}{subsubsection.91.2.1.1}%
\contentsline {subsubsection}{\numberline {91.2.1.2}Multiple Regression}{987}{subsubsection.91.2.1.2}%
\contentsline {subsubsection}{\numberline {91.2.1.3}A Gentle Introduction to Machine Learning}{987}{subsubsection.91.2.1.3}%
\contentsline {subsubsection}{\numberline {91.2.1.4}Regression Function}{987}{subsubsection.91.2.1.4}%
\contentsline {subsection}{\numberline {91.2.2}Optimal Predictor}{988}{subsection.91.2.2}%
\contentsline {subsubsection}{\numberline {91.2.2.1}Residuals, Reducible and Irreducible Error}{988}{subsubsection.91.2.2.1}%
\contentsline {subsubsection}{\numberline {91.2.2.2}Local Regression (Smoothing)}{988}{subsubsection.91.2.2.2}%
\contentsline {subsection}{\numberline {91.2.3}Curse of Dimensionality and Parametric Models}{988}{subsection.91.2.3}%
\contentsline {subsubsection}{\numberline {91.2.3.1}Trade-offs}{989}{subsubsection.91.2.3.1}%
\contentsline {subsection}{\numberline {91.2.4}Assessing Model Accuracy and Bias-Variance Trade-off}{990}{subsection.91.2.4}%
\contentsline {subsubsection}{\numberline {91.2.4.1}Bias-Variance Trade-off}{990}{subsubsection.91.2.4.1}%
\contentsline {section}{\numberline {91.3}Multiple Regression}{991}{section.91.3}%
\contentsline {section}{\numberline {91.4}R-squared}{991}{section.91.4}%
\contentsline {subsection}{\numberline {91.4.1}R-Squared in Simple Linear Regression}{991}{subsection.91.4.1}%
\contentsline {section}{\numberline {91.5}Assessing Confounding Effects in Multiple Regression}{992}{section.91.5}%
\contentsline {section}{\numberline {91.6}Cross-Validation}{995}{section.91.6}%
\contentsline {chapter}{\numberline {92}Classification}{997}{chapter.92}%
\contentsline {section}{\numberline {92.1}Classification: Some Details}{997}{section.92.1}%
\contentsline {section}{\numberline {92.2}k-Nearest Neighbor Classification}{998}{section.92.2}%
\contentsline {subsection}{\numberline {92.2.1}Minkowski Distance}{998}{subsection.92.2.1}%
\contentsline {section}{\numberline {92.3}Decision and Classification Trees}{999}{section.92.3}%
\contentsline {subsection}{\numberline {92.3.1}Decision Trees}{999}{subsection.92.3.1}%
\contentsline {subsubsection}{\numberline {92.3.1.1}\href {https://youtu.be/_L39rN6gz7Y?si=KtY-CsLGeAbIJN-f}{Decision and Classification Trees, Clearly Explained}}{999}{subsubsection.92.3.1.1}%
\contentsline {subsubsection}{\numberline {92.3.1.2}\href {https://youtu.be/wpNl-JwwplA?si=R7qiQ4rVzsrW1GAI}{StatQuest: Decision Trees, Part 2 - Feature Selection and Missing Data}}{999}{subsubsection.92.3.1.2}%
\contentsline {subsection}{\numberline {92.3.2}Regression Trees}{999}{subsection.92.3.2}%
\contentsline {subsubsection}{\numberline {92.3.2.1}\href {https://youtu.be/g9c66TUylZ4?si=aXOCqkDl-fGAFRx2}{Regression Trees, Clearly Explained!!!}}{999}{subsubsection.92.3.2.1}%
\contentsline {subsubsection}{\numberline {92.3.2.2}\href {https://youtu.be/D0efHEJsfHo?si=OKizIPtcrWDOSCRF}{How to Prune Regression Trees, Clearly Explained!!!}}{999}{subsubsection.92.3.2.2}%
\contentsline {section}{\numberline {92.4}The Confusion Matrix}{999}{section.92.4}%
\contentsline {subsection}{\numberline {92.4.1}Sensitivity and Specificity}{999}{subsection.92.4.1}%
\contentsline {section}{\numberline {92.5}Naive Bayes}{999}{section.92.5}%
\contentsline {section}{\numberline {92.6}Gaussian Naive Bayes}{999}{section.92.6}%
\contentsline {chapter}{\numberline {93}Clustering}{1001}{chapter.93}%
\contentsline {section}{\numberline {93.1}DBSCAN}{1001}{section.93.1}%
\contentsline {section}{\numberline {93.2}k-Means Clustering}{1001}{section.93.2}%
\contentsline {section}{\numberline {93.3}DDMO-Additional Videos}{1001}{section.93.3}%
\contentsline {section}{\numberline {93.4}DDMO-Exercises}{1002}{section.93.4}%
\contentsline {subsubsection}{\numberline {93.4.0.1}Covariance}{1004}{subsubsection.93.4.0.1}%
\contentsline {part}{\numberline {XV}Appendix}{1009}{part.15}%
\contentsline {chapter}{\numberline {94}Gaussian Processes---Some Background Information}{1011}{chapter.94}%
\contentsline {section}{\numberline {94.1}Gaussian Process Prior}{1011}{section.94.1}%
\contentsline {section}{\numberline {94.2}Covariance Function}{1011}{section.94.2}%
\contentsline {subsection}{\numberline {94.2.1}Positive Definiteness}{1013}{subsection.94.2.1}%
\contentsline {section}{\numberline {94.3}Construction of the Covariance Matrix}{1013}{section.94.3}%
\contentsline {section}{\numberline {94.4}Generation of Random Samples and Plotting the Realizations of the Random Function}{1016}{section.94.4}%
\contentsline {section}{\numberline {94.5}Properties of the 1d Example}{1018}{section.94.5}%
\contentsline {subsection}{\numberline {94.5.1}Several Bumps:}{1018}{subsection.94.5.1}%
\contentsline {subsection}{\numberline {94.5.2}Smoothness:}{1019}{subsection.94.5.2}%
\contentsline {subsection}{\numberline {94.5.3}Scale of Two:}{1019}{subsection.94.5.3}%
\contentsline {section}{\numberline {94.6}Jupyter Notebook}{1021}{section.94.6}%
\contentsline {chapter}{References}{1023}{chapter*.216}%
\providecommand \tocbasic@end@toc@file {}\tocbasic@end@toc@file