light 2

Author: bartzbeielstein

src/spotPython/data/lightning_hyper_dict.json

@@ -0,0 +1,111 @@
+{    "GoogleNet":
+    {
+    "act_fn": {
+            "levels": ["Sigmoid",
+                       "Tanh",
+                       "ReLU",
+                       "LeakyReLU",
+                       "ELU",
+                       "Swish"],
+            "type": "factor",
+            "default": "ReLU",
+            "transform": "None",
+            "class_name": "spotPython.torch.activation",
+            "core_model_parameter_type": "instance",
+            "lower": 0,
+            "upper": 5},
+    "optimizer_name": {
+        "levels": [
+                   "Adam"
+                ],
+        "type": "factor",
+        "default": "Adam",
+        "transform": "None",
+        "class_name": "torch.optim",
+        "core_model_parameter_type": "str",
+        "lower": 0,
+        "upper": 0}
+    },
+    "NetLinearBase":
+    {
+    "l1": {
+        "type": "int",
+        "default": 3,
+        "transform": "transform_power_2_int",
+        "lower": 3,
+        "upper": 8},
+    "epochs": {
+            "type": "int",
+            "default": 4,
+            "transform": "transform_power_2_int",
+            "lower": 4,
+            "upper": 9},
+    "batch_size": {
+                "type": "int",
+                "default": 4,
+                "transform": "transform_power_2_int",
+                "lower": 1,
+                "upper": 4},
+    "act_fn": {
+            "levels": ["Sigmoid",
+                       "Tanh",
+                       "ReLU",
+                       "LeakyReLU",
+                       "ELU",
+                       "Swish"],
+            "type": "factor",
+            "default": "ReLU",
+            "transform": "None",
+            "class_name": "spotPython.torch.activation",
+            "core_model_parameter_type": "instance()",
+            "lower": 0,
+            "upper": 5},
+    "optimizer": {
+        "levels": ["Adadelta",
+                   "Adagrad",
+                   "Adam",
+                   "AdamW",
+                   "SparseAdam",
+                   "Adamax",
+                   "ASGD",
+                   "NAdam",
+                   "RAdam",
+                   "RMSprop",
+                   "Rprop",
+                   "SGD"],
+        "type": "factor",
+        "default": "SGD",
+        "transform": "None",
+        "class_name": "torch.optim",
+        "core_model_parameter_type": "str",
+        "lower": 0,
+        "upper": 11},
+    "dropout_prob": {
+        "type": "float",
+        "default": 0.01,
+        "transform": "None",
+        "lower": 0.0,
+        "upper": 0.25},
+    "lr_mult": {
+        "type": "float",
+        "default": 1.0,
+        "transform": "None",
+        "lower": 0.1,
+        "upper": 10.0},
+    "patience": {
+        "type": "int",
+        "default": 2,
+        "transform": "transform_power_2_int",
+        "lower": 2,
+        "upper": 6
+    },
+    "initialization": {
+        "levels": ["Default", "Kaiming", "Xavier"],
+        "type": "factor",
+        "default": "Default",
+        "transform": "None",
+        "core_model_parameter_type": "str",
+        "lower": 0,
+        "upper": 2}
+    }
+}

src/spotPython/data/lightning_hyper_dict.py

@@ -0,0 +1,39 @@
+import json
+from spotPython.data import base
+
+
+class LightningHyperDict(base.FileConfig):
+    """Lightning hyperparameter dictionary.
+
+    This class extends the FileConfig class to provide a dictionary for storing hyperparameters.
+
+    Attributes:
+        filename (str):
+            The name of the file where the hyperparameters are stored.
+    """
+
+    def __init__(self):
+        """Initialize the LightHyperDict object.
+
+        Examples:
+            >>> lhd = LightHyperDict()
+        """
+        super().__init__(
+            filename="lightning_hyper_dict.json",
+        )
+
+    def load(self) -> dict:
+        """Load the hyperparameters from the file.
+
+        Returns:
+            dict: A dictionary containing the hyperparameters.
+
+        Examples:
+            >>> lhd = LightHyperDict()
+            >>> hyperparams = lhd.load()
+            >>> print(hyperparams)
+            {'learning_rate': 0.001, 'batch_size': 32, 'epochs': 10}
+        """
+        with open(self.path, "r") as f:
+            d = json.load(f)
+        return d

src/spotPython/fun/hyperlightning.py

@@ -21,6 +21,21 @@
 class HyperLightning:
     """
     Hyperparameter Tuning for Lightning.
+
+    Args:
+        seed (int): seed for the random number generator. See Numpy Random Sampling.
+        log_level (int): log level for the logger.
+
+    Attributes:
+        seed (int): seed for the random number generator.
+        rng (Generator): random number generator.
+        fun_control (dict): dictionary containing control parameters for the hyperparameter tuning.
+        log_level (int): log level for the logger.
+
+    Examples:
+        >>> hyper_light = HyperLight(seed=126, log_level=50)
+        >>> print(hyper_light.seed)
+        126
     """
 
     def __init__(self, seed: int = 126, log_level: int = 50) -> None:
@@ -90,11 +105,33 @@ def fun(self, X: np.ndarray, fun_control: dict = None) -> np.ndarray:
                 array containing the evaluation results.
 
         Examples:
-            >>> hyper_light = HyperLight(seed=126, log_level=50)
-                X = np.array([[1, 2], [3, 4]])
-                fun_control = {"weights": np.array([1, 0, 0])}
-                hyper_light.fun(X, fun_control)
-                array([nan, nan])
+            >>> MAX_TIME = 1
+                INIT_SIZE = 5
+                WORKERS = 0
+                PREFIX="TEST"
+                from spotPython.utils.init import fun_control_init
+                from spotPython.utils.file import get_experiment_name, get_spot_tensorboard_path
+                from spotPython.utils.device import getDevice
+                experiment_name = get_experiment_name(prefix=PREFIX)
+                fun_control = fun_control_init(
+                    spot_tensorboard_path=get_spot_tensorboard_path(experiment_name),
+                    num_workers=WORKERS,
+                    device=getDevice(),
+                    _L_in=3,
+                    _L_out=10,
+                    TENSORBOARD_CLEAN=True)
+                from spotPython.light.cnn.googlenet import GoogleNet
+                from spotPython.data.lightning_hyper_dict import LightningHyperDict
+                from spotPython.hyperparameters.values import add_core_model_to_fun_control
+                add_core_model_to_fun_control(core_model=GoogleNet,
+                                            fun_control=fun_control,
+                                            hyper_dict= LightningHyperDict)
+                from spotPython.hyperparameters.values import get_default_hyperparameters_as_array
+                X_start = get_default_hyperparameters_as_array(fun_control)
+                from spotPython.fun.hyperlightning import HyperLightning
+                hyper_light = HyperLightning(seed=126, log_level=50)
+                hyper_light.fun(X=X_start, fun_control=fun_control)
+
         """
         z_res = np.array([], dtype=float)
         if fun_control is not None:
@@ -104,6 +141,8 @@ def fun(self, X: np.ndarray, fun_control: dict = None) -> np.ndarray:
         # type information and transformations are considered in generate_one_config_from_var_dict:
         for config in generate_one_config_from_var_dict(var_dict, self.fun_control):
             logger.debug(f"\nconfig: {config}")
+            print(f"\ncore_model: {fun_control['core_model']}")
+            print(f"config: {config}")
             # extract parameters like epochs, batch_size, lr, etc. from config
             # config_id = generate_config_id(config)
             try:

src/spotPython/light/cnn/netcnnbase.py

@@ -1,9 +1,13 @@
 import lightning as L
 import torch
 from torch import nn
-from spotPython.light.utils import create_model
+
+# from spotPython.light.utils import create_model
 import torch.optim as optim
 
+# from spotPython.light.cnn.googlenet import GoogleNet
+import spotPython.light.cnn.googlenet
+
 
 class NetCNNBase(L.LightningModule):
     def __init__(self, config, fun_control):
@@ -31,11 +35,30 @@ def __init__(self, config, fun_control):
                 torch.Size([1, 10])
 
         """
+        print("NetCNNBase: Starting")
+        print(f"NetCNNBase: config: {config}")
+        print(f"NetCNNBase: fun_control['core_model']: {fun_control['core_model']}")
+        config = {
+            "c_in": 3,
+            "c_out": 10,
+            "act_fn": nn.ReLU,
+            "optimizer_name": "Adam",
+            "optimizer_hparams": {"lr": 1e-3, "weight_decay": 1e-4},
+        }
+        print("fun_control['core_model']: ", fun_control["core_model"])
+        print("fun_control['core_model'].type: ", fun_control["core_model"].type)
+        # fun_control = {"core_model": GoogleNet}
+        fun_control = {"core_model": spotPython.light.cnn.googlenet.GoogleNet}
         super().__init__()
         # Exports the hyperparameters to a YAML file, and create "self.hparams" namespace
-        self.save_hyperparameters()
+        self.save_hyperparameters()  # "fun_control" is not a hyperparameter )
+        print(f"config: {config}")
         # Create model
-        self.model = create_model(config, fun_control)
+        print("Creating model")
+        # self.model = create_model(config, fun_control)
+        self.model = fun_control["core_model"](**config)
+        print("Model created")
+        print(f"self.model: {self.model}")
         # Create loss module
         self.loss_module = nn.CrossEntropyLoss()
         # Example input for visualizing the graph in Tensorboard

src/spotPython/light/trainmodel.py

@@ -39,6 +39,8 @@ def train_model(config: dict, fun_control: dict):
             {'test': 0.8772, 'val': 0.8772}
 
     """
+    print("train_model: Starting")
+    print(f"train_model: config: {config}")
     save_name = "saved_models"
     # Create PyTorch Lightning data loaders
     CHECKPOINT_PATH = os.environ.get("PATH_CHECKPOINT", "saved_models/ConvNets")
@@ -85,13 +87,14 @@ def train_model(config: dict, fun_control: dict):
     # END TODO
 
     # Create a PyTorch Lightning trainer with the generation callback
+    print("train_model: Creating trainer")
     trainer = L.Trainer(
         default_root_dir=os.path.join(CHECKPOINT_PATH, save_name),  # Where to save models
         # We run on a single GPU (if possible)
         accelerator="auto",
         devices=1,
         # How many epochs to train for if no patience is set
-        max_epochs=180,
+        max_epochs=4,
         callbacks=[
             ModelCheckpoint(
                 save_weights_only=True, mode="max", monitor="val_acc"
@@ -101,6 +104,7 @@ def train_model(config: dict, fun_control: dict):
     )  # In case your notebook crashes due to the progress bar, consider increasing the refresh rate
     trainer.logger._log_graph = True  # If True, we plot the computation graph in tensorboard
     trainer.logger._default_hp_metric = None  # Optional logging argument that we don't need
+    print("train_model: Created trainer")
 
     # Check whether pretrained model exists. If yes, load it and skip training
     pretrained_filename = os.path.join(CHECKPOINT_PATH, save_name + ".ckpt")
@@ -110,6 +114,7 @@ def train_model(config: dict, fun_control: dict):
         model = NetCNNBase.load_from_checkpoint(pretrained_filename)
     else:
         L.seed_everything(42)  # To be reproducable
+        print("train_model: Creating model")
         model = NetCNNBase(config=config, fun_control=fun_control)  # Create model
         trainer.fit(model, train_loader, val_loader)
         model = NetCNNBase.load_from_checkpoint(