Skip to content

Commit 7b248c2

Browse files
bartzbeielsteinbartzbeielstein
committed
v0.6.11
cnn implementation
1 parent 4c705ab commit 7b248c2

10 files changed

Lines changed: 404 additions & 114 deletions

File tree

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
77

88
[project]
99
name = "spotPython"
10-
version = "0.6.10"
10+
version = "0.6.11"
1111
authors = [
1212
{ name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
1313
]

‎src/spotPython/budget/ocba.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@ def get_ocba(means, vars, delta) -> int32:
1616
variances, and incremental budget using the OCBA algorithm.
1717
1818
References:
19-
[1]: Chun-Hung Chen and Loo Hay Lee: Stochastic Simulation Optimization: An Optimal Computer Budget Allocation, pp. 49 and pp. 215
20-
[2]: C.S.M Currie and T. Monks: How to choose the best setup for a system. A tutorial for the Simulation Workshop 2021, see:
21-
[sw21_tutorial.ipynb](https://colab.research.google.com/github/TomMonks/sim-tools/blob/master/examples/sw21_tutorial.ipynb) and
22-
[sim-tools](https://github.com/TomMonks/sim-tools)
19+
[1]: Chun-Hung Chen and Loo Hay Lee: Stochastic Simulation Optimization: An Optimal Computer Budget Allocation,
20+
pp. 49 and pp. 215
21+
[2]: C.S.M Currie and T. Monks: How to choose the best setup for a system.
22+
A tutorial for the Simulation Workshop 2021, see:
23+
https://colab.research.google.com/github/TomMonks/sim-tools/blob/master/examples/sw21_tutorial.ipynb
24+
and
25+
https://github.com/TomMonks/sim-tools
2326
2427
Args:
2528
means (numpy.array): An array of means.

‎src/spotPython/build/kriging.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,8 @@ def __init__(
158158
159159
References:
160160
161-
[[1](https://scikit-learn.org/stable/auto_examples/gaussian_process/plot_gpr_noisy_targets.html)] scikit-learn: Gaussian Processes regression: basic introductory example
161+
[[1](https://scikit-learn.org/stable/auto_examples/gaussian_process/plot_gpr_noisy_targets.html)]
162+
scikit-learn: Gaussian Processes regression: basic introductory example
162163
163164
"""
164165
super().__init__(name, seed, log_level)

‎src/spotPython/fun/hyperlightning.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import logging
2+
import numpy as np
3+
from numpy.random import default_rng
4+
from numpy import array
5+
6+
# here we use train_model from spotPython.light.trainmodel
7+
# and not from spot.light.traintest:
8+
from spotPython.light.trainmodel import train_model
9+
from spotPython.hyperparameters.values import (
10+
assign_values,
11+
generate_one_config_from_var_dict,
12+
)
13+
14+
logger = logging.getLogger(__name__)
15+
py_handler = logging.FileHandler(f"{__name__}.log", mode="w")
16+
py_formatter = logging.Formatter("%(name)s %(asctime)s %(levelname)s %(message)s")
17+
py_handler.setFormatter(py_formatter)
18+
logger.addHandler(py_handler)
19+
20+
21+
class HyperLightning:
22+
"""
23+
Hyperparameter Tuning for Lightning.
24+
"""
25+
26+
def __init__(self, seed: int = 126, log_level: int = 50) -> None:
27+
self.seed = seed
28+
self.rng = default_rng(seed=self.seed)
29+
self.fun_control = {
30+
"seed": None,
31+
"data": None,
32+
"step": 10_000,
33+
"horizon": None,
34+
"grace_period": None,
35+
"metric_river": None,
36+
"metric_sklearn": None,
37+
"weights": array([1, 0, 0]),
38+
"weight_coeff": 0.0,
39+
"log_level": log_level,
40+
"var_name": [],
41+
"var_type": [],
42+
}
43+
self.log_level = self.fun_control["log_level"]
44+
logger.setLevel(self.log_level)
45+
logger.info(f"Starting the logger at level {self.log_level} for module {__name__}:")
46+
47+
def check_X_shape(self, X: np.ndarray) -> np.ndarray:
48+
"""
49+
Checks the shape of the input array X and raises an exception if it is not valid.
50+
51+
Args:
52+
X (np.ndarray):
53+
input array.
54+
55+
Returns:
56+
np.ndarray:
57+
input array with valid shape.
58+
59+
Raises:
60+
Exception:
61+
if the shape of the input array is not valid.
62+
63+
Examples:
64+
>>> hyper_light = HyperLight(seed=126, log_level=50)
65+
>>> X = np.array([[1, 2], [3, 4]])
66+
>>> hyper_light.check_X_shape(X)
67+
array([[1, 2],
68+
[3, 4]])
69+
"""
70+
try:
71+
X.shape[1]
72+
except ValueError:
73+
X = np.array([X])
74+
if X.shape[1] != len(self.fun_control["var_name"]):
75+
raise Exception("Invalid shape of input array X.")
76+
return X
77+
78+
def fun(self, X: np.ndarray, fun_control: dict = None) -> np.ndarray:
79+
"""
80+
Evaluates the function for the given input array X and control parameters.
81+
82+
Args:
83+
X (np.ndarray):
84+
input array.
85+
fun_control (dict):
86+
dictionary containing control parameters for the hyperparameter tuning.
87+
88+
Returns:
89+
(np.ndarray):
90+
array containing the evaluation results.
91+
92+
Examples:
93+
>>> hyper_light = HyperLight(seed=126, log_level=50)
94+
X = np.array([[1, 2], [3, 4]])
95+
fun_control = {"weights": np.array([1, 0, 0])}
96+
hyper_light.fun(X, fun_control)
97+
array([nan, nan])
98+
"""
99+
z_res = np.array([], dtype=float)
100+
if fun_control is not None:
101+
self.fun_control.update(fun_control)
102+
self.check_X_shape(X)
103+
var_dict = assign_values(X, self.fun_control["var_name"])
104+
# type information and transformations are considered in generate_one_config_from_var_dict:
105+
for config in generate_one_config_from_var_dict(var_dict, self.fun_control):
106+
logger.debug(f"\nconfig: {config}")
107+
# extract parameters like epochs, batch_size, lr, etc. from config
108+
# config_id = generate_config_id(config)
109+
try:
110+
print("fun: Calling train_model")
111+
df_eval = train_model(config, self.fun_control)
112+
print("fun: train_model returned")
113+
except Exception as err:
114+
logger.error(f"Error in fun(). Call to train_model failed. {err=}, {type(err)=}")
115+
logger.error("Setting df_eval to np.nan")
116+
df_eval = np.nan
117+
z_val = self.fun_control["weights"] * df_eval
118+
z_res = np.append(z_res, z_val)
119+
return z_res

‎src/spotPython/light/cifar10/cifar10datamodule.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,7 @@ def setup(self, stage: Optional[str] = None) -> None:
5353
# https://lightning.ai/docs/pytorch/latest/notebooks/course_UvA-DL/04-inception-resnet-densenet.html
5454
DATA_MEANS = (0.49139968, 0.48215841, 0.44653091)
5555
DATA_STDS = (0.24703223, 0.24348513, 0.26158784)
56-
transform = transforms.Compose(
57-
[transforms.ToTensor(), transforms.Normalize(DATA_MEANS, DATA_STDS)]
58-
)
56+
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(DATA_MEANS, DATA_STDS)])
5957
# Assign train/val datasets for use in dataloaders
6058
if stage == "fit" or stage is None:
6159
data_full = CIFAR10(root=self.data_dir, train=True, transform=transform)
@@ -75,8 +73,9 @@ def train_dataloader(self) -> DataLoader:
7573
7674
"""
7775
print("train_dataloader: self.batch_size", self.batch_size)
78-
return DataLoader(self.data_train, batch_size=self.batch_size, shuffle=True, drop_last = True,
79-
num_workers=self.num_workers)
76+
return DataLoader(
77+
self.data_train, batch_size=self.batch_size, shuffle=True, drop_last=True, num_workers=self.num_workers
78+
)
8079

8180
def val_dataloader(self) -> DataLoader:
8281
"""
@@ -88,8 +87,9 @@ def val_dataloader(self) -> DataLoader:
8887
8988
"""
9089
print("val_dataloader: self.batch_size", self.batch_size)
91-
return DataLoader(self.data_val, batch_size=self.batch_size, shuffle=False,
92-
drop_last=False, num_workers=self.num_workers)
90+
return DataLoader(
91+
self.data_val, batch_size=self.batch_size, shuffle=False, drop_last=False, num_workers=self.num_workers
92+
)
9393

9494
def test_dataloader(self) -> DataLoader:
9595
"""
@@ -101,5 +101,6 @@ def test_dataloader(self) -> DataLoader:
101101
102102
"""
103103
print("train_data_loader: self.batch_size", self.batch_size)
104-
return DataLoader(self.data_test, batch_size=self.batch_size, shuffle=False,
105-
drop_last=False, num_workers=self.num_workers)
104+
return DataLoader(
105+
self.data_test, batch_size=self.batch_size, shuffle=False, drop_last=False, num_workers=self.num_workers
106+
)

‎src/spotPython/light/cnn/googlenet.py‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,48 @@
1-
import os
2-
import urllib.request
31
from types import SimpleNamespace
4-
from urllib.error import HTTPError
5-
6-
import lightning as L
7-
import matplotlib
8-
import matplotlib.pyplot as plt
9-
import matplotlib_inline.backend_inline
10-
import numpy as np
11-
import seaborn as sns
12-
import tabulate
13-
import torch
142
import torch.nn as nn
15-
import torch.optim as optim
16-
import torch.utils.data as data
17-
import torchvision
3+
from spotPython.light.cnn.inceptionblock import InceptionBlock
184

19-
from IPython.display import HTML, display
20-
from lightning.pytorch.callbacks import LearningRateMonitor, ModelCheckpoint
21-
from PIL import Image
22-
from torchvision import transforms
23-
from torchvision.datasets import CIFAR10
245

25-
matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf") # For export
26-
matplotlib.rcParams["lines.linewidth"] = 2.0
27-
sns.reset_orig()
6+
class GoogleNet(nn.Module):
7+
"""GoogleNet architecture
288
9+
Args:
10+
num_classes (int):
11+
Number of classes for the classification task. Defaults to 10.
12+
act_fn_name (str):
13+
Name of the activation function. Defaults to "relu".
14+
**kwargs:
15+
Additional keyword arguments.
2916
30-
class GoogleNet(nn.Module):
31-
def __init__(self, num_classes=10, act_fn_name="relu", **kwargs):
17+
Attributes:
18+
hparams (SimpleNamespace):
19+
Namespace containing the hyperparameters.
20+
input_net (nn.Sequential):
21+
Input network.
22+
inception_blocks (nn.Sequential):
23+
Inception blocks.
24+
output_net (nn.Sequential):
25+
Output network.
26+
27+
Returns:
28+
(torch.Tensor):
29+
Output tensor of the GoogleNet architecture
30+
31+
Examples:
32+
>>> from spotPython.light.cnn.googlenet import GoogleNet
33+
import torch
34+
import torch.nn as nn
35+
model = GoogleNet()
36+
x = torch.randn(1, 3, 32, 32)
37+
y = model(x)
38+
y.shape
39+
torch.Size([1, 10])
40+
"""
41+
42+
def __init__(self, num_classes: int = 10, act_fn_name: str = "relu", **kwargs):
3243
super().__init__()
44+
# TODO: Replace this by act_fn handlers specified in the config file:
45+
act_fn_by_name = {"tanh": nn.Tanh, "relu": nn.ReLU, "leakyrelu": nn.LeakyReLU, "gelu": nn.GELU}
3346
self.hparams = SimpleNamespace(
3447
num_classes=num_classes, act_fn_name=act_fn_name, act_fn=act_fn_by_name[act_fn_name]
3548
)
@@ -100,7 +113,7 @@ def _create_network(self):
100113
)
101114

102115
def _init_params(self):
103-
# Based on our discussion in Tutorial 4, we should initialize the
116+
# We should initialize the
104117
# convolutions according to the activation function
105118
for m in self.modules():
106119
if isinstance(m, nn.Conv2d):
@@ -113,4 +126,4 @@ def forward(self, x):
113126
x = self.input_net(x)
114127
x = self.inception_blocks(x)
115128
x = self.output_net(x)
116-
return x
129+
return x

‎src/spotPython/light/cnn/inceptionblock.py‎

Lines changed: 41 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,50 @@
1-
import os
2-
import urllib.request
3-
from types import SimpleNamespace
4-
from urllib.error import HTTPError
5-
6-
import lightning as L
7-
import matplotlib
8-
import matplotlib.pyplot as plt
9-
import matplotlib_inline.backend_inline
10-
import numpy as np
11-
import seaborn as sns
12-
import tabulate
131
import torch
142
import torch.nn as nn
15-
import torch.optim as optim
16-
import torch.utils.data as data
17-
import torchvision
18-
19-
from IPython.display import HTML, display
20-
from lightning.pytorch.callbacks import LearningRateMonitor, ModelCheckpoint
21-
from PIL import Image
22-
from torchvision import transforms
23-
from torchvision.datasets import CIFAR10
243

25-
matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf") # For export
26-
matplotlib.rcParams["lines.linewidth"] = 2.0
27-
sns.reset_orig()
28-
29-
# PyTorch
30-
# Torchvision
314

325
class InceptionBlock(nn.Module):
336
def __init__(self, c_in, c_red: dict, c_out: dict, act_fn):
347
"""
35-
Inputs:
36-
c_in - Number of input feature maps from the previous layers
37-
c_red - Dictionary with keys "3x3" and "5x5" specifying the output of the dimensionality reducing 1x1 convolutions
38-
c_out - Dictionary with keys "1x1", "3x3", "5x5", and "max"
39-
act_fn - Activation class constructor (e.g. nn.ReLU)
8+
Inception block as used in GoogLeNet.
9+
10+
Description from
11+
[P. Lippe:INCEPTION, RESNET AND DENSENET](https://lightning.ai/docs/pytorch/stable/)
12+
An Inception block applies four convolution blocks separately on the same feature map:
13+
a 1x1, 3x3, and 5x5 convolution, and a max pool operation.
14+
This allows the network to look at the same data with different receptive fields.
15+
Of course, learning only 5x5 convolution would be theoretically more powerful.
16+
However, this is not only more computation and memory heavy but also tends to overfit much easier.
17+
The 1x1 convolutions are used to reduce the number of input channels to the 3x3 and 5x5 convolutions,
18+
which reduces the number of parameters and computation.
19+
20+
Args:
21+
c_in:
22+
Number of input feature maps from the previous layers
23+
c_red:
24+
Dictionary with keys "3x3" and "5x5" specifying
25+
the output of the dimensionality reducing 1x1 convolutions
26+
c_out:
27+
Dictionary with keys "1x1", "3x3", "5x5", and "max"
28+
act_fn:
29+
Activation class constructor (e.g. nn.ReLU)
30+
31+
Returns:
32+
torch.Tensor:
33+
Output tensor of the inception block
34+
35+
Examples:
36+
>>> from spotPython.light.cnn.googlenet import InceptionBlock
37+
import torch
38+
import torch.nn as nn
39+
block = InceptionBlock(3,
40+
{"3x3": 32, "5x5": 16},
41+
{"1x1": 16, "3x3": 32, "5x5": 8, "max": 8},
42+
nn.ReLU)
43+
x = torch.randn(1, 3, 32, 32)
44+
y = block(x)
45+
y.shape
46+
torch.Size([1, 64, 32, 32])
47+
4048
"""
4149
super().__init__()
4250

@@ -79,4 +87,4 @@ def forward(self, x):
7987
x_5x5 = self.conv_5x5(x)
8088
x_max = self.max_pool(x)
8189
x_out = torch.cat([x_1x1, x_3x3, x_5x5, x_max], dim=1)
82-
return x_out
90+
return x_out

0 commit comments

Comments
 (0)