v0.6.10

bartzbeielstein · bartzbeielstein · commit 68099874cd79 · 2023-07-24T16:45:33.000+02:00
docs
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.6.8"
+version = "0.6.10"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]
diff --git a/src/spotPython/light/cifar10datamodule.py b/src/spotPython/light/cifar10datamodule.py
@@ -67,7 +67,8 @@ def train_dataloader(self) -> DataLoader:
 
         """
         print("train_dataloader: self.batch_size", self.batch_size)
-        return DataLoader(self.data_train, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers)
+        return DataLoader(self.data_train, batch_size=self.batch_size, shuffle=True, drop_last = True,
+                          num_workers=self.num_workers)
 
     def val_dataloader(self) -> DataLoader:
         """
@@ -79,7 +80,8 @@ def val_dataloader(self) -> DataLoader:
 
         """
         print("val_dataloader: self.batch_size", self.batch_size)
-        return DataLoader(self.data_val, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers)
+        return DataLoader(self.data_val, batch_size=self.batch_size, shuffle=False,
+                          drop_last=False, num_workers=self.num_workers)
 
     def test_dataloader(self) -> DataLoader:
         """
@@ -91,4 +93,5 @@ def test_dataloader(self) -> DataLoader:
 
         """
         print("train_data_loader: self.batch_size", self.batch_size)
-        return DataLoader(self.data_test, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers)
+        return DataLoader(self.data_test, batch_size=self.batch_size, shuffle=False,
+                          drop_last=False, num_workers=self.num_workers)
diff --git a/src/spotPython/light/cnn/googlenet.py b/src/spotPython/light/cnn/googlenet.py
@@ -0,0 +1,117 @@
+import os
+import urllib.request
+from types import SimpleNamespace
+from urllib.error import HTTPError
+
+import lightning as L
+import matplotlib
+import matplotlib.pyplot as plt
+import matplotlib_inline.backend_inline
+import numpy as np
+import seaborn as sns
+import tabulate
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.utils.data as data
+import torchvision
+
+%matplotlib inline
+from IPython.display import HTML, display
+from lightning.pytorch.callbacks import LearningRateMonitor, ModelCheckpoint
+from PIL import Image
+from torchvision import transforms
+from torchvision.datasets import CIFAR10
+
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
+matplotlib.rcParams["lines.linewidth"] = 2.0
+sns.reset_orig()
+
+
+class GoogleNet(nn.Module):
+    def __init__(self, num_classes=10, act_fn_name="relu", **kwargs):
+        super().__init__()
+        self.hparams = SimpleNamespace(
+            num_classes=num_classes, act_fn_name=act_fn_name, act_fn=act_fn_by_name[act_fn_name]
+        )
+        self._create_network()
+        self._init_params()
+
+    def _create_network(self):
+        # A first convolution on the original image to scale up the channel size
+        self.input_net = nn.Sequential(
+            nn.Conv2d(3, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), self.hparams.act_fn()
+        )
+        # Stacking inception blocks
+        self.inception_blocks = nn.Sequential(
+            InceptionBlock(
+                64,
+                c_red={"3x3": 32, "5x5": 16},
+                c_out={"1x1": 16, "3x3": 32, "5x5": 8, "max": 8},
+                act_fn=self.hparams.act_fn,
+            ),
+            InceptionBlock(
+                64,
+                c_red={"3x3": 32, "5x5": 16},
+                c_out={"1x1": 24, "3x3": 48, "5x5": 12, "max": 12},
+                act_fn=self.hparams.act_fn,
+            ),
+            nn.MaxPool2d(3, stride=2, padding=1),  # 32x32 => 16x16
+            InceptionBlock(
+                96,
+                c_red={"3x3": 32, "5x5": 16},
+                c_out={"1x1": 24, "3x3": 48, "5x5": 12, "max": 12},
+                act_fn=self.hparams.act_fn,
+            ),
+            InceptionBlock(
+                96,
+                c_red={"3x3": 32, "5x5": 16},
+                c_out={"1x1": 16, "3x3": 48, "5x5": 16, "max": 16},
+                act_fn=self.hparams.act_fn,
+            ),
+            InceptionBlock(
+                96,
+                c_red={"3x3": 32, "5x5": 16},
+                c_out={"1x1": 16, "3x3": 48, "5x5": 16, "max": 16},
+                act_fn=self.hparams.act_fn,
+            ),
+            InceptionBlock(
+                96,
+                c_red={"3x3": 32, "5x5": 16},
+                c_out={"1x1": 32, "3x3": 48, "5x5": 24, "max": 24},
+                act_fn=self.hparams.act_fn,
+            ),
+            nn.MaxPool2d(3, stride=2, padding=1),  # 16x16 => 8x8
+            InceptionBlock(
+                128,
+                c_red={"3x3": 48, "5x5": 16},
+                c_out={"1x1": 32, "3x3": 64, "5x5": 16, "max": 16},
+                act_fn=self.hparams.act_fn,
+            ),
+            InceptionBlock(
+                128,
+                c_red={"3x3": 48, "5x5": 16},
+                c_out={"1x1": 32, "3x3": 64, "5x5": 16, "max": 16},
+                act_fn=self.hparams.act_fn,
+            ),
+        )
+        # Mapping to classification output
+        self.output_net = nn.Sequential(
+            nn.AdaptiveAvgPool2d((1, 1)), nn.Flatten(), nn.Linear(128, self.hparams.num_classes)
+        )
+
+    def _init_params(self):
+        # Based on our discussion in Tutorial 4, we should initialize the
+        # convolutions according to the activation function
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, nonlinearity=self.hparams.act_fn_name)
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+    def forward(self, x):
+        x = self.input_net(x)
+        x = self.inception_blocks(x)
+        x = self.output_net(x)
+        return x
diff --git a/src/spotPython/light/cnn/inceptionblock.py b/src/spotPython/light/cnn/inceptionblock.py
@@ -0,0 +1,82 @@
+import os
+import urllib.request
+from types import SimpleNamespace
+from urllib.error import HTTPError
+
+import lightning as L
+import matplotlib
+import matplotlib.pyplot as plt
+import matplotlib_inline.backend_inline
+import numpy as np
+import seaborn as sns
+import tabulate
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.utils.data as data
+import torchvision
+
+from IPython.display import HTML, display
+from lightning.pytorch.callbacks import LearningRateMonitor, ModelCheckpoint
+from PIL import Image
+from torchvision import transforms
+from torchvision.datasets import CIFAR10
+
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
+matplotlib.rcParams["lines.linewidth"] = 2.0
+sns.reset_orig()
+
+# PyTorch
+# Torchvision
+
+class InceptionBlock(nn.Module):
+    def __init__(self, c_in, c_red: dict, c_out: dict, act_fn):
+        """
+        Inputs:
+            c_in - Number of input feature maps from the previous layers
+            c_red - Dictionary with keys "3x3" and "5x5" specifying the output of the dimensionality reducing 1x1 convolutions
+            c_out - Dictionary with keys "1x1", "3x3", "5x5", and "max"
+            act_fn - Activation class constructor (e.g. nn.ReLU)
+        """
+        super().__init__()
+
+        # 1x1 convolution branch
+        self.conv_1x1 = nn.Sequential(
+            nn.Conv2d(c_in, c_out["1x1"], kernel_size=1), nn.BatchNorm2d(c_out["1x1"]), act_fn()
+        )
+
+        # 3x3 convolution branch
+        self.conv_3x3 = nn.Sequential(
+            nn.Conv2d(c_in, c_red["3x3"], kernel_size=1),
+            nn.BatchNorm2d(c_red["3x3"]),
+            act_fn(),
+            nn.Conv2d(c_red["3x3"], c_out["3x3"], kernel_size=3, padding=1),
+            nn.BatchNorm2d(c_out["3x3"]),
+            act_fn(),
+        )
+
+        # 5x5 convolution branch
+        self.conv_5x5 = nn.Sequential(
+            nn.Conv2d(c_in, c_red["5x5"], kernel_size=1),
+            nn.BatchNorm2d(c_red["5x5"]),
+            act_fn(),
+            nn.Conv2d(c_red["5x5"], c_out["5x5"], kernel_size=5, padding=2),
+            nn.BatchNorm2d(c_out["5x5"]),
+            act_fn(),
+        )
+
+        # Max-pool branch
+        self.max_pool = nn.Sequential(
+            nn.MaxPool2d(kernel_size=3, padding=1, stride=1),
+            nn.Conv2d(c_in, c_out["max"], kernel_size=1),
+            nn.BatchNorm2d(c_out["max"]),
+            act_fn(),
+        )
+
+    def forward(self, x):
+        x_1x1 = self.conv_1x1(x)
+        x_3x3 = self.conv_3x3(x)
+        x_5x5 = self.conv_5x5(x)
+        x_max = self.max_pool(x)
+        x_out = torch.cat([x_1x1, x_3x3, x_5x5, x_max], dim=1)
+        return x_out
diff --git a/src/spotPython/light/cnn/netcnnbase.py b/src/spotPython/light/cnn/netcnnbase.py
@@ -0,0 +1,106 @@
+import lightning as L
+import torch
+from torch import nn
+
+# import torchmetrics
+import torch.nn.functional as F
+from torchmetrics.functional import accuracy
+from spotPython.hyperparameters.optimizer import optimizer_handler
+
+
+import os
+import urllib.request
+from types import SimpleNamespace
+from urllib.error import HTTPError
+
+import lightning as L
+import matplotlib
+import matplotlib.pyplot as plt
+import matplotlib_inline.backend_inline
+import numpy as np
+import seaborn as sns
+import tabulate
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.utils.data as data
+import torchvision
+
+%matplotlib inline
+from IPython.display import HTML, display
+from lightning.pytorch.callbacks import LearningRateMonitor, ModelCheckpoint
+from PIL import Image
+from torchvision import transforms
+from torchvision.datasets import CIFAR10
+
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
+matplotlib.rcParams["lines.linewidth"] = 2.0
+sns.reset_orig()
+
+# PyTorch
+# Torchvision
+
+
+class NetCNNBase(L.LightningModule):
+    def __init__(self, model_name, model_hparams, optimizer_name, optimizer_hparams):
+        """
+        Inputs:
+            model_name - Name of the model/CNN to run. Used for creating the model (see function below)
+            model_hparams - Hyperparameters for the model, as dictionary.
+            optimizer_name - Name of the optimizer to use. Currently supported: Adam, SGD
+            optimizer_hparams - Hyperparameters for the optimizer, as dictionary. This includes learning rate, weight decay, etc.
+        """
+        super().__init__()
+        # Exports the hyperparameters to a YAML file, and create "self.hparams" namespace
+        self.save_hyperparameters()
+        # Create model
+        self.model = create_model(model_name, model_hparams)
+        # Create loss module
+        self.loss_module = nn.CrossEntropyLoss()
+        # Example input for visualizing the graph in Tensorboard
+        self.example_input_array = torch.zeros((1, 3, 32, 32), dtype=torch.float32)
+
+    def forward(self, imgs):
+        # Forward function that is run when visualizing the graph
+        return self.model(imgs)
+
+    def configure_optimizers(self):
+        # We will support Adam or SGD as optimizers.
+        if self.hparams.optimizer_name == "Adam":
+            # AdamW is Adam with a correct implementation of weight decay (see here
+            # for details: https://arxiv.org/pdf/1711.05101.pdf)
+            optimizer = optim.AdamW(self.parameters(), **self.hparams.optimizer_hparams)
+        elif self.hparams.optimizer_name == "SGD":
+            optimizer = optim.SGD(self.parameters(), **self.hparams.optimizer_hparams)
+        else:
+            assert False, f'Unknown optimizer: "{self.hparams.optimizer_name}"'
+
+        # We will reduce the learning rate by 0.1 after 100 and 150 epochs
+        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[100, 150], gamma=0.1)
+        return [optimizer], [scheduler]
+
+    def training_step(self, batch, batch_idx):
+        # "batch" is the output of the training data loader.
+        imgs, labels = batch
+        preds = self.model(imgs)
+        loss = self.loss_module(preds, labels)
+        acc = (preds.argmax(dim=-1) == labels).float().mean()
+
+        # Logs the accuracy per epoch to tensorboard (weighted average over batches)
+        self.log("train_acc", acc, on_step=False, on_epoch=True)
+        self.log("train_loss", loss)
+        return loss  # Return tensor to call ".backward" on
+
+    def validation_step(self, batch, batch_idx):
+        imgs, labels = batch
+        preds = self.model(imgs).argmax(dim=-1)
+        acc = (labels == preds).float().mean()
+        # By default logs it per epoch (weighted average over batches)
+        self.log("val_acc", acc)
+
+    def test_step(self, batch, batch_idx):
+        imgs, labels = batch
+        preds = self.model(imgs).argmax(dim=-1)
+        acc = (labels == preds).float().mean()
+        # By default logs it per epoch (weighted average over batches), and returns it afterwards
+        self.log("test_acc", acc)
diff --git a/src/spotPython/torch/mapk.py b/src/spotPython/torch/mapk.py
@@ -57,10 +57,26 @@ def update(self, predicted: torch.Tensor, actual: torch.Tensor):
         Returns:
             (NoneType): None
 
+        Examples:
+            >>> from spotPython.torch.mapk import MAPK
+            >>> import torch
+            >>> mapk = MAPK(k=2)
+            >>> target = torch.tensor([0, 1, 2, 2])
+            >>> preds = torch.tensor(
+            ...     [
+            ...         [0.5, 0.2, 0.2],  # 0 is in top 2
+            ...         [0.3, 0.4, 0.2],  # 1 is in top 2
+            ...         [0.2, 0.4, 0.3],  # 2 is in top 2
+            ...         [0.7, 0.2, 0.1],  # 2 isn't in top 2
+            ...     ]
+            ... )
+            >>> mapk.update(preds, target)
+            >>> print(mapk.compute()) # tensor(0.6250)
+
         Raises:
-            AssertionError:
-                If `actual` is not a 1D tensor or if `predicted` is not a 2D tensor
-                or if `actual` and `predicted` do not have the same number of elements.
+            AssertionError: If the actual tensor is not 1D or the predicted tensor is not 2D.
+            AssertionError: If the number of elements in the actual and predicted tensors are not equal.
+
         """
         assert len(actual.shape) == 1, "actual must be a 1D tensor"
         assert len(predicted.shape) == 2, "predicted must be a 2D tensor"

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"`
`7`	`7`
`8`	`8`	`[project]`
`9`	`9`	`name = "spotPython"`
`10`		`-version = "0.6.8"`
	`10`	`+version = "0.6.10"`
`11`	`11`	`authors = [`
`12`	`12`	`{ name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }`
`13`	`13`	`]`