--made changes to the way GPU is handled

-- changed test cases to reflect the change
-- changed documentation
-- other bug fixes and improvements

Author: manujosephv

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "python.pythonPath": "env\\Scripts\\python.exe",
+    "python.pythonPath": "new_env\\Scripts\\python.exe",
     "python.testing.unittestEnabled": false,
     "python.testing.nosetestsEnabled": false,
     "python.testing.pytestEnabled": false,

docs/training.md

@@ -8,7 +8,7 @@ The parameters that you would set most frequently are:
 
 -   `batch_size`: int: Number of samples in each batch of training. Defaults to `64`
 -   `max_epochs`: int: Maximum number of epochs to be run. The maximum is in case of Early Stopping where this becomes the maximum and without Early Stopping, this is the number of epochs that will be run Defaults to `10`
--   `gpus`: int: The index of the GPU to be used. If zero, will use CPU. Defaults to `0`
+-   `gpus`: int: The index of the GPU to be used. -1 will make use of all available GPUs If None, will use CPU. Defaults to `None`
 -   `load_best`: int: Flag to load the best model saved during training. This will be ignored if checkpoint saving is turned off. Defaults to True
 
 ### Usage Example

docs/tutorials/01-Basic_Usage.ipynb

@@ -152,7 +152,7 @@
     "    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate\n",
     "    batch_size=1024,\n",
     "    max_epochs=100,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1, #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -593,4 +593,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

docs/tutorials/02-Advanced_Usage.ipynb

@@ -206,7 +206,7 @@
     "fast_dev_run: false\n",
     "max_epochs: 20\n",
     "min_epochs: 1\n",
-    "gpus: 1\n",
+    "gpus: -1\n",
     "accumulate_grad_batches: 1\n",
     "auto_lr_find: false\n",
     "check_val_every_n_epoch: 1\n",
@@ -641,7 +641,7 @@
     "    max_epochs=epochs,\n",
     "    early_stopping=None,\n",
     "    accumulate_grad_batches=2,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1, #index of the GPU to use. -1 means using all available GPUs. None, means CPU\n",
     ")\n",
     "\n",
     "\n",
@@ -14480,4 +14480,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

docs/tutorials/03-Extracting and Using Learned Embeddings.ipynb

@@ -201,7 +201,7 @@
     "    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate\n",
     "    batch_size=1024,\n",
     "    max_epochs=1000,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "model_config = CategoryEmbeddingModelConfig(\n",

docs/tutorials/04-Implementing New Architectures.ipynb

@@ -280,7 +280,7 @@
     "    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate\n",
     "    batch_size=1024,\n",
     "    max_epochs=100,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -623,4 +623,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

docs/tutorials/05-Experiment_Tracking.ipynb

@@ -152,7 +152,7 @@
     "    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate\n",
     "    batch_size=1024,\n",
     "    max_epochs=100,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -253,7 +253,7 @@
     "    batch_size=64,\n",
     "    accumulate_grad_batches=16,\n",
     "    max_epochs=100,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -357,7 +357,7 @@
     "#     accumulate_grad_batches=16,\n",
     "    max_epochs=100,\n",
     "    min_epochs=10,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -462,7 +462,7 @@
     "    max_epochs=50,\n",
     "    min_epochs=10,\n",
     "    early_stopping=None,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -569,4 +569,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

docs/tutorials/06-Imbalanced Classification.ipynb

@@ -144,7 +144,7 @@
     "    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate\n",
     "    batch_size=1024,\n",
     "    max_epochs=100,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "optimizer_config = OptimizerConfig()\n",
     "\n",
@@ -426,4 +426,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

docs/tutorials/07-Probabilistic Regression with MDN.ipynb

@@ -267,7 +267,7 @@
     "    batch_size=batch_size,\n",
     "    max_epochs=epochs,\n",
     "    early_stopping_patience = 5,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "# optimizer_config = OptimizerConfig(lr_scheduler=\"OneCycleLR\", lr_scheduler_params={\"max_lr\":0.005, \"epochs\": epochs, \"steps_per_epoch\":steps_per_epoch})\n",
     "optimizer_config = OptimizerConfig(lr_scheduler=\"ReduceLROnPlateau\", lr_scheduler_params={\"patience\":3})\n",
@@ -611,7 +611,7 @@
     "    batch_size=batch_size,\n",
     "    max_epochs=epochs,\n",
     "    early_stopping_patience = 5,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "# optimizer_config = OptimizerConfig(lr_scheduler=\"OneCycleLR\", lr_scheduler_params={\"max_lr\":0.005, \"epochs\": epochs, \"steps_per_epoch\":steps_per_epoch})\n",
     "optimizer_config = OptimizerConfig(lr_scheduler=\"ReduceLROnPlateau\", lr_scheduler_params={\"patience\":3})\n",
@@ -856,7 +856,7 @@
     "    max_epochs=epochs,\n",
     "    early_stopping_patience = 5,\n",
     "    early_stopping=None,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "# optimizer_config = OptimizerConfig(lr_scheduler=\"OneCycleLR\", lr_scheduler_params={\"max_lr\":0.005, \"epochs\": epochs, \"steps_per_epoch\":steps_per_epoch})\n",
     "optimizer_config = OptimizerConfig(lr_scheduler=\"ReduceLROnPlateau\", lr_scheduler_params={\"patience\":3})\n",
@@ -1367,7 +1367,7 @@
     "    batch_size=batch_size,\n",
     "    max_epochs=epochs,\n",
     "    early_stopping_patience = 5,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "# optimizer_config = OptimizerConfig(lr_scheduler=\"OneCycleLR\", lr_scheduler_params={\"max_lr\":0.005, \"epochs\": epochs, \"steps_per_epoch\":steps_per_epoch})\n",
     "optimizer_config = OptimizerConfig(lr_scheduler=\"ReduceLROnPlateau\", lr_scheduler_params={\"patience\":3})\n",
@@ -1608,7 +1608,7 @@
     "    max_epochs=epochs,\n",
     "    early_stopping_patience = 5,\n",
     "    early_stopping=None,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "# optimizer_config = OptimizerConfig(lr_scheduler=\"OneCycleLR\", lr_scheduler_params={\"max_lr\":0.005, \"epochs\": epochs, \"steps_per_epoch\":steps_per_epoch})\n",
     "optimizer_config = OptimizerConfig(lr_scheduler=\"ReduceLROnPlateau\", lr_scheduler_params={\"patience\":3})\n",
@@ -2297,7 +2297,7 @@
     "    max_epochs=epochs,\n",
     "    early_stopping_patience = 5,\n",
     "#     early_stopping=None,\n",
-    "    gpus=1, #index of the GPU to use. 0, means CPU\n",
+    "    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU\n",
     ")\n",
     "# optimizer_config = OptimizerConfig(lr_scheduler=\"OneCycleLR\", lr_scheduler_params={\"max_lr\":0.005, \"epochs\": epochs, \"steps_per_epoch\":steps_per_epoch})\n",
     "optimizer_config = OptimizerConfig(lr_scheduler=\"ReduceLROnPlateau\", lr_scheduler_params={\"patience\":3})\n",
@@ -2766,4 +2766,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

examples/to_test_classification.py

@@ -97,7 +97,7 @@
 #     metrics=["f1", "accuracy"],
 #     metrics_params=[{"num_classes": num_classes, "average": "macro"}, {}],
 # )
-trainer_config = TrainerConfig(gpus=1, fast_dev_run=False, max_epochs=5, batch_size=1024)
+trainer_config = TrainerConfig(gpus=-1, auto_select_gpus=True, fast_dev_run=False, max_epochs=5, batch_size=1024)
 experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example", 
                                      run_name="node_forest_cov", 
                                      exp_watch="gradients",