update progress bars

cfa/cloudops/_cloudclient.py

@@ -4,6 +4,7 @@
 import os
 import warnings
 from graphlib import CycleError, TopologicalSorter
+from pathlib import Path
 from typing import Literal, Optional
 
 import networkx as nx
@@ -1762,6 +1763,54 @@ def download_file(
         blob_helpers.download_file(c_client, src_path, dest_path, do_check, check_size)
         logger.info(f"Downloaded file '{src_path}' to '{dest_path}'.")
 
+    def download_files(
+        self,
+        file_paths: str | list[str],
+        container_name: str,
+        dest_path: str = ".",
+    ) -> None:
+        """Download a list of files from Azure Blob Storage to the local filesystem.
+
+        Downloads a file from a blob storage container to a local destination path.
+        Supports verification of the download to ensure data integrity.
+
+        Args:
+            file_paths (list[str]): List of file paths within the blob container to download.
+                Each path should be the full blob path including any directory structure.
+            container_name (str): Name of the blob storage container containing the file.
+            dest_path (str): Local filesystem path where the file should be saved.
+                Can be relative or absolute. Parent directories will be created if needed.
+
+        Example:
+
+            client = CloudClient()
+            client.download_files(
+                file_paths=["data/results_1.csv", "data/results_2.csv"],
+                container_name="job-outputs"
+                dest_path="./my_folder",
+            )
+
+        Note:
+            If the destination directory doesn't exist, it will be created automatically.
+            The download will overwrite any existing file at the destination path.
+        """
+        logger.debug(f"Downloading files from container {container_name}.")
+
+        for file_path in file_paths:
+            parent_path = Path(file_path).parent
+            root_path = dest_path if dest_path.endswith("/") else f"{dest_path}/"
+            Path(f"{root_path}{parent_path}").mkdir(parents=True, exist_ok=True)
+        blob.download_from_storage_container(
+            file_paths=file_paths,
+            blob_storage_container_name=container_name,
+            blob_service_client=self.blob_service_client,
+            local_root_dir=dest_path,
+            remote_root_dir=".",
+        )
+        logger.info(
+            f"Downloaded files from container '{container_name}' to path {dest_path}."
+        )
+
     def download_folder(
         self,
         src_path: str,

cfa/cloudops/batch_helpers.py

@@ -26,6 +26,7 @@
     UserIdentity,
 )
 from azure.mgmt.batch import BatchManagementClient
+from tqdm.auto import tqdm
 
 from cfa.cloudops.task import (
     get_container_settings,
@@ -395,7 +396,9 @@ def monitor_tasks(
 
 
 def download_job_stats(
-    job_name: str, batch_service_client: object, file_name: str | None = None
+    job_name: str,
+    batch_service_client: object,
+    file_name: str | None = None,
 ) -> None:
     """Download comprehensive statistics for all tasks in a job to a CSV file.
 
@@ -436,18 +439,15 @@ def download_job_stats(
         The file is created in the current working directory. Tasks that haven't
         completed may not have all timing information available.
     """
+
     logger.debug(f"Downloading job statistics for job: {job_name}")
+
     if file_name is None:
         file_name = f"{job_name}-stats"
-        logger.debug(f"Using default filename: {file_name}")
-    else:
-        logger.debug(f"Using custom filename: {file_name}")
 
-    logger.debug("Retrieving task list from batch service")
-    r = batch_service_client.list_tasks(job_name)
-    logger.debug("Task list retrieved successfully")
+    tasks = batch_service_client.list_tasks(job_name)
 
-    fields = [
+    headers = [
         "task_id",
         "command",
         "creation",
@@ -458,29 +458,46 @@ def download_job_stats(
         "pool",
         "node_id",
     ]
-    with open(rf"{file_name}.csv", "w") as f:
-        logger.debug(f"initializing {file_name}.csv.")
+
+    with open(f"{file_name}.csv", "w", newline="") as f:
         writer = csv.writer(f, delimiter="|")
-        writer.writerow(fields)
-    for item in r:
-        st = item.execution_info.start_time
-        et = item.execution_info.end_time
-        rt = et - st
-        id = item.id
-        creation = item.creation_time
-        start = item.execution_info.start_time.strftime("%Y-%m-%d %H:%M:%S")
-        end = item.execution_info.end_time.strftime("%Y-%m-%d %H:%M:%S")
-        exit_code = item.execution_info.exit_code
-        node_id = item.node_info.node_id
-        cli = item.command_line.split(" -")[0]
-        pool = item.node_info.pool_id
-        fields = [id, cli, creation, start, end, rt, exit_code, pool, node_id]
-        with open(rf"{file_name}.csv", "a") as f:
-            writer = csv.writer(f, delimiter="|")
-            writer.writerow(fields)
-            logger.debug(f"Wrote task {item.id} statistics to CSV")
-
-    logger.info(f"Job statistics download completed. File saved as: {file_name}.csv")
+        writer.writerow(headers)
+
+        for item in tqdm(tasks, desc="Writing task statistics", unit="task"):
+            execution_info = getattr(item, "execution_info", None)
+            node_info = getattr(item, "node_info", None)
+
+            start_time = (
+                execution_info.start_time if execution_info is not None else None
+            )
+
+            end_time = execution_info.end_time if execution_info is not None else None
+
+            creation_time = getattr(item, "creation_time", None)
+
+            runtime = (
+                end_time - start_time
+                if start_time is not None and end_time is not None
+                else None
+            )
+
+            row = [
+                item.id,
+                item.command_line.split(" -")[0] if item.command_line else "",
+                creation_time.strftime("%Y-%m-%d %H:%M:%S") if creation_time else "",
+                start_time.strftime("%Y-%m-%d %H:%M:%S") if start_time else "",
+                end_time.strftime("%Y-%m-%d %H:%M:%S") if end_time else "",
+                runtime,
+                execution_info.exit_code if execution_info else None,
+                node_info.pool_id if node_info else "",
+                node_info.node_id if node_info else "",
+            ]
+
+            writer.writerow(row)
+
+            logger.debug(f"Wrote task {item.id} statistics")
+
+    logger.info(f"Job statistics written to {file_name}.csv")
 
 
 def check_job_exists(job_name: str, batch_client: object):

cfa/cloudops/blob.py

@@ -17,7 +17,7 @@
     ImmutabilityPolicy,
     aio,
 )
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from .client import get_blob_service_client
 from .util import ensure_listlike
@@ -258,7 +258,9 @@ def download_from_storage_container(
         blob_service_client = get_blob_service_client(**kwargs)
         logger.debug("Blob service client created successfully")
 
-    for i_file, file_path in enumerate(file_paths):
+    for i_file, file_path in enumerate(
+        tqdm(file_paths, desc="Downloading files", unit="file")
+    ):
         if i_file % (1 + int(n_total_files / 10)) == 0:
             logger.debug(f"Download progress: {i_file}/{n_total_files} files completed")
 
@@ -1049,7 +1051,9 @@ def update_blob_protection(
     logger.debug(f"Processing {n_total_files} files for legal hold toggle")
 
     try:
-        for i_file, file_path in enumerate(files):
+        for i_file, file_path in enumerate(
+            tqdm(files, desc="Updating blob protection", unit="file")
+        ):
             if i_file % (1 + int(n_total_files / 10)) == 0:
                 logger.debug(f"Progress: {i_file}/{n_total_files} files processed")
 

cfa/cloudops/blob_helpers.py

@@ -12,6 +12,7 @@
     StorageStreamDownloader,
 )
 from humanize import naturalsize as ns
+from tqdm.auto import tqdm
 
 from .blob import format_extensions, upload_to_storage_container
 
@@ -961,7 +962,7 @@ def download_folder(
             if cont.lower() != "y":
                 print("Download aborted.")
                 return None
-    for blob in flist:
+    for blob in tqdm(flist, desc="Downloading files", unit="file"):
         download_file(
             c_client,
             blob,
@@ -1062,7 +1063,7 @@ def delete_blob_folder(
     logger.debug(f"Found {len(_files)} blobs to delete in folder '{folder_path}'")
 
     # call delete_blob_snapshots for each file
-    for i, file in enumerate(_files, 1):
+    for i, file in enumerate(tqdm(_files, desc="Deleting blobs", unit="blob"), 1):
         logger.debug(f"Deleting blob {i}/{len(_files)}: '{file}'")
         delete_blob_snapshots(
             blob_name=file,

changelog.md

@@ -7,11 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 The versioning pattern is `major.minor.patch`.
 
 ---
+## 1.1.1
+
+- add notebook-friendly progress bars for task-stat writing and blob download, deletion, and protection operations
+- add `jupyter` and `ipywidgets` as development dependencies for notebook-friendly progress-bar testing
+
+
 ## 1.1.0
 
 - add type hint support for the public `cfa.cloudops` API
 - include `py.typed` and `__init__.pyi` typing resources in package builds
 
+## 1.0.1
+
+- update credential checks to return subscription, Batch account, and Storage account information in a DataFrame, including an empty result with stable columns when credential checks fail
+- update the Ruff pre-commit hook to 0.16.7
+
 ## 1.0.0
 
 - replace `DefaultAzureCredential` with a custom `ChainedTokenCredential` builder (`_build_default_credential`) that exposes `exclude_*` flags for granular control over which credential types are attempted

pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "cfa.cloudops"
-version = "1.1.0"
+version = "1.1.1"
 description = "Cloud storage, batch, functions, MLOps assistance"
 authors = [
     {name = "Ryan Raasch", email = "xng3@cdc.gov"}
 ]
 readme = "README.md"
-requires-python = ">=3.10, <4.0"
+requires-python = ">=3.12, <4.0"
 dependencies = [
     "tqdm>=4.67.1",
     "pandas>=2.3.2",
@@ -29,7 +29,15 @@ dependencies = [
     "toml>=0.10.2",
     "networkx>=3.4.2",
     "anyio>=4.10.0",
-    "duckdb>=1.4.4"
+    "duckdb>=1.4.4",
+    "numpy>=2.5.3"
+]
+
+[project.optional-dependencies]
+notebook = [
+    "jupyter>=1.1.1",
+    "ipywidgets>=8.1.9",
+    "ipykernel>=7.3.0",
 ]
 
 [build-system]
@@ -95,7 +103,7 @@ dev = [
     "aiohttp>=3.14.3",
     "azure-mgmt-authorization>=4.0.0",
     "coverage-badge>=1.1.2",
-    "ipykernel>=7.1.0",
+    "ipykernel>=7.3.0",
     "mkdocs>=1.6.1",
     "mkdocs-material>=9.6.16",
     "mkdocstrings[python]>=0.30.0",
@@ -105,6 +113,5 @@ dev = [
     "pytest-asyncio>=1.2.0",
     "pytest-mock>=3.14.0",
     "pytest-cov>=6.1.1",
-    "ruff>=0.9.9",
-    "tomli>=2.2.1",
+    "ruff>=0.9.9"
 ]