bug: ignore non-deployment entries in the project's live directory

lean/commands/backtest.py

@@ -16,7 +16,7 @@
 from click import command, option, argument
 
 from lean.click import LeanCommand, PathParameter, backtest_parameter_option, CaseInsensitiveChoice
-from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH
+from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH, OUTPUT_DIRECTORY_NAME_FORMAT
 from lean.container import container, Logger
 from lean.models.utils import DebuggingMethod
 from lean.models.cli import cli_data_downloaders, cli_addon_modules
@@ -323,7 +323,7 @@ def backtest(project: Path,
     algorithm_file = project_manager.find_algorithm_file(Path(project))
     lean_config_manager = container.lean_config_manager
     if output is None:
-        output = algorithm_file.parent / "backtests" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        output = algorithm_file.parent / "backtests" / datetime.now().strftime(OUTPUT_DIRECTORY_NAME_FORMAT)
 
     environment_name = "backtesting"
     debugging_method = None

lean/commands/live/deploy.py

@@ -16,7 +16,7 @@
 from click import option, argument
 from lean.click import LeanCommand, PathParameter, CaseInsensitiveChoice
 from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format
-from lean.constants import DEFAULT_ENGINE_IMAGE
+from lean.constants import DEFAULT_ENGINE_IMAGE, OUTPUT_DIRECTORY_NAME_FORMAT
 from lean.container import container
 from lean.models.cli import (cli_brokerages, cli_data_queue_handlers, cli_data_downloaders,
                              cli_addon_modules, cli_history_provider)
@@ -168,7 +168,7 @@ def deploy(project: Path,
     algorithm_file = project_manager.find_algorithm_file(Path(project))
 
     if output is None:
-        output = algorithm_file.parent / "live" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        output = algorithm_file.parent / "live" / datetime.now().strftime(OUTPUT_DIRECTORY_NAME_FORMAT)
 
     lean_config_manager = container.lean_config_manager
 

lean/commands/optimize.py

@@ -18,8 +18,9 @@
 from click import command, argument, option, IntRange
 
 from lean.click import LeanCommand, PathParameter, ensure_options, CaseInsensitiveChoice
+from lean.components.config.output_config_manager import get_output_directory_timestamp
 from lean.components.docker.lean_runner import LeanRunner
-from lean.constants import DEFAULT_ENGINE_IMAGE
+from lean.constants import DEFAULT_ENGINE_IMAGE, OUTPUT_DIRECTORY_NAME_FORMAT
 from lean.container import container
 from lean.models.api import QCParameter, QCBacktest
 from lean.models.click_options import options_from_json, get_configs_for_options
@@ -40,19 +41,8 @@ def _get_latest_backtest_runtime(algorithm_directory: Path) -> timedelta:
     if not backtests_directory.exists():
         raise missing_backtest_error
 
-    def is_backtest_output_directory(path: Path) -> bool:
-        try:
-            datetime.strptime(path.name, "%Y-%m-%d_%H-%M-%S")
-        except ValueError:
-            return False
-        else:
-            return path.is_dir()
-
-    def get_filename_timestamp(path: Path) -> datetime:
-        return datetime.strptime(path.name, "%Y-%m-%d_%H-%M-%S")
-
-    backtests_directories = [f for f in backtests_directory.iterdir() if is_backtest_output_directory(f)]
-    latest_backtest_directory = max(backtests_directories, key=get_filename_timestamp, default=None)
+    backtests_directories = [f for f in backtests_directory.iterdir() if get_output_directory_timestamp(f) is not None]
+    latest_backtest_directory = max(backtests_directories, key=get_output_directory_timestamp, default=None)
 
     if latest_backtest_directory is None:
         raise missing_backtest_error
@@ -219,7 +209,7 @@ def optimize(project: Path,
         latest_backtest_runtime = _get_latest_backtest_runtime(algorithm_file.parent)
 
     if output is None:
-        output = algorithm_file.parent / "optimizations" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        output = algorithm_file.parent / "optimizations" / datetime.now().strftime(OUTPUT_DIRECTORY_NAME_FORMAT)
 
     optimizer_config_manager = container.optimizer_config_manager
     config = None

lean/components/config/output_config_manager.py

@@ -11,11 +11,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from datetime import datetime
 from pathlib import Path
 from typing import List, Optional
 
 from lean.components.config.lean_config_manager import LeanConfigManager
 from lean.components.config.storage import Storage
+from lean.constants import OUTPUT_DIRECTORY_NAME_FORMAT
+
+
+def get_output_directory_timestamp(directory: Path) -> Optional[datetime]:
+    """Returns the start time in the name of a backtest, optimization or live output directory, or None for other entries."""
+    if not directory.is_dir():
+        return None
+
+    try:
+        return datetime.strptime(directory.name, OUTPUT_DIRECTORY_NAME_FORMAT)
+    except ValueError:
+        return None
 
 
 class OutputConfigManager:

lean/components/util/live_utils.py

@@ -13,9 +13,11 @@
 
 from click import prompt, confirm
 
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 from lean.components.api.api_client import APIClient
+from lean.components.config.output_config_manager import get_output_directory_timestamp
 from lean.components.util.logger import Logger
 from lean.models.errors import RequestFailedError
 from lean.models.json_module import LiveInitialStateInput, JsonModule
@@ -35,10 +37,29 @@ def __setitem__(self, key: Any, item: Any) -> Any:
         self.data[key] = item
 
 
+def _get_local_deployment_time(directory: Path) -> Optional[datetime]:
+    """Returns when a local live deployment ran, or None when the directory is not a deployment."""
+    if not directory.is_dir():
+        return None
+
+    deployment_time = get_output_directory_timestamp(directory)
+    if deployment_time is not None:
+        try:
+            # the directory name holds the deployment's start time, in local time
+            return deployment_time.astimezone(timezone.utc)
+        except (OverflowError, OSError):
+            pass
+
+    # a custom --output directory: use the time its result file was last written
+    result_files = list(directory.glob("L-*.json"))
+    if not result_files:
+        return None
+    return datetime.fromtimestamp(max(f.stat().st_mtime for f in result_files), tz=timezone.utc)
+
+
 def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Path) -> List[Dict[str, Any]]:
-    from os import listdir, path
     from json import loads
-    from datetime import datetime, timezone
+    from lean.container import container
 
     cloud_last_time = datetime.min.replace(tzinfo=timezone.utc)
 
@@ -61,17 +82,19 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa
                                tzinfo=timezone.utc)
 
     local_last_time = datetime.min.replace(tzinfo=timezone.utc)
-    live_deployment_path = f"{project_name}/live"
-    if path.isdir(live_deployment_path):
-        local_deployment_time = [datetime.strptime(subdir, "%Y-%m-%d_%H-%M-%S").astimezone().astimezone(timezone.utc) for subdir in listdir(live_deployment_path)]
-        if local_deployment_time:
-            local_last_time = sorted(local_deployment_time, reverse = True)[0]
+    live_deployment_path = Path(project_name) / "live"
+    if live_deployment_path.is_dir():
+        for entry in live_deployment_path.iterdir():
+            deployment_time = _get_local_deployment_time(entry)
+            if deployment_time is None:
+                container.logger.debug(f"Ignoring '{entry}' in the live directory, it is not a deployment")
+            else:
+                local_last_time = max(local_last_time, deployment_time)
 
     if cloud_last_time > local_last_time:
         last_state = api_client.get("live/portfolio/read", {"projectId": project_id})
         previous_portfolio_state = last_state["portfolio"]
     elif cloud_last_time < local_last_time:
-        from lean.container import container
         output_directory = container.output_config_manager.get_latest_output_directory("live")
         if not output_directory:
             return None

lean/constants.py

@@ -119,3 +119,6 @@
 
 # List pending datasets
 LIST_PENDING_DATASETS = environ.get("LIST_PENDING_DATASETS", "false").lower() == "true"
+
+# Local backtests, optimizations and live deployments write their output to a directory named after their start time
+OUTPUT_DIRECTORY_NAME_FORMAT = "%Y-%m-%d_%H-%M-%S"

tests/commands/test_live.py

@@ -13,6 +13,7 @@
 
 import itertools
 import sys
+from importlib import import_module
 import json
 from pathlib import Path
 import traceback
@@ -24,6 +25,7 @@
 
 from lean.commands import lean
 from lean.components import reserved_names, output_reserved_names
+from lean.components.util.live_utils import configure_initial_cash_balance
 from lean.constants import DEFAULT_ENGINE_IMAGE
 from lean.container import container
 from lean.models.docker import DockerImage
@@ -1376,3 +1378,30 @@ def environment_modifier():
 
     create_lean_option(brokerage_name, data_provider_live_name, None, api_client,
                        environment_modifier=environment_modifier)
+
+
+# Finder creates .DS_Store files, --output can point to any directory name, and old names fail to convert on Windows
+@pytest.mark.parametrize("stray_entry,is_directory", [(".DS_Store", False), ("my-run", True), ("1960-01-01_00-00-00", True)])
+def test_live_ignores_entries_in_the_live_directory_that_are_not_deployments(stray_entry: str, is_directory: bool) -> None:
+    def environment_modifier():
+        live_directory = Path.cwd() / "Python Project" / "live"
+        deployment_directory = live_directory / "2024-09-26_17-25-28"
+        deployment_directory.mkdir(parents=True)
+        with open(deployment_directory / "L-3875119070.json", "w+") as out_file:
+            json.dump({"cash": {"USD": {"symbol": "USD", "amount": 100000.0}}}, out_file)
+        with open(deployment_directory / "config", "w+") as out_file:
+            json.dump({"algorithm-language": "Python", "parameters": {}, "id": "3875119070"}, out_file)
+        if is_directory:
+            (live_directory / stray_entry).mkdir()
+        else:
+            (live_directory / stray_entry).touch()
+
+    # patch by module object: the dotted path resolves to the `live` click group on Python < 3.11
+    deploy_module = import_module("lean.commands.live.deploy")
+    with mock.patch.object(deploy_module, "configure_initial_cash_balance",
+                           wraps=configure_initial_cash_balance) as configure_cash:
+        create_lean_option("Paper Trading", "Polygon", None, mock.MagicMock(), environment_modifier=environment_modifier)
+
+    # the deployment next to the stray entry is still read and its cash balance offered
+    previous_cash_state = configure_cash.call_args[0][3]
+    assert previous_cash_state["USD"]["amount"] == 100000.0

tests/components/util/test_live_utils.py

@@ -0,0 +1,99 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from pathlib import Path
+from typing import Any, Dict
+from unittest import mock
+
+import pytest
+
+from lean.components.util.live_utils import _get_last_portfolio
+from lean.container import container
+from tests.test_helpers import create_fake_lean_cli_directory
+
+LOCAL_STATE = {"Cash": {"USD": {"Symbol": "USD", "Amount": 100000.0}}}
+CLOUD_STATE = {"cash": {"USD": {"Symbol": "USD", "Amount": 5000.0}}}
+
+
+def _live_directory() -> Path:
+    return Path.cwd() / "Python Project" / "live"
+
+
+def _create_deployment(directory: Path, state: Dict[str, Any]) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    (directory / "config").write_text(json.dumps({"id": 3875119070}), encoding="utf-8")
+    (directory / "L-3875119070.json").write_text(json.dumps(state), encoding="utf-8")
+
+
+def _get_last_local_portfolio() -> Any:
+    return _get_last_portfolio(mock.MagicMock(), None, Path.cwd() / "Python Project")
+
+
+def test_get_last_portfolio_reads_the_latest_local_deployment() -> None:
+    create_fake_lean_cli_directory()
+    _create_deployment(_live_directory() / "2024-09-26_17-25-28", LOCAL_STATE)
+
+    assert _get_last_local_portfolio() == {"cash": LOCAL_STATE["Cash"]}
+
+
+@pytest.mark.parametrize("stray_entry,is_directory", [(".DS_Store", False), ("my-run", True)])
+def test_get_last_portfolio_ignores_entries_that_are_not_deployments(stray_entry: str, is_directory: bool) -> None:
+    create_fake_lean_cli_directory()
+    _create_deployment(_live_directory() / "2024-09-26_17-25-28", LOCAL_STATE)
+    if is_directory:
+        (_live_directory() / stray_entry).mkdir()
+    else:
+        (_live_directory() / stray_entry).touch()
+
+    with mock.patch.object(container.logger, "debug") as debug:
+        assert _get_last_local_portfolio() == {"cash": LOCAL_STATE["Cash"]}
+
+    assert any(stray_entry in str(call) for call in debug.call_args_list)
+
+
+def test_get_last_portfolio_survives_a_deployment_name_before_1970() -> None:
+    create_fake_lean_cli_directory()
+    _create_deployment(_live_directory() / "2024-09-26_17-25-28", LOCAL_STATE)
+    # converting this name to UTC raises OSError on Windows; elsewhere it is just an older deployment
+    (_live_directory() / "1960-01-01_00-00-00").mkdir()
+
+    assert _get_last_local_portfolio() == {"cash": LOCAL_STATE["Cash"]}
+
+
+def test_get_last_portfolio_returns_none_without_deployments() -> None:
+    create_fake_lean_cli_directory()
+    _live_directory().mkdir()
+    (_live_directory() / ".DS_Store").touch()
+
+    assert _get_last_local_portfolio() is None
+
+
+def test_get_last_portfolio_reads_a_deployment_in_a_custom_output_directory() -> None:
+    create_fake_lean_cli_directory()
+    _create_deployment(_live_directory() / "my-run", LOCAL_STATE)
+
+    assert _get_last_local_portfolio() == {"cash": LOCAL_STATE["Cash"]}
+
+
+def test_get_last_portfolio_prefers_the_cloud_deployment_when_it_is_more_recent() -> None:
+    create_fake_lean_cli_directory()
+    _create_deployment(_live_directory() / "2024-09-26_17-25-28", LOCAL_STATE)
+
+    api_client = mock.MagicMock()
+    api_client.get.side_effect = lambda endpoint, *args, **kwargs: {
+        "live/read": {"success": True, "status": "Stopped", "stopped": "2030-01-01 00:00:00", "launched": "2029-12-31 00:00:00"},
+        "live/portfolio/read": {"portfolio": CLOUD_STATE}
+    }[endpoint]
+
+    assert _get_last_portfolio(api_client, 1234, Path.cwd() / "Python Project") == CLOUD_STATE