diff --git a/src/clusterfuzz/_internal/metrics/logs.py b/src/clusterfuzz/_internal/metrics/logs.py index f5d00c2d551..a66fa138ba4 100644 --- a/src/clusterfuzz/_internal/metrics/logs.py +++ b/src/clusterfuzz/_internal/metrics/logs.py @@ -54,17 +54,13 @@ _default_extras = {} -def _is_running_on_k8s(): - """Returns whether or not we're running on K8s.""" - # We do this here to avoid circular imports with environment. - return os.getenv('IS_K8S_ENV') == 'true' - - def _increment_error_count(): """"Increment the error count metric.""" - if _is_running_on_k8s(): + if environment.is_running_on_swarming(): + task_name = 'swarming' + elif environment.is_running_on_k8s(): task_name = 'k8s' - elif _is_running_on_app_engine(): + elif environment.is_running_on_app_engine(): task_name = 'appengine' else: task_name = os.getenv('TASK_NAME', 'unknown') @@ -79,15 +75,6 @@ def _is_local(): os.getenv('SERVER_SOFTWARE', '').startswith('Development/')) -def _is_running_on_app_engine(): - """Return whether or not we're running on App Engine (production or - development).""" - return os.getenv('GAE_ENV') or ( - os.getenv('SERVER_SOFTWARE') and - (os.getenv('SERVER_SOFTWARE').startswith('Development/') or - os.getenv('SERVER_SOFTWARE').startswith('Google App Engine/'))) - - def _console_logging_enabled(): """Return bool on where console logging is enabled, usually for tests.""" return bool(os.getenv('LOG_TO_CONSOLE')) @@ -558,7 +545,6 @@ def configure_swarming(name: str, extras: dict[str, str] | None = None) -> None: global _default_extras _default_extras = extras - logging.basicConfig(level=logging.INFO) if _cloud_logging_enabled(): configure_cloud_logging() @@ -579,11 +565,11 @@ def configure(name, extras=None): configure_swarming(name, extras) return - if _is_running_on_k8s(): + if environment.is_running_on_k8s(): configure_k8s() return - if _is_running_on_app_engine(): + if environment.is_running_on_app_engine(): configure_appengine() return @@ -613,7 +599,7 @@ def get_logger(): if _logger: return _logger - if _is_running_on_app_engine() or _is_running_on_k8s(): + if environment.is_running_on_app_engine() or environment.is_running_on_k8s(): # Running on App Engine. set_logger(logging.getLogger()) @@ -645,7 +631,7 @@ def get_source_location(): def _add_appengine_trace(extras): """Add App Engine tracing information.""" - if not _is_running_on_app_engine(): + if not environment.is_running_on_app_engine(): return from libs import auth @@ -724,7 +710,7 @@ def emit(level, message, exc_info=None, **extras): path_name, line_number, method_name = get_source_location() - if _is_running_on_app_engine(): + if environment.is_running_on_app_engine(): if exc_info == (None, None, None): # Don't pass exc_info at all, as otherwise cloud logging will append # "NoneType: None" to the message. diff --git a/src/clusterfuzz/_internal/system/environment.py b/src/clusterfuzz/_internal/system/environment.py index 1292b5fa946..cea2241979c 100644 --- a/src/clusterfuzz/_internal/system/environment.py +++ b/src/clusterfuzz/_internal/system/environment.py @@ -60,6 +60,7 @@ class UtaskMainRuntime(enum.Enum): BATCH = 'batch' KATA_CONTAINER = 'kata_container' INSTANCE_GROUP = 'instance_group' + SWARMING = 'swarming' def _eval_value(value_string): @@ -723,7 +724,7 @@ def is_untrusted_worker(): return get_value('UNTRUSTED_WORKER') -def is_uworker(): +def is_uworker() -> bool: """Return whether or not the current bot is a uworker. This is not the same as OSS-Fuzz's untrusted worker.""" return get_value('UWORKER') @@ -732,19 +733,22 @@ def is_uworker(): def get_runtime() -> UtaskMainRuntime: """ Get the current runtime for running the tasks. - It can be KATA_CONTAINER, BATCH or INSTANCE_GROUP. + It can be KATA_CONTAINER, BATCH, SWARMING or INSTANCE_GROUP. :return: Enum UtaskMainRuntime with one of KATA_CONTAINER, - BATCH or INSTANCE_GROUP + BATCH, SWARMING or INSTANCE_GROUP :rtype: UtaskMainRuntime """ - if is_uworker() and is_running_on_k8s(): - return UtaskMainRuntime.KATA_CONTAINER + if not is_uworker(): + return UtaskMainRuntime.INSTANCE_GROUP + + if is_running_on_swarming(): + return UtaskMainRuntime.SWARMING - if is_uworker() and not is_running_on_k8s(): - return UtaskMainRuntime.BATCH + if is_running_on_k8s(): + return UtaskMainRuntime.KATA_CONTAINER - return UtaskMainRuntime.INSTANCE_GROUP + return UtaskMainRuntime.BATCH def is_running_on_swarming() -> bool: @@ -785,9 +789,12 @@ def parse_environment_definition(environment_string): return values -def is_running_on_k8s(): +def is_running_on_k8s() -> bool: """Returns whether or not we're running on K8s.""" - return os.getenv('IS_K8S_ENV') == 'true' + env_value = get_value('IS_K8S_ENV', False) + if isinstance(env_value, str): + return env_value.lower() == 'true' + return bool(env_value) def base_platform(override): diff --git a/src/clusterfuzz/_internal/tests/appengine/handlers/issue_redirector_test.py b/src/clusterfuzz/_internal/tests/appengine/handlers/issue_redirector_test.py index 318074a89a1..b6a36d6cab0 100644 --- a/src/clusterfuzz/_internal/tests/appengine/handlers/issue_redirector_test.py +++ b/src/clusterfuzz/_internal/tests/appengine/handlers/issue_redirector_test.py @@ -29,9 +29,9 @@ def setUp(self): test_helpers.patch(self, [ 'clusterfuzz._internal.issue_management.issue_tracker_utils.get_issue_url', 'libs.helpers.get_testcase', - 'clusterfuzz._internal.metrics.logs._is_running_on_app_engine', + 'clusterfuzz._internal.system.environment.is_running_on_app_engine', ]) - self.mock._is_running_on_app_engine.return_value = True # pylint: disable=protected-access + self.mock.is_running_on_app_engine.return_value = True import server self.app = webtest.TestApp(server.app) diff --git a/src/clusterfuzz/_internal/tests/appengine/server_test.py b/src/clusterfuzz/_internal/tests/appengine/server_test.py index 825c0a042a6..f8c6fee04d1 100644 --- a/src/clusterfuzz/_internal/tests/appengine/server_test.py +++ b/src/clusterfuzz/_internal/tests/appengine/server_test.py @@ -22,12 +22,12 @@ class ServerTest(unittest.TestCase): def setUp(self): helpers.patch(self, [ - 'clusterfuzz._internal.metrics.logs._is_running_on_app_engine', + 'clusterfuzz._internal.system.environment.is_running_on_app_engine', ]) - self.mock._is_running_on_app_engine.return_value = True # pylint: disable=protected-access + self.mock.is_running_on_app_engine.return_value = True - # pylint: disable=protected-access def test(self): + # pylint: disable=import-outside-toplevel import server self.assertIsNotNone(server.handlers) self.assertIsNotNone(server.cron_routes) diff --git a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py index 68ad9153a10..fd92f73173b 100644 --- a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py +++ b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py @@ -495,14 +495,14 @@ def setUp(self): 'clusterfuzz._internal.metrics.logs.set_logger', 'logging.config.dictConfig', 'logging.getLogger', - 'clusterfuzz._internal.metrics.logs._is_running_on_app_engine', + 'clusterfuzz._internal.system.environment.is_running_on_app_engine', 'clusterfuzz._internal.metrics.logs.suppress_unwanted_warnings', 'google.cloud.logging.Client', ]) def test_configure(self): """Test configure.""" - self.mock._is_running_on_app_engine.return_value = False # pylint: disable=protected-access + self.mock.is_running_on_app_engine.return_value = False logs._logger = None # pylint: disable=protected-access logger = mock.MagicMock() self.mock.getLogger.return_value = logger @@ -517,7 +517,7 @@ def test_configure(self): def test_configure_appengine(self): """Test configure on App Engine.""" - self.mock._is_running_on_app_engine.return_value = True # pylint: disable=protected-access + self.mock.is_running_on_app_engine.return_value = True logs.configure('test') self.assertEqual(0, self.mock.dictConfig.call_count) @@ -549,7 +549,7 @@ class EmitTest(unittest.TestCase): def setUp(self): helpers.patch(self, [ 'clusterfuzz._internal.metrics.logs.get_logger', - 'clusterfuzz._internal.metrics.logs._is_running_on_app_engine', + 'clusterfuzz._internal.system.environment.is_running_on_app_engine', 'clusterfuzz._internal.datastore.data_types.Testcase.get_fuzz_target', 'clusterfuzz._internal.base.utils.get_instance_name', ]) @@ -578,7 +578,7 @@ def setUp(self): # Reset the `common_ctx` metadata as it may be setted by other test runs. logs.log_contexts.delete_metadata('common_ctx') logs.log_contexts.clear() - self.mock._is_running_on_app_engine.return_value = False # pylint: disable=protected-access + self.mock.is_running_on_app_engine.return_value = False def tearDown(self): os.environ.clear() diff --git a/src/clusterfuzz/_internal/tests/core/system/environment_test.py b/src/clusterfuzz/_internal/tests/core/system/environment_test.py index e70159d94cb..3751e7b842d 100644 --- a/src/clusterfuzz/_internal/tests/core/system/environment_test.py +++ b/src/clusterfuzz/_internal/tests/core/system/environment_test.py @@ -51,6 +51,35 @@ def test_set_bot_environment_default_variables(self): self.assertEqual(environment.get_value('VERSION_PATTERN'), '') self.assertEqual(environment.get_value('WATCH_FOR_PROCESS_EXIT'), False) + def test_get_runtime_instance_group(self): + """Tests that get_runtime() returns INSTANCE_GROUP when not a uworker.""" + environment.set_value('UWORKER', False) + self.assertEqual(environment.get_runtime(), + environment.UtaskMainRuntime.INSTANCE_GROUP) + + def test_get_runtime_kata_container(self): + """Tests that get_runtime() returns KATA_CONTAINER when running on k8s.""" + environment.set_value('UWORKER', True) + environment.set_value('IS_K8S_ENV', True) + self.assertEqual(environment.get_runtime(), + environment.UtaskMainRuntime.KATA_CONTAINER) + + def test_get_runtime_swarming(self): + """Tests that get_runtime() returns SWARMING when running on swarming.""" + environment.set_value('UWORKER', True) + environment.set_value('IS_K8S_ENV', False) + environment.set_value('SWARMING_BOT', True) + self.assertEqual(environment.get_runtime(), + environment.UtaskMainRuntime.SWARMING) + + def test_get_runtime_batch(self): + """Tests that get_runtime() returns BATCH as fallback.""" + environment.set_value('UWORKER', True) + environment.set_value('IS_K8S_ENV', False) + environment.set_value('SWARMING_BOT', False) + self.assertEqual(environment.get_runtime(), + environment.UtaskMainRuntime.BATCH) + class GetExecutableFileNameTest(unittest.TestCase): """Tests for get_executable_filename."""