From b698ce088eaf177ddee2a923e1d2790ee368c8eb Mon Sep 17 00:00:00 2001 From: Colm Dougan Date: Thu, 20 Nov 2025 14:40:51 +0000 Subject: [PATCH] HDDS-14005. EventNotification: Added the concept of dynamically configurable EventListener plugins --- .../om/eventlistener/OMEventListener.java | 32 +++++ .../OMEventListenerPluginContext.java | 26 ++++ .../ozone/om/eventlistener/package-info.java | 21 ++++ .../OMEventListenerPluginContextImpl.java | 35 ++++++ .../OMEventListenerPluginManager.java | 115 +++++++++++++++++ .../ozone/om/eventlistener/package-info.java | 21 ++++ .../ozone/om/eventlistener/BarPlugin.java | 57 +++++++++ .../ozone/om/eventlistener/FooPlugin.java | 57 +++++++++ .../TestOMEventListenerPluginManager.java | 119 ++++++++++++++++++ 9 files changed, 483 insertions(+) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListener.java create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContext.java create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContextImpl.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginManager.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/BarPlugin.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/FooPlugin.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/TestOMEventListenerPluginManager.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListener.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListener.java new file mode 100644 index 000000000000..5f20301d458d --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListener.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; + +/** + * Interface for event listener plugin implementations. + */ +public interface OMEventListener { + + void initialize(OzoneConfiguration conf, OMEventListenerPluginContext pluginContext); + + void start(); + + void shutdown(); +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContext.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContext.java new file mode 100644 index 000000000000..41b349754a71 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContext.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +/** + * A narrow set of functionality we are ok with exposing to plugin + * implementations. + */ +public interface OMEventListenerPluginContext { + +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java new file mode 100644 index 000000000000..dbda5e337cce --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes for the OM Event Listener implementation. + */ +package org.apache.hadoop.ozone.om.eventlistener; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContextImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContextImpl.java new file mode 100644 index 000000000000..12573c625695 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginContextImpl.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +import org.apache.hadoop.ozone.om.OzoneManager; + +/** + * A narrow set of functionality we are ok with exposing to plugin + * implementations. + */ +public final class OMEventListenerPluginContextImpl implements OMEventListenerPluginContext { + private final OzoneManager ozoneManager; // NOPMD: unused. will be used soon + + public OMEventListenerPluginContextImpl(OzoneManager ozoneManager) { + this.ozoneManager = ozoneManager; + } + + // TODO: fill this out with capabilities we would like to expose to + // plugin implementations. +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginManager.java new file mode 100644 index 000000000000..e4331206b056 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/OMEventListenerPluginManager.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a manager for plugins which implement OMEventListener which + * manages the lifecycle of constructing starting/stopping configured + * plugins. + */ +public class OMEventListenerPluginManager { + public static final Logger LOG = LoggerFactory.getLogger(OMEventListenerPluginManager.class); + + public static final String PLUGIN_DEST_BASE = "ozone.om.plugin.destination"; + + private final List plugins; + + public OMEventListenerPluginManager(OzoneManager ozoneManager, OzoneConfiguration conf) { + this.plugins = loadAll(ozoneManager, conf); + } + + public List getLoaded() { + return plugins; + } + + public void startAll() { + for (OMEventListener plugin : plugins) { + plugin.start(); + } + } + + public void shutdownAll() { + for (OMEventListener plugin : plugins) { + plugin.shutdown(); + } + } + + // Configuration is based on ranger plugins + // + // For example, a plugin named FooPlugin would be configured via + // OzoneConfiguration properties as follows: + // + // conf.set("ozone.om.plugin.destination.foo", "enabled"); + // conf.set("ozone.om.plugin.destination.foo.classname", "org.apache.hadoop.ozone.om.eventlistener.FooPlugin"); + // + static List loadAll(OzoneManager ozoneManager, OzoneConfiguration conf) { + List plugins = new ArrayList<>(); + + Map props = conf.getPropsMatchPrefixAndTrimPrefix(PLUGIN_DEST_BASE); + List destNameList = new ArrayList<>(); + for (Map.Entry entry : props.entrySet()) { + String destName = entry.getKey(); + String value = entry.getValue(); + LOG.info("Found event listener plugin with name={} and value={}", destName, value); + + if (value.equalsIgnoreCase("enable") || value.equalsIgnoreCase("enabled") || value.equalsIgnoreCase("true")) { + destNameList.add(destName); + LOG.info("Event listener plugin {}{} is set to {}", PLUGIN_DEST_BASE, destName, value); + } + } + + OMEventListenerPluginContext pluginContext = new OMEventListenerPluginContextImpl(ozoneManager); + + for (String destName : destNameList) { + try { + Class cls = resolvePluginClass(conf, destName); + LOG.info("Event listener plugin class is {}", cls); + + OMEventListener impl = cls.newInstance(); + impl.initialize(conf, pluginContext); + + plugins.add(impl); + } catch (Exception ex) { + LOG.error("Can't make instance of event listener plugin {}{}", PLUGIN_DEST_BASE, destName, ex); + } + } + + return plugins; + } + + private static Class resolvePluginClass(OzoneConfiguration conf, + String destName) { + String classnameProp = PLUGIN_DEST_BASE + destName + ".classname"; + LOG.info("Gettting classname for {} with propety {}", destName, classnameProp); + Class cls = conf.getClass(classnameProp, null, OMEventListener.class); + if (null == cls) { + throw new RuntimeException(String.format( + "Unable to load plugin %s, classname property %s is missing or does not implement OMEventListener", + destName, classnameProp)); + } + return cls; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java new file mode 100644 index 000000000000..dbda5e337cce --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/eventlistener/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes for the OM Event Listener implementation. + */ +package org.apache.hadoop.ozone.om.eventlistener; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/BarPlugin.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/BarPlugin.java new file mode 100644 index 000000000000..e58c588353e2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/BarPlugin.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; + +/** + * Test utility for creating a dummy plugin. + */ +public class BarPlugin implements OMEventListener { + + private boolean initialized = false; + private boolean started = false; + private boolean shutdown = false; + + @Override + public void initialize(OzoneConfiguration conf, OMEventListenerPluginContext pluginContext) { + initialized = true; + } + + @Override + public void start() { + started = true; + } + + @Override + public void shutdown() { + shutdown = true; + } + + public boolean isInitialized() { + return initialized; + } + + public boolean isStarted() { + return started; + } + + public boolean isShutdown() { + return shutdown; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/FooPlugin.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/FooPlugin.java new file mode 100644 index 000000000000..e919eda603af --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/FooPlugin.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; + +/** + * Test utility for creating a dummy plugin. + */ +public class FooPlugin implements OMEventListener { + + private boolean initialized = false; + private boolean started = false; + private boolean shutdown = false; + + @Override + public void initialize(OzoneConfiguration conf, OMEventListenerPluginContext pluginContext) { + initialized = true; + } + + @Override + public void start() { + started = true; + } + + @Override + public void shutdown() { + shutdown = true; + } + + public boolean isInitialized() { + return initialized; + } + + public boolean isStarted() { + return started; + } + + public boolean isShutdown() { + return shutdown; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/TestOMEventListenerPluginManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/TestOMEventListenerPluginManager.java new file mode 100644 index 000000000000..70913c6e8201 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/eventlistener/TestOMEventListenerPluginManager.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.eventlistener; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests {@link OMEventListenerPluginManager}. + */ +@ExtendWith(MockitoExtension.class) +public class TestOMEventListenerPluginManager { + + @Mock + private OzoneManager ozoneManager; + + static List getLoadedPlugins(OMEventListenerPluginManager pluginManager) { + List loadedClasses = new ArrayList<>(); + for (OMEventListener plugin : pluginManager.getLoaded()) { + loadedClasses.add(plugin.getClass().getName()); + } + + // normalize + Collections.sort(loadedClasses); + + return loadedClasses; + } + + private static class BrokenFooPlugin { + + } + + @Test + public void testLoadSinglePlugin() throws InterruptedException { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set("ozone.om.plugin.destination.foo", "enabled"); + conf.set("ozone.om.plugin.destination.foo.classname", "org.apache.hadoop.ozone.om.eventlistener.FooPlugin"); + + OMEventListenerPluginManager pluginManager = new OMEventListenerPluginManager(ozoneManager, conf); + + Assertions.assertEquals(Arrays.asList("org.apache.hadoop.ozone.om.eventlistener.FooPlugin"), + getLoadedPlugins(pluginManager)); + } + + @Test + public void testLoadMultiplePlugins() throws InterruptedException { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set("ozone.om.plugin.destination.foo", "enabled"); + conf.set("ozone.om.plugin.destination.foo.classname", "org.apache.hadoop.ozone.om.eventlistener.FooPlugin"); + conf.set("ozone.om.plugin.destination.bar", "enabled"); + conf.set("ozone.om.plugin.destination.bar.classname", "org.apache.hadoop.ozone.om.eventlistener.BarPlugin"); + + OMEventListenerPluginManager pluginManager = new OMEventListenerPluginManager(ozoneManager, conf); + + Assertions.assertEquals(Arrays.asList("org.apache.hadoop.ozone.om.eventlistener.BarPlugin", + "org.apache.hadoop.ozone.om.eventlistener.FooPlugin"), + + getLoadedPlugins(pluginManager)); + } + + @Test + public void testPluginMissingClassname() throws InterruptedException { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set("ozone.om.plugin.destination.foo", "enabled"); + + OMEventListenerPluginManager pluginManager = new OMEventListenerPluginManager(ozoneManager, conf); + + Assertions.assertEquals(Arrays.asList(), + getLoadedPlugins(pluginManager)); + } + + @Test + public void testPluginClassDoesNotExist() throws InterruptedException { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set("ozone.om.plugin.destination.foo", "enabled"); + conf.set("ozone.om.plugin.destination.foo.classname", "org.apache.hadoop.ozone.om.eventlistener.NotExistingPlugin"); + + OMEventListenerPluginManager pluginManager = new OMEventListenerPluginManager(ozoneManager, conf); + + Assertions.assertEquals(Arrays.asList(), + getLoadedPlugins(pluginManager)); + } + + @Test + public void testPluginClassDoesNotImplementInterface() throws InterruptedException { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set("ozone.om.plugin.destination.foo", "enabled"); + conf.set("ozone.om.plugin.destination.foo.classname", "org.apache.hadoop.ozone.om.eventlistener.BrokenFooPlugin"); + + OMEventListenerPluginManager pluginManager = new OMEventListenerPluginManager(ozoneManager, conf); + + Assertions.assertEquals(Arrays.asList(), + getLoadedPlugins(pluginManager)); + } +}