diff --git a/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java b/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java index 9bb2fefb717..03608dd2778 100644 --- a/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java +++ b/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java @@ -302,7 +302,12 @@ final class ExecutionInfoLocationObjectFactory implements IHopMetadataObjectFact public Object createObject(String id, Object parentObject) throws HopException { PluginRegistry registry = PluginRegistry.getInstance(); IPlugin plugin = registry.findPluginWithId(ExecutionInfoLocationPluginType.class, id); - return registry.loadClass(plugin); + IExecutionInfoLocation location = registry.loadClass(plugin, IExecutionInfoLocation.class); + // The ID is the key the location is stored under, not always a field of its own: without + // it the location can't be saved again + location.setPluginId(plugin.getIds()[0]); + location.setPluginName(plugin.getName()); + return location; } @Override diff --git a/engine/src/test/java/org/apache/hop/core/metadata/MetadataObjectIdTest.java b/engine/src/test/java/org/apache/hop/core/metadata/MetadataObjectIdTest.java new file mode 100644 index 00000000000..c15f7cb432a --- /dev/null +++ b/engine/src/test/java/org/apache/hop/core/metadata/MetadataObjectIdTest.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.core.metadata; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.hop.core.database.DatabasePluginType; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.plugins.IPlugin; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.execution.ExecutionInfoLocation; +import org.apache.hop.execution.local.FileExecutionInfoLocation; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.metadata.api.HopMetadata; +import org.apache.hop.metadata.api.HopMetadataDefaultObjectFactory; +import org.apache.hop.metadata.api.HopMetadataObject; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadata; +import org.apache.hop.metadata.api.IHopMetadataObjectFactory; +import org.apache.hop.metadata.api.IHopMetadataSerializer; +import org.apache.hop.metadata.plugin.MetadataPluginType; +import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration; +import org.apache.hop.pipeline.engine.PipelineEnginePluginType; +import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration; +import org.apache.hop.workflow.engine.WorkflowEnginePluginType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * An object that a metadata property creates through an object factory (an execution information + * location, a sampler, a run configuration, ...) is saved under its ID, and created again from that + * ID when it is read. Read and saved again, as in a metadata export or the metadata that remote and + * distributed engines send their workers, every registered plugin must come back as itself. + */ +@ExtendWith(RestoreHopEngineEnvironmentExtension.class) +class MetadataObjectIdTest { + + /** As the location files of the Neo4j integration tests: the plugin ID is only the key. */ + @Test + void aLocationStoredOnlyUnderItsKeyKeepsItsTypeWhenSavedAgain() throws Exception { + String key = ExecutionInfoLocation.class.getAnnotation(HopMetadata.class).key(); + String exported = + "{\"" + + key + + "\":[{\"name\":\"audit\",\"dataLoggingDelay\":\"2000\",\"dataLoggingInterval\":\"5000\"," + + "\"executionInfoLocation\":{\"local-folder\":{\"rootFolder\":\"/tmp/executions\"}}}]}"; + + String exportedAgain = new SerializableMetadataProvider(exported).toJson(); + ExecutionInfoLocation location = + new SerializableMetadataProvider(exportedAgain) + .getSerializer(ExecutionInfoLocation.class) + .load("audit"); + + FileExecutionInfoLocation fileLocation = + assertInstanceOf(FileExecutionInfoLocation.class, location.getExecutionInfoLocation()); + assertEquals("local-folder", fileLocation.getPluginId()); + assertEquals("/tmp/executions", fileLocation.getRootFolder()); + } + + @Test + void everyRegisteredPluginComesBackAsItselfWhenSavedAgain() throws Exception { + PluginRegistry registry = PluginRegistry.getInstance(); + List failures = new ArrayList<>(); + Map roundTrips = new TreeMap<>(); + + for (IPlugin metadataPlugin : registry.getPlugins(MetadataPluginType.class)) { + Class holderClass = registry.loadClass(metadataPlugin).getClass(); + for (Field field : factoryFields(holderClass)) { + Class valueType = valueType(field); + IHopMetadataObjectFactory factory = + valueType + .getAnnotation(HopMetadataObject.class) + .objectFactory() + .getDeclaredConstructor() + .newInstance(); + String property = holderClass.getSimpleName() + "." + field.getName(); + for (String id : pluginIds(registry, valueType)) { + // As read from a file, and as the copy an editor or a run works on + Object read = factory.createObject(id, null); + List objects = new ArrayList<>(List.of(read)); + Method clone = publicClone(read); + if (clone != null) { + objects.add(clone.invoke(read)); + } + for (Object object : objects) { + roundTrips.merge(property, 1, Integer::sum); + String problem = roundTrip(holderClass, field, object); + if (problem != null) { + failures.add(property + ", " + id + ": " + problem); + } + } + } + } + } + + assertTrue(failures.isEmpty(), String.join("\n", failures)); + // The plugins of the engine module itself are covered + for (String property : + List.of( + "ExecutionInfoLocation.executionInfoLocation", + "ExecutionDataProfile.samplers", + "PipelineRunConfiguration.engineRunConfiguration", + "WorkflowRunConfiguration.engineRunConfiguration")) { + assertTrue(roundTrips.getOrDefault(property, 0) > 0, property + " not tested: " + roundTrips); + } + } + + /** The properties holding an object that a factory of its own saves and creates. */ + private static List factoryFields(Class holderClass) { + List fields = new ArrayList<>(); + for (Class c = holderClass; c != null && c != Object.class; c = c.getSuperclass()) { + for (Field field : c.getDeclaredFields()) { + if (field.getAnnotation(HopMetadataProperty.class) == null) { + continue; + } + Class valueType = valueType(field); + HopMetadataObject annotation = + valueType == null ? null : valueType.getAnnotation(HopMetadataObject.class); + if (annotation != null + && annotation.objectFactory() != HopMetadataDefaultObjectFactory.class) { + fields.add(field); + } + } + } + return fields; + } + + /** The type of the field, or of the elements of a list. */ + private static Class valueType(Field field) { + if (!List.class.isAssignableFrom(field.getType())) { + return field.getType(); + } + if (field.getGenericType() instanceof ParameterizedType type + && type.getActualTypeArguments()[0] instanceof Class elementType) { + return elementType; + } + return null; + } + + /** The IDs of the registered plugins the factory creates objects of the type for. */ + private static List pluginIds(PluginRegistry registry, Class valueType) { + List ids = new ArrayList<>(); + for (var pluginType : registry.getPluginTypes()) { + boolean matches; + if (valueType == IPipelineEngineRunConfiguration.class) { + matches = pluginType == PipelineEnginePluginType.class; + } else if (valueType == IWorkflowEngineRunConfiguration.class) { + matches = pluginType == WorkflowEnginePluginType.class; + } else if (valueType == IDatabase.class) { + // The database plugin type doesn't name its main type + matches = pluginType == DatabasePluginType.class; + } else { + matches = false; + } + for (IPlugin plugin : registry.getPlugins(pluginType)) { + if (matches || plugin.getMainType() == valueType) { + ids.add(plugin.getIds()[0]); + } + } + } + return ids; + } + + private static Method publicClone(Object object) { + try { + return object.getClass().getMethod("clone"); + } catch (NoSuchMethodException e) { + return null; + } + } + + /** Null when the object is read back as itself from the JSON it was saved in. */ + @SuppressWarnings("unchecked") + private static String roundTrip(Class holderClass, Field field, Object object) { + try { + IHopMetadata holder = (IHopMetadata) holderClass.getDeclaredConstructor().newInstance(); + holder.setName("probe"); + field.setAccessible(true); + boolean list = List.class.isAssignableFrom(field.getType()); + field.set(holder, list ? new ArrayList<>(List.of(object)) : object); + + SerializableMetadataProvider saved = new SerializableMetadataProvider(); + ((IHopMetadataSerializer) + saved.getSerializer((Class) holderClass)) + .save(holder); + SerializableMetadataProvider read = new SerializableMetadataProvider(saved.toJson()); + Object loaded = read.getSerializer((Class) holderClass).load("probe"); + + Object value = loaded == null ? null : field.get(loaded); + if (list && value != null) { + value = ((List) value).isEmpty() ? null : ((List) value).get(0); + } + if (value == null) { + return "not read back"; + } + return value.getClass() == object.getClass() + ? null + : "read back as " + value.getClass().getName(); + } catch (Exception e) { + return e.toString(); + } + } +}