From c66e87258b39167da3ea13db9a1376728bbfe6ad Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Thu, 12 Nov 2026 11:22:23 +0100 Subject: [PATCH 1/2] Detect language: choose the languages to detect and a faster low accuracy mode, fixes #9433 --- .../pipeline/transforms/detectlanguage.adoc | 31 ++ .../transforms/language/DetectLanguage.java | 115 ++++- .../language/DetectLanguageChoice.java | 50 +++ .../language/DetectLanguageData.java | 7 + .../language/DetectLanguageDialog.java | 170 +++----- .../language/DetectLanguageMeta.java | 204 ++++++++- .../messages/messages_en_US.properties | 16 + .../language/DetectLanguageAsyncTest.java | 22 +- .../language/DetectLanguageMetaTest.java | 178 +++++++- .../language/DetectLanguageTest.java | 401 ++++++++++++++++++ 10 files changed, 1024 insertions(+), 170 deletions(-) create mode 100644 plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java create mode 100644 plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc index ace4d07dedc..6268a61e9f9 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc @@ -35,8 +35,39 @@ https://github.com/pemistahl/lingua?tab=readme-ov-file#3-which-languages-are-sup |Option|Description |Transform name|Name of the transform. This name must be unique throughout the pipeline. +|Corpus text field|The field with the text to examine. +A variable is allowed. +|Use Parallelism|Detect the language of several rows at the same time. +The rows can leave the transform in another order than they came in. +|Low accuracy mode|Use only the three-letter language models. +This is faster and uses far less memory, but it is less accurate: short texts of a few words are often given the wrong language. +Use it for longer texts. +|Languages to detect|The languages the text can be in, by their Lingua name (`ENGLISH`, `FRENCH`, ...). +A variable is allowed. +Leave the list empty to detect every spoken language. +List at least two languages. |=== +== Output fields + +[options="header"] +|=== +|Field|Description +|detected_language|The name of the detected language, for example `ENGLISH`, or `UNKNOWN`. +|detected_language_confidence|Lingua's score for that language, relative to the other languages. +The detected language is the one with the highest score, so the value is always 1: it is not a probability and does not tell how sure the detection is. +Empty when the language is `UNKNOWN`. +|=== + +A row with an empty text passes with `UNKNOWN` and no confidence. +The language is also `UNKNOWN` when the two most likely languages score exactly the same. + +== Performance + +Detecting every spoken language loads the models of all 74 spoken languages: about 1.4 GB of memory, released when the transform finishes. +When you know which languages the data can contain, list them: with five languages the transform is about 2.5 times as fast and needs about 175 MB. +A text in a language that is not listed is given one of the listed languages, or `UNKNOWN`. + == Installation This transform is not part of the default Hop client. diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java index bf569b32e74..611d2aac1d9 100644 --- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java @@ -18,7 +18,6 @@ package org.apache.hop.pipeline.transforms.language; import static com.github.pemistahl.lingua.api.Language.UNKNOWN; -import static com.github.pemistahl.lingua.api.LanguageDetectorBuilder.fromAllSpokenLanguages; import static java.lang.System.arraycopy; import static java.util.concurrent.ForkJoinPool.commonPool; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -27,7 +26,9 @@ import com.github.pemistahl.lingua.api.Language; import com.github.pemistahl.lingua.api.LanguageDetector; +import com.github.pemistahl.lingua.api.LanguageDetectorBuilder; import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.SortedMap; import java.util.concurrent.ExecutorService; @@ -44,6 +45,7 @@ public class DetectLanguage extends BaseTransform PKG = DetectLanguage.class; private static final String CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD = "DetectLanguage.Exception.CouldnotFindField"; + private static final String ERROR_CODE = "DetectLanguage001"; public DetectLanguage( TransformMeta transformMeta, @@ -71,15 +73,52 @@ public DetectLanguage( private final Object lock = new Object(); - private final LanguageDetector detector = - fromAllSpokenLanguages().withPreloadedLanguageModels().build(); + /** The outcome of one detection: the language name and its confidence, null when unknown. */ + record Detection(String language, Double confidence) {} - private void awaitJobs() { - awaitJobs(DRAIN_TIMEOUT_MILLIS); + /** + * The Lingua builder for the configured options. + * + * @param languages the languages to choose from; empty means all spoken languages + * @param lowAccuracy use only the trigram models + */ + static LanguageDetectorBuilder createBuilder(List languages, boolean lowAccuracy) { + LanguageDetectorBuilder builder = + languages.isEmpty() + ? LanguageDetectorBuilder.fromAllSpokenLanguages() + : LanguageDetectorBuilder.fromLanguages(languages.toArray(new Language[0])); + if (lowAccuracy) { + builder = builder.withLowAccuracyMode(); + } + return builder.withPreloadedLanguageModels(); } - private void awaitJobs(final long timeoutMillis) { + @Override + public boolean init() { + if (!super.init()) { + return false; + } + try { + data.detector = createBuilder(meta.resolveLanguages(this), meta.isLowAccuracy()).build(); + return true; + } catch (Exception e) { + logError(BaseMessages.getString(PKG, "DetectLanguage.Error.DetectorNotBuilt"), e); + return false; + } + } + + /** + * Waits for accepted asynchronous work, at most {@link #DRAIN_TIMEOUT_MILLIS}. + * + * @return true when every job finished, false when some were abandoned + */ + private boolean awaitJobs() { + return awaitJobs(DRAIN_TIMEOUT_MILLIS); + } + + private boolean awaitJobs(final long timeoutMillis) { boolean interrupted = false; + boolean drained; final long deadline = System.currentTimeMillis() + timeoutMillis; synchronized (lock) { long remaining = timeoutMillis; @@ -93,7 +132,8 @@ private void awaitJobs(final long timeoutMillis) { } remaining = deadline - System.currentTimeMillis(); } - if (jobs > 0) { + drained = jobs == 0; + if (!drained) { // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and // abandon the remaining jobs. logBasic( @@ -107,6 +147,7 @@ private void awaitJobs(final long timeoutMillis) { if (interrupted) { Thread.currentThread().interrupt(); } + return drained; } /** @@ -150,8 +191,16 @@ private void finish() throws HopException { @Override public void dispose() { // The common pool is shared. Drain our work, but never shut the pool down. - awaitJobs(); - detector.unloadLanguageModels(); + // After an error or a stop the transform ends without finish(), and parallel jobs can still be + // queued. They use the detector: unloading under them would make them fail on a missing + // detector, or load the models again with nothing left to unload them. + boolean drained = awaitJobs(); + if (drained && data.detector != null) { + // Lingua keeps its models in static maps: release what this transform asked for. Another + // Detect language transform that is still running loads them again on demand. + data.detector.unloadLanguageModels(); + data.detector = null; + } final boolean shouldLog; synchronized (lock) { // Only log a failure we have not already surfaced through the run loop, so it is not logged @@ -189,7 +238,9 @@ public boolean processRow() throws HopException { data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - if (isEmpty(meta.getCorpusField())) { + // The field combo accepts variables + data.corpusField = resolve(meta.getCorpusField()); + if (isEmpty(data.corpusField)) { throw new HopException( BaseMessages.getString(PKG, "DetectLanguage.Error.CorpusFieldMissing")); } @@ -198,7 +249,6 @@ public boolean processRow() throws HopException { } // End If first try { - process(r); if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -206,6 +256,7 @@ public boolean processRow() throws HopException { "DetectLanguage.LineNumber", getLinesRead() + " : " + getInputRowMeta().getString(r))); } + process(r); } catch (Exception e) { // An infrastructure/worker failure must not be attributed to the current input row. checkFailure(); @@ -216,8 +267,10 @@ public boolean processRow() throws HopException { } private void process(Object[] inputRow) throws HopException { - String corpus = trim((String) inputRow[data.indexOfCorpusField]); + String corpus = trim(getInputRowMeta().getString(inputRow, data.indexOfCorpusField)); if (isBlank(corpus)) { + // Nothing to detect: the row passes with an unknown language and no confidence. + putDetection(inputRow, new Detection(UNKNOWN.name(), null)); return; } @@ -230,11 +283,13 @@ private void process(Object[] inputRow) throws HopException { private void handleRowError(Object[] row, Exception e) throws HopException { if (!getTransformMeta().isDoingErrorHandling()) { - throw new HopException("Error processing row", e); + throw new HopException( + BaseMessages.getString(PKG, "DetectLanguage.ErrorInTransformRunning") + e.getMessage(), + e); } // Error output has the same serialization requirement as normal output. synchronized (this) { - putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "DetectLanguage001"); + putError(getInputRowMeta(), row, 1, e.toString(), data.corpusField, ERROR_CODE); } } @@ -257,11 +312,18 @@ private void processAsync(String text, Object[] inputRow) throws HopException { () -> { try { try { - processSync(text, inputRow); + // Stopped: nobody wants the row anymore, and dispose() waits for the queue to + // drain. + if (!isStopped() || isSafeStopped()) { + processSync(text, inputRow); + } } catch (Exception e) { handleRowError(inputRow, e); } } catch (Throwable e) { + // Throwable, not Exception: nobody reads the pool's Future, so an Error caught + // nowhere else would make the row disappear without a trace. An Error says nothing + // about the row: it always stops the transform. synchronized (lock) { if (failure == null) { failure = e; @@ -286,16 +348,14 @@ private void processAsync(String text, Object[] inputRow) throws HopException { } } - @SuppressWarnings( - "javabugs:S2259") // the probabilities come from the map's own keys and are never null private void cacheIndexPositions() throws HopException { if (data.indexOfCorpusField < 0) { - data.indexOfCorpusField = data.previousRowMeta.indexOfValue(meta.getCorpusField()); + data.indexOfCorpusField = data.previousRowMeta.indexOfValue(data.corpusField); if (data.indexOfCorpusField < 0) { // The field is unreachable! throw new HopException( BaseMessages.getString( - PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD, meta.getCorpusField())); + PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD, data.corpusField)); } } if (data.indexOfDetectedLanguage < 0) { @@ -321,9 +381,13 @@ private void cacheIndexPositions() throws HopException { } } + /** + * Detects the language of a text: the most likely language, unless the second one is as likely or + * closer than Lingua's minimum relative distance, in which case the language is unknown. + */ @SuppressWarnings( "javabugs:S2259") // the probabilities come from the map's own keys and are never null - private void processSync(String corpus, Object[] inputRow) throws HopTransformException { + static Detection detect(LanguageDetector detector, String corpus) { SortedMap confidenceValues = detector.computeLanguageConfidenceValues(corpus); Language detectedLanguage = UNKNOWN; Double confidence = null; @@ -348,11 +412,18 @@ private void processSync(String corpus, Object[] inputRow) throws HopTransformEx } } } + return new Detection(detectedLanguage.name(), confidence); + } + + private void processSync(String corpus, Object[] inputRow) throws HopTransformException { + putDetection(inputRow, detect(data.detector, corpus)); + } + private void putDetection(Object[] inputRow, Detection detection) throws HopTransformException { Object[] outputRow = new Object[inputRow.length + 2 + OVER_ALLOCATE_SIZE]; arraycopy(inputRow, 0, outputRow, 0, inputRow.length); - outputRow[data.indexOfDetectedLanguage] = detectedLanguage.name(); - outputRow[data.indexOfDetectedLanguageConfidence] = confidence; + outputRow[data.indexOfDetectedLanguage] = detection.language(); + outputRow[data.indexOfDetectedLanguageConfidence] = detection.confidence(); putRow(data.outputRowMeta, outputRow); } diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java new file mode 100644 index 00000000000..7b48e58e34f --- /dev/null +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.language; + +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.gui.plugin.GuiTableColumn; +import org.apache.hop.core.gui.plugin.GuiTableColumnType; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** One language the Detect language transform may answer. */ +@Getter +@Setter +public class DetectLanguageChoice { + + /** The Lingua language name, for example ENGLISH. A variable is allowed. */ + @HopMetadataProperty( + key = "name", + injectionKey = "LANGUAGE", + injectionKeyDescription = "DetectLanguage.Injection.Language") + @GuiTableColumn( + id = "name", + order = "10", + type = GuiTableColumnType.COMBO, + comboValuesMethod = "languageNames", + label = "i18n::DetectLanguage.Column.Language.Label", + variables = true, + width = 250) + private String name = ""; + + public DetectLanguageChoice() {} + + public DetectLanguageChoice(String name) { + this.name = name; + } +} diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java index ef5d72bf98b..12f16dbfb34 100644 --- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java @@ -17,6 +17,7 @@ package org.apache.hop.pipeline.transforms.language; +import com.github.pemistahl.lingua.api.LanguageDetector; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.pipeline.transform.BaseTransformData; import org.apache.hop.pipeline.transform.ITransformData; @@ -30,6 +31,12 @@ public class DetectLanguageData extends BaseTransformData implements ITransformD public IRowMeta outputRowMeta; public int NrPrevFields; + /** The name of the corpus field, its variables resolved. */ + public String corpusField; + + /** Built in init() from the configured languages and accuracy mode. */ + public LanguageDetector detector; + public DetectLanguageData() { super(); indexOfCorpusField = -1; diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java index c5a1fcb7664..372b679cee1 100644 --- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java @@ -14,48 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hop.pipeline.transforms.language; -import static org.apache.hop.core.util.Utils.isEmpty; -import static org.apache.hop.i18n.BaseMessages.getString; -import static org.eclipse.swt.SWT.BORDER; -import static org.eclipse.swt.SWT.CHECK; -import static org.eclipse.swt.SWT.CURSOR_WAIT; -import static org.eclipse.swt.SWT.READ_ONLY; -import static org.eclipse.swt.SWT.RIGHT; - -import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.LogChannel; import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.pipeline.PipelineMeta; -import org.apache.hop.pipeline.transform.ITransformDialog; -import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; -import org.apache.hop.ui.core.dialog.ErrorDialog; -import org.apache.hop.ui.core.widget.ComboItems; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter; +import org.apache.hop.ui.core.widget.ComboVar; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; -import org.eclipse.swt.SWT; -import org.eclipse.swt.custom.CCombo; -import org.eclipse.swt.events.FocusEvent; -import org.eclipse.swt.events.FocusListener; -import org.eclipse.swt.graphics.Cursor; -import org.eclipse.swt.layout.FormAttachment; -import org.eclipse.swt.layout.FormData; -import org.eclipse.swt.widgets.Button; -import org.eclipse.swt.widgets.Label; -import org.eclipse.swt.widgets.Listener; +import org.eclipse.swt.widgets.Combo; +import org.eclipse.swt.widgets.Control; import org.eclipse.swt.widgets.Shell; -public class DetectLanguageDialog extends BaseTransformDialog implements ITransformDialog { +public class DetectLanguageDialog extends BaseTransformDialog { private static final Class PKG = DetectLanguageDialog.class; - private boolean gotPreviousFields = false; - - private CCombo wCorpusFieldName; - private Button wParallelism; private final DetectLanguageMeta input; + private GuiCompositeWidgets widgets; public DetectLanguageDialog( Shell parent, @@ -69,82 +49,66 @@ public DetectLanguageDialog( @Override public String open() { createShell(BaseMessages.getString(PKG, "DetectLanguageDialog.Shell.Title")); - buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); - Listener lsMod = e -> input.setChanged(); changed = input.hasChanged(); - // CorpusFieldName field - Label wlCorpusFieldName = new Label(shell, RIGHT); - wlCorpusFieldName.setText(getString(PKG, "DetectLanguageDialog.CorpusFieldName.Label")); - PropsUi.setLook(wlCorpusFieldName); - FormData fdlCorpusFieldName = new FormData(); - fdlCorpusFieldName.left = new FormAttachment(0, 0); - fdlCorpusFieldName.right = new FormAttachment(middle, -margin); - fdlCorpusFieldName.top = new FormAttachment(wSpacer, margin); - wlCorpusFieldName.setLayoutData(fdlCorpusFieldName); - - wCorpusFieldName = new CCombo(shell, BORDER | READ_ONLY); - PropsUi.setLook(wCorpusFieldName); - wCorpusFieldName.addListener(SWT.Modify, lsMod); - FormData fdCorpusFieldName = new FormData(); - fdCorpusFieldName.left = new FormAttachment(middle, 0); - fdCorpusFieldName.top = new FormAttachment(wSpacer, margin); - fdCorpusFieldName.right = new FormAttachment(100, 0); - wCorpusFieldName.setLayoutData(fdCorpusFieldName); - wCorpusFieldName.addFocusListener( - new FocusListener() { - @Override - public void focusLost(FocusEvent e) {} - + widgets = + GuiCompositeWidgets.addScrolledComposite( + shell, + variables, + wTransformName, + wOk, + DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID, + input); + widgets.setWidgetsListener( + new GuiCompositeWidgetsAdapter() { @Override - public void focusGained(FocusEvent e) { - Cursor busy = new Cursor(shell.getDisplay(), CURSOR_WAIT); - shell.setCursor(busy); - get(); - shell.setCursor(null); - busy.dispose(); + public void widgetModified( + GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) { + input.setChanged(); } }); - // parallelism - Label wlParallelism = new Label(shell, RIGHT); - wlParallelism.setText(getString(PKG, "DetectLanguageDialog.Parallelism.Label")); - PropsUi.setLook(wlParallelism); - FormData fdlParallelism = new FormData(); - fdlParallelism.left = new FormAttachment(0, 0); - fdlParallelism.top = new FormAttachment(wCorpusFieldName, margin); - fdlParallelism.right = new FormAttachment(middle, -margin); - wlParallelism.setLayoutData(fdlParallelism); - - wParallelism = new Button(shell, CHECK); - wParallelism.setSelection(input.isParallelism()); - PropsUi.setLook(wParallelism); - wParallelism.setToolTipText(getString(PKG, "DetectLanguageDialog.Parallelism.Tooltip")); - FormData fdParallelism = new FormData(); - fdParallelism.left = new FormAttachment(middle, 0); - fdParallelism.top = new FormAttachment(wCorpusFieldName, margin); - fdParallelism.right = new FormAttachment(100, 0); - wParallelism.setLayoutData(fdParallelism); - wParallelism.addListener(SWT.Selection, lsMod); - - getData(); + setFieldComboValues(); + // Filling the field combo is not a change input.setChanged(changed); + focusTransformName(); BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); - return transformName; } - /** Copy information from the meta-data input to the dialog fields. */ - public void getData() { - - if (input.getCorpusField() != null) { - wCorpusFieldName.setText(input.getCorpusField()); + /** + * Stream field names cannot come from {@code comboValuesMethod}, which is handed only a log + * channel and a metadata provider, so the field combo is filled once the widgets exist. The + * configured field stays when the previous fields cannot be read. + */ + private void setFieldComboValues() { + try { + IRowMeta fields = pipelineMeta.getPrevTransformFields(variables, transformName); + String[] names = fields == null ? new String[0] : fields.getFieldNames(); + widgets.setComboValues(DetectLanguageMeta.WIDGET_CORPUS_FIELD, names); + // Setting items clears the widget's text, so put the configured field back afterwards. + if (!Utils.isEmpty(input.getCorpusField())) { + setComboText(DetectLanguageMeta.WIDGET_CORPUS_FIELD, input.getCorpusField()); + } + } catch (Exception e) { + LogChannel.UI.logError( + BaseMessages.getString(PKG, "DetectLanguageDialog.FailedToGetFields.DialogMessage"), e); } + } - wParallelism.setSelection(input.isParallelism()); + private void setComboText(String widgetId, String text) { + Control control = widgets.getWidgetsMap().get(widgetId); + if (control == null || control.isDisposed()) { + return; + } + if (control instanceof ComboVar comboVar) { + comboVar.setText(text); + } else if (control instanceof Combo combo) { + combo.setText(text); + } } private void cancel() { @@ -154,32 +118,12 @@ private void cancel() { } private void ok() { - if (isEmpty(wTransformName.getText())) { + if (Utils.isEmpty(wTransformName.getText())) { return; } - input.setCorpusField(wCorpusFieldName.getText()); - input.setParallelism(wParallelism.getSelection()); - - transformName = wTransformName.getText(); // return value - + widgets.getWidgetsContents(input, DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID); + transformName = wTransformName.getText(); + input.setChanged(); dispose(); } - - private void get() { - if (!gotPreviousFields) { - try { - IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); - if (r != null) { - ComboItems.setItemsKeepingText(wCorpusFieldName, r.getFieldNames()); - } - gotPreviousFields = true; - } catch (HopException ke) { - new ErrorDialog( - shell, - getString(PKG, "DetectLanguageDialog.FailedToGetFields.DialogTitle"), - getString(PKG, "DetectLanguageDialog.FailedToGetFields.DialogMessage"), - ke); - } - } - } } diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java index efb62eddbce..0f65485c4e0 100644 --- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java @@ -14,21 +14,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hop.pipeline.transforms.language; import static org.apache.hop.core.ICheckResult.TYPE_RESULT_ERROR; import static org.apache.hop.core.ICheckResult.TYPE_RESULT_OK; +import static org.apache.hop.core.ICheckResult.TYPE_RESULT_WARNING; import static org.apache.hop.core.util.Utils.isEmpty; +import com.github.pemistahl.lingua.api.Language; +import java.util.ArrayList; import java.util.List; +import java.util.Locale; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import org.apache.hop.core.CheckResult; import org.apache.hop.core.ICheckResult; import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.gui.plugin.GuiWidgetGroupType; +import org.apache.hop.core.logging.ILogChannel; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.value.ValueMetaNumber; import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.util.StringUtil; import org.apache.hop.core.variables.IVariables; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.metadata.api.HopMetadataProperty; @@ -37,6 +50,9 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.TransformMeta; +@Getter +@Setter +@GuiPlugin @Transform( id = "DetectLanguage", image = "detectlanguage.svg", @@ -48,18 +64,88 @@ public class DetectLanguageMeta extends BaseTransformMeta { private static final Class PKG = DetectLanguageMeta.class; + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "DetectLanguage.Dialog"; + public static final String WIDGET_CORPUS_FIELD = "corpusField"; + public static final String WIDGET_PARALLELISM = "parallelism"; + public static final String WIDGET_LOW_ACCURACY = "lowAccuracy"; + public static final String WIDGET_LANGUAGES = "languages"; + + private static final String GROUP_OPTIONS = "i18n::DetectLanguage.Group.Options"; + private static final String GROUP_LANGUAGES = "i18n::DetectLanguage.Group.Languages"; + + @GuiWidgetElement( + id = WIDGET_CORPUS_FIELD, + order = "0100", + type = GuiElementType.COMBO, + label = "i18n::DetectLanguageDialog.CorpusFieldName.Label", + toolTip = "i18n::DetectLanguageDialog.CorpusFieldName.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.BOXES, + group = GROUP_OPTIONS, + groupOrder = "0100") @HopMetadataProperty( key = "corpusField", injectionKey = "FIELD", injectionKeyDescription = "DetectLanguage.Injection.CorpusField") private String corpusField; + @GuiWidgetElement( + id = WIDGET_PARALLELISM, + order = "0200", + type = GuiElementType.CHECKBOX, + label = "i18n::DetectLanguageDialog.Parallelism.Label", + toolTip = "i18n::DetectLanguageDialog.Parallelism.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.BOXES, + group = GROUP_OPTIONS, + groupOrder = "0100") @HopMetadataProperty( key = "parallelism", injectionKey = "PARALLISM", injectionKeyDescription = "DetectLanguage.Injection.Parallelism") private boolean parallelism = false; + /** + * Lingua's low accuracy mode: only the trigram models are used. Faster and much lighter on + * memory, less accurate on short text. Off for pipelines saved before the option existed. + */ + @GuiWidgetElement( + id = WIDGET_LOW_ACCURACY, + order = "0300", + type = GuiElementType.CHECKBOX, + label = "i18n::DetectLanguageDialog.LowAccuracy.Label", + toolTip = "i18n::DetectLanguageDialog.LowAccuracy.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.BOXES, + group = GROUP_OPTIONS, + groupOrder = "0100") + @HopMetadataProperty( + key = "lowAccuracy", + injectionKey = "LOW_ACCURACY", + injectionKeyDescription = "DetectLanguage.Injection.LowAccuracy") + private boolean lowAccuracy = false; + + /** + * The languages the transform may answer. Empty means every spoken language Lingua knows, which + * is what the transform did before the option existed. + */ + @GuiWidgetElement( + id = WIDGET_LANGUAGES, + order = "0400", + type = GuiElementType.TABLE, + toolTip = "i18n::DetectLanguageDialog.Languages.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.BOXES, + group = GROUP_LANGUAGES, + groupOrder = "0200", + tableRows = 8) + @HopMetadataProperty( + key = "language", + groupKey = "languages", + injectionGroupKey = "LANGUAGES", + injectionGroupDescription = "DetectLanguage.Injection.Languages") + private List languages = new ArrayList<>(); + public DetectLanguageMeta() { super(); } @@ -67,6 +153,77 @@ public DetectLanguageMeta() { @Override public void setDefault() { parallelism = false; + lowAccuracy = false; + languages = new ArrayList<>(); + } + + /** The names offered in the languages table: every language Lingua can detect. */ + public List languageNames(ILogChannel log, IHopMetadataProvider metadataProvider) { + return Language.all().stream().map(Language::name).sorted().toList(); + } + + /** + * Resolves the configured languages. + * + * @return the languages to detect, without duplicates and in the configured order; empty when + * none is configured, which means all spoken languages + * @throws HopException when a name is not a Lingua language or when exactly one language is + * configured: detection needs at least two to choose from + */ + public List resolveLanguages(IVariables variables) throws HopException { + return resolveLanguages(variables, null); + } + + /** + * Resolves the configured languages, leaving out the names that still hold a variable. + * + * @param unresolved when not null, a name that still holds a variable after resolving is added + * here instead of failing: Verify does not know the values a pipeline only gets when it runs + */ + List resolveLanguages(IVariables variables, List unresolved) + throws HopException { + List resolved = new ArrayList<>(); + if (languages == null) { + return resolved; + } + for (DetectLanguageChoice choice : languages) { + String name = choice == null ? null : choice.getName(); + if (variables != null) { + name = variables.resolve(name); + } + if (StringUtils.isBlank(name)) { + continue; + } + if (unresolved != null && StringUtil.containsVariableToken(name)) { + unresolved.add(name.trim()); + continue; + } + Language language = findLanguage(name.trim()); + if (language == null) { + throw new HopException( + BaseMessages.getString(PKG, "DetectLanguage.Error.UnknownLanguage", name.trim())); + } + if (!resolved.contains(language)) { + resolved.add(language); + } + } + // With names still to be resolved, the number of languages is not known yet + if (resolved.size() == 1 && (unresolved == null || unresolved.isEmpty())) { + throw new HopException( + BaseMessages.getString( + PKG, "DetectLanguage.Error.SingleLanguage", resolved.get(0).name())); + } + return resolved; + } + + private static Language findLanguage(String name) { + String upper = name.toUpperCase(Locale.ROOT); + for (Language language : Language.all()) { + if (language.name().equals(upper)) { + return language; + } + } + return null; } @Override @@ -77,7 +234,6 @@ public void getFields( TransformMeta nextTransform, IVariables variables, IHopMetadataProvider metadataProvider) { - // name = getParentTransformMeta().getName() valueMetaString(r, name, "detected_language"); valueMetaNumber(r, name, "detected_language_confidence"); @@ -107,7 +263,6 @@ public void check( IVariables variables, IHopMetadataProvider metadataProvider) { CheckResult cr; - if (isEmpty(corpusField)) { cr = new CheckResult( @@ -123,6 +278,33 @@ public void check( } remarks.add(cr); + try { + List unresolved = new ArrayList<>(); + List resolved = resolveLanguages(variables, unresolved); + if (!unresolved.isEmpty()) { + cr = + new CheckResult( + TYPE_RESULT_WARNING, + BaseMessages.getString( + PKG, + "DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime", + String.join(", ", unresolved)), + transformMeta); + } else { + cr = + new CheckResult( + TYPE_RESULT_OK, + resolved.isEmpty() + ? BaseMessages.getString(PKG, "DetectLanguageMeta.CheckResult.AllLanguages") + : BaseMessages.getString( + PKG, "DetectLanguageMeta.CheckResult.LanguagesOK", resolved.size()), + transformMeta); + } + } catch (HopException e) { + cr = new CheckResult(TYPE_RESULT_ERROR, e.getMessage().trim(), transformMeta); + } + remarks.add(cr); + // See if we have input streams leading to this transform! if (input.length > 0) { cr = @@ -145,20 +327,4 @@ public void check( public boolean supportsErrorHandling() { return true; } - - public String getCorpusField() { - return corpusField; - } - - public void setCorpusField(String corpusField) { - this.corpusField = corpusField; - } - - public boolean isParallelism() { - return parallelism; - } - - public void setParallelism(boolean parallelism) { - this.parallelism = parallelism; - } } diff --git a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties index be58a587f55..a319f4c8a0d 100644 --- a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties +++ b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties @@ -40,3 +40,19 @@ DetectLanguageMeta.CheckResult.ReceivingInfoFromOtherTransforms=Transform is rec DetectLanguageMeta.CheckResult.ErrorOccurred=An error occurred: DetectLanguageDialog.Parallelism.Label=Use Parallelism DetectLanguageDialog.Parallelism.Tooltip=Process the rows in parallel +DetectLanguageDialog.CorpusFieldName.Tooltip=The field with the text to examine +DetectLanguageDialog.LowAccuracy.Label=Low accuracy mode +DetectLanguageDialog.LowAccuracy.Tooltip=Use only the three-letter language models: faster and much lighter on memory, but less accurate. Meant for longer texts: short texts of a few words are often given the wrong language. +DetectLanguageDialog.Languages.Tooltip=The languages the text can be in. Leave the list empty to detect every spoken language. A short list is faster and uses less memory; a text in a language that is not listed is given one of the listed languages or UNKNOWN. +DetectLanguage.Group.Options=Options +DetectLanguage.Group.Languages=Languages to detect +DetectLanguage.Column.Language.Label=Language +DetectLanguage.Injection.LowAccuracy=Low accuracy mode +DetectLanguage.Injection.Languages=Languages to detect +DetectLanguage.Injection.Language=Language name +DetectLanguage.Error.UnknownLanguage=''{0}'' is not a language this transform can detect. Use a name from the list, for example ENGLISH. +DetectLanguage.Error.SingleLanguage=Only one language is listed ({0}). List at least two languages, or leave the list empty to detect every spoken language. +DetectLanguage.Error.DetectorNotBuilt=Unable to prepare the language detection +DetectLanguageMeta.CheckResult.AllLanguages=Every spoken language is detected. +DetectLanguageMeta.CheckResult.LanguagesOK={0} languages are detected. +DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime=These languages are only known when the pipeline runs: {0} diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java index 3806cae0642..75ca03e3046 100644 --- a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java +++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java @@ -23,18 +23,15 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.github.pemistahl.lingua.api.LanguageDetector; -import com.github.pemistahl.lingua.api.LanguageDetectorBuilder; import java.lang.reflect.Field; import java.util.List; import java.util.TreeMap; @@ -48,12 +45,12 @@ import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.TransformMeta; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.mockito.MockedStatic; @Timeout(10) class DetectLanguageAsyncTest { @@ -72,15 +69,14 @@ void setUp() throws Exception { PipelineMeta pipelineMeta = new PipelineMeta(); transformMeta = spy(new TransformMeta("test", meta)); pipelineMeta.addTransform(transformMeta); - try (MockedStatic factory = - mockStatic(LanguageDetectorBuilder.class)) { - LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class, RETURNS_SELF); - LanguageDetector detector = mock(LanguageDetector.class); - factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder); - when(builder.build()).thenReturn(detector); - when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>()); - transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null)); - } + LanguageDetector detector = mock(LanguageDetector.class); + when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>()); + data.detector = detector; + data.corpusField = "input"; + transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null)); + RowMeta inputRowMeta = new RowMeta(); + inputRowMeta.addValueMeta(new ValueMetaString("input")); + transform.setInputRowMeta(inputRowMeta); transform.first = false; data.indexOfCorpusField = 0; data.indexOfDetectedLanguage = 1; diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java index e4cda78ed17..c13196035c5 100644 --- a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java +++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java @@ -14,15 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hop.pipeline.transforms.language; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.pemistahl.lingua.api.Language; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import org.apache.hop.core.ICheckResult; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.w3c.dom.Node; class DetectLanguageMetaTest { @RegisterExtension @@ -30,11 +43,170 @@ class DetectLanguageMetaTest { @Test void testLoadSave() throws HopException { - List attributes = List.of("corpusField", "parallelism"); + List attributes = List.of("corpusField", "parallelism", "lowAccuracy"); + // the languages list has its own round trip test below LoadSaveTester loadSaveTester = - new LoadSaveTester<>(DetectLanguageMeta.class, attributes); + new LoadSaveTester<>( + DetectLanguageMeta.class, + attributes, + List.of("languages"), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + null); loadSaveTester.testSerialization(); } + + /** A pipeline saved before the language list and the accuracy mode existed. */ + @Test + void aTransformSavedBeforeTheNewOptionsKeepsItsBehavior() throws Exception { + DetectLanguageMeta meta = + load( + "commentY"); + + assertEquals("comment", meta.getCorpusField()); + assertTrue(meta.isParallelism()); + assertFalse(meta.isLowAccuracy()); + assertTrue(meta.getLanguages().isEmpty()); + // no languages listed: every spoken language, as before + assertTrue(meta.resolveLanguages(new Variables()).isEmpty()); + } + + @Test + void theNewOptionsSurviveAnXmlRoundTrip() throws Exception { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.setCorpusField("comment"); + meta.setLowAccuracy(true); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + meta.getLanguages().add(new DetectLanguageChoice("FRENCH")); + + DetectLanguageMeta copy = load("" + meta.getXml() + ""); + + assertEquals("comment", copy.getCorpusField()); + assertFalse(copy.isParallelism()); + assertTrue(copy.isLowAccuracy()); + assertEquals( + List.of("ENGLISH", "FRENCH"), + copy.getLanguages().stream().map(DetectLanguageChoice::getName).toList()); + } + + @Test + void languagesResolveWithVariablesAnyCaseAndNoDuplicates() throws Exception { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.getLanguages().add(new DetectLanguageChoice("english")); + meta.getLanguages().add(new DetectLanguageChoice(" ${SECOND} ")); + meta.getLanguages().add(new DetectLanguageChoice("")); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + Variables variables = new Variables(); + variables.setVariable("SECOND", "French"); + + assertEquals(List.of(Language.ENGLISH, Language.FRENCH), meta.resolveLanguages(variables)); + } + + @Test + void anUnknownLanguageIsRefused() { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + meta.getLanguages().add(new DetectLanguageChoice("KLINGON")); + + HopException e = assertThrows(HopException.class, () -> meta.resolveLanguages(new Variables())); + assertTrue(e.getMessage().contains("KLINGON")); + } + + @Test + void aSingleLanguageIsRefused() { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + + HopException e = assertThrows(HopException.class, () -> meta.resolveLanguages(new Variables())); + assertTrue(e.getMessage().contains("ENGLISH")); + } + + @Test + void theLanguageListOffersEveryLinguaLanguage() { + List names = new DetectLanguageMeta().languageNames(null, null); + + assertEquals(Language.all().size(), names.size()); + assertTrue(names.contains("ENGLISH")); + assertFalse(names.contains("UNKNOWN")); + } + + @Test + void checkReportsABadLanguageList() { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.setCorpusField("comment"); + meta.getLanguages().add(new DetectLanguageChoice("KLINGON")); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setName("Detect"); + meta.check( + remarks, + null, + transformMeta, + null, + new String[] {"in"}, + new String[0], + null, + new Variables(), + new MemoryMetadataProvider()); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_ERROR + && r.getText().contains("KLINGON"))); + } + + /** A language from a variable that only gets a value when the pipeline runs is not an error. */ + @Test + void checkLeavesLanguagesFromRuntimeVariablesToTheRun() { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.setCorpusField("comment"); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}")); + + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setName("Detect"); + meta.check( + remarks, + null, + transformMeta, + null, + new String[] {"in"}, + new String[0], + null, + new Variables(), + new MemoryMetadataProvider()); + + assertTrue(remarks.stream().noneMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR)); + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_WARNING + && r.getText().contains("${SECOND_LANGUAGE}"))); + } + + @Test + void runningWithAnUnresolvedLanguageStillFails() { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}")); + + assertThrows(HopException.class, () -> meta.resolveLanguages(new Variables())); + } + + private static DetectLanguageMeta load(String xml) throws Exception { + Node node = XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), "transform"); + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.loadXml(node, new MemoryMetadataProvider()); + return meta; + } } diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java new file mode 100644 index 00000000000..6f936af4e29 --- /dev/null +++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.language; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.github.pemistahl.lingua.api.Language; +import com.github.pemistahl.lingua.api.LanguageDetectorBuilder; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.hop.core.BlockingRowSet; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class DetectLanguageTest { + + private static final String ENGLISH_TEXT = + "The weather is lovely today and we are going for a long walk in the park"; + private static final String FRENCH_TEXT = + "Il fait très beau aujourd'hui et nous allons nous promener dans le parc"; + private static final String GERMAN_TEXT = + "Das Wetter ist heute sehr schön und wir gehen im Park spazieren"; + + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + private TransformMockHelper mockHelper; + + /** Disposed after every test, also a failed one: the Lingua models live in static maps. */ + private final List transforms = new ArrayList<>(); + + @BeforeEach + void setUp() { + mockHelper = + new TransformMockHelper<>( + "Detect language", DetectLanguageMeta.class, DetectLanguageData.class); + when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(mockHelper.iLogChannel); + when(mockHelper.pipeline.isRunning()).thenReturn(true); + when(mockHelper.transformMeta.getCopies(any())).thenReturn(1); + when(mockHelper.transformMeta.isPartitioned()).thenReturn(false); + when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(false); + } + + @AfterEach + void tearDown() { + for (TestDetectLanguage transform : transforms) { + transform.dispose(); + } + mockHelper.cleanUp(); + } + + /** Without options the detector is what the transform always built. */ + @Test + void theDefaultDetectorCoversEverySpokenLanguageWithHighAccuracy() { + LanguageDetectorBuilder builder = DetectLanguage.createBuilder(List.of(), false); + + assertEquals(Language.allSpokenOnes(), builder.getLanguages$lingua()); + assertFalse(builder.isLowAccuracyModeEnabled$lingua()); + assertTrue(builder.isEveryLanguageModelPreloaded$lingua()); + } + + @Test + void theConfiguredLanguagesAndAccuracyModeReachTheDetector() { + LanguageDetectorBuilder builder = + DetectLanguage.createBuilder(List.of(Language.ENGLISH, Language.FRENCH), true); + + assertEquals(List.of(Language.ENGLISH, Language.FRENCH), builder.getLanguages$lingua()); + assertTrue(builder.isLowAccuracyModeEnabled$lingua()); + } + + @Test + void rowsGetTheirLanguageAndBlankTextsPassAsUnknown() throws Exception { + DetectLanguage transform = createTransform(meta(false, false)); + assertTrue(transform.init()); + + BlockingRowSet output = + run(transform, row(1L, ENGLISH_TEXT), row(2L, null), row(3L, FRENCH_TEXT), row(4L, " ")); + + Object[] english = output.getRowWait(1, TimeUnit.SECONDS); + assertEquals(1L, english[0]); + assertEquals("ENGLISH", english[2]); + assertNotNull(english[3]); + + // a null or blank text used to drop the row + Object[] empty = output.getRowWait(1, TimeUnit.SECONDS); + assertEquals(2L, empty[0]); + assertEquals("UNKNOWN", empty[2]); + assertNull(empty[3]); + + assertEquals("FRENCH", output.getRowWait(1, TimeUnit.SECONDS)[2]); + + Object[] blank = output.getRowWait(1, TimeUnit.SECONDS); + assertEquals(4L, blank[0]); + assertEquals("UNKNOWN", blank[2]); + assertNull(blank[3]); + + assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS)); + } + + @Test + void theCorpusFieldCanComeFromAVariable() throws Exception { + DetectLanguageMeta meta = meta(false, false); + meta.setCorpusField("${TEXT_FIELD}"); + DetectLanguage transform = createTransform(meta); + assertTrue(transform.init()); + transform.setVariable("TEXT_FIELD", "comment"); + + BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT)); + + assertEquals("GERMAN", output.getRowWait(1, TimeUnit.SECONDS)[2]); + } + + @Test + void lowAccuracyModeFillsBothFields() throws Exception { + DetectLanguage transform = createTransform(meta(false, true)); + assertTrue(transform.init()); + + BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT)); + + Object[] german = output.getRowWait(1, TimeUnit.SECONDS); + assertEquals("GERMAN", german[2]); + assertNotNull(german[3]); + } + + @Test + void parallelModeWritesEveryRow() throws Exception { + DetectLanguage transform = createTransform(meta(true, false)); + assertTrue(transform.init()); + + BlockingRowSet output = + run(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L, GERMAN_TEXT)); + + Map languageById = new HashMap<>(); + for (int i = 0; i < 3; i++) { + Object[] row = output.getRowWait(5, TimeUnit.SECONDS); + assertNotNull(row); + languageById.put(row[0], row[2]); + } + assertEquals(Map.of(1L, "ENGLISH", 2L, "FRENCH", 3L, "GERMAN"), languageById); + } + + /** A failing parallel job used to leave the transform waiting forever at the end. */ + @Test + void aFailingParallelJobStopsTheTransform() { + DetectLanguage transform = createTransform(meta(true, false), DetectLanguageTest::noRoom); + assertTrue(transform.init()); + + HopException e = + assertTimeoutPreemptively( + Duration.ofSeconds(30), + () -> assertThrows(HopException.class, () -> run(transform, row(1L, ENGLISH_TEXT)))); + assertTrue(e.getMessage().contains("no room for this row")); + } + + @Test + void aFailingParallelRowGoesToTheErrorHop() throws Exception { + when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true); + TestDetectLanguage transform = createTransform(meta(true, false), DetectLanguageTest::noRoom); + assertTrue(transform.init()); + + assertTimeoutPreemptively( + Duration.ofSeconds(30), () -> run(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT))); + + List errorRows = new ArrayList<>(transform.errorRows); + assertEquals(2, errorRows.size()); + for (ErrorRow errorRow : errorRows) { + assertEquals("comment", errorRow.fieldNames()); + assertEquals("DetectLanguage001", errorRow.errorCodes()); + } + } + + /** An Error says nothing about the row: it stops the transform, also with error handling on. */ + @Test + void anErrorInAParallelJobStopsTheTransform() { + when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true); + TestDetectLanguage transform = + createTransform( + meta(true, false), + (rowMeta, row) -> { + throw new NoClassDefFoundError("simulated failure"); + }); + assertTrue(transform.init()); + + HopException e = + assertTimeoutPreemptively( + Duration.ofSeconds(30), + () -> assertThrows(HopException.class, () -> run(transform, row(1L, ENGLISH_TEXT)))); + assertTrue(e.getMessage().contains("simulated failure")); + assertTrue(transform.errorRows.isEmpty()); + } + + /** + * After an error or a stop the transform is disposed without reaching the end of its input. + * Parallel jobs that still run use the detector, so dispose waits for them. + */ + @Test + void disposeWaitsForTheParallelJobsStillRunning() throws Exception { + CountDownLatch writing = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + TestDetectLanguage transform = + createTransform( + meta(true, false), + (rowMeta, row) -> { + writing.countDown(); + awaitQuietly(release); + }); + assertTrue(transform.init()); + + connect(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L, GERMAN_TEXT)); + for (int i = 0; i < 3; i++) { + assertTrue(transform.processRow()); + } + assertTrue(writing.await(30, TimeUnit.SECONDS)); + + // What RunThread does when processRow fails or the pipeline is stopped + transform.stopAll(); + Thread disposer = new Thread(transform::dispose, "dispose"); + disposer.start(); + disposer.join(300); + assertTrue(disposer.isAlive(), "dispose returned while a job was still running"); + assertNotNull(transform.getData().detector); + + release.countDown(); + disposer.join(30_000); + assertFalse(disposer.isAlive()); + assertNull(transform.getData().detector); + assertTrue(transform.errorRows.isEmpty()); + } + + @Test + void anUnknownLanguageFailsAtStart() { + DetectLanguageMeta meta = meta(false, false); + meta.getLanguages().add(new DetectLanguageChoice("KLINGON")); + + assertFalse(createTransform(meta).init()); + } + + private static DetectLanguageMeta meta(boolean parallelism, boolean lowAccuracy) { + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.setCorpusField("comment"); + meta.setParallelism(parallelism); + meta.setLowAccuracy(lowAccuracy); + // three languages keep the models small; the default is covered by the builder test + meta.getLanguages().add(new DetectLanguageChoice("ENGLISH")); + meta.getLanguages().add(new DetectLanguageChoice("FRENCH")); + meta.getLanguages().add(new DetectLanguageChoice("GERMAN")); + return meta; + } + + private static Object[] row(Long id, String comment) { + return new Object[] {id, comment}; + } + + private static void noRoom(IRowMeta rowMeta, Object[] row) throws HopTransformException { + throw new HopTransformException("no room for this row"); + } + + private static void awaitQuietly(CountDownLatch latch) { + try { + latch.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Connects an input holding the rows, and an output, to the transform. */ + private static BlockingRowSet connect(DetectLanguage transform, Object[]... rows) { + IRowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaInteger("id")); + rowMeta.addValueMeta(new ValueMetaString("comment")); + + BlockingRowSet input = new BlockingRowSet(100); + input.setThreadNameFromToCopy("previous", 0, "Detect language", 0); + input.setRowMeta(rowMeta); + for (Object[] row : rows) { + input.putRow(rowMeta, row); + } + input.setDone(); + + BlockingRowSet output = new BlockingRowSet(100); + output.setThreadNameFromToCopy("Detect language", 0, "next", 0); + + transform.addRowSetToInputRowSets(input); + transform.addRowSetToOutputRowSets(output); + return output; + } + + /** Feeds the rows through the transform until it reports the end of its input. */ + private static BlockingRowSet run(DetectLanguage transform, Object[]... rows) + throws HopException { + BlockingRowSet output = connect(transform, rows); + List results = new ArrayList<>(); + boolean more = true; + while (more) { + more = transform.processRow(); + results.add(more); + } + assertEquals(rows.length + 1, results.size()); + return output; + } + + private TestDetectLanguage createTransform(DetectLanguageMeta meta) { + return createTransform(meta, null); + } + + private TestDetectLanguage createTransform(DetectLanguageMeta meta, RowWrite write) { + when(mockHelper.transformMeta.getTransform()).thenReturn(meta); + TestDetectLanguage transform = new TestDetectLanguage(mockHelper, meta, write); + transforms.add(transform); + return transform; + } + + /** What a test's transform does with an output row instead of writing it. */ + @FunctionalInterface + private interface RowWrite { + void write(IRowMeta rowMeta, Object[] row) throws HopTransformException; + } + + private record ErrorRow(Object id, String fieldNames, String errorCodes) {} + + /** The transform, with the writes a test controls and the error rows it sends. */ + private static class TestDetectLanguage extends DetectLanguage { + private final RowWrite write; + final List errorRows = Collections.synchronizedList(new ArrayList<>()); + + TestDetectLanguage( + TransformMockHelper helper, + DetectLanguageMeta meta, + RowWrite write) { + super( + helper.transformMeta, + meta, + new DetectLanguageData(), + 0, + helper.pipelineMeta, + helper.pipeline); + this.write = write; + } + + @Override + public void putRow(IRowMeta rowMeta, Object[] row) throws HopTransformException { + if (write == null) { + super.putRow(rowMeta, row); + } else { + write.write(rowMeta, row); + } + } + + @Override + public void putError( + IRowMeta rowMeta, + Object[] row, + long nrErrors, + String errorDescriptions, + String fieldNames, + String errorCodes) { + errorRows.add(new ErrorRow(row[0], fieldNames, errorCodes)); + } + } +} From fc85b22a867285026bf3eaff6fc64eb2784051fd Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Mon, 16 Nov 2026 15:08:26 +0100 Subject: [PATCH 2/2] Detect language: integration test for the language choice, low accuracy mode and blank texts, #9433 Co-Authored-By: Claude Opus 5.5 --- .../transforms/0128-detect-language.hpl | 217 ++++++++++++++++++ .../golden-0128-detect-language-chosen.csv | 6 + .../datasets/golden-0128-detect-language.csv | 6 + .../transforms/main-0128-detect-language.hwf | 79 +++++++ .../golden-0128-detect-language-chosen.json | 24 ++ .../dataset/golden-0128-detect-language.json | 24 ++ .../unit-test/0128-detect-language UNIT.json | 49 ++++ 7 files changed, 405 insertions(+) create mode 100644 integration-tests/transforms/0128-detect-language.hpl create mode 100644 integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv create mode 100644 integration-tests/transforms/datasets/golden-0128-detect-language.csv create mode 100644 integration-tests/transforms/main-0128-detect-language.hwf create mode 100644 integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json create mode 100644 integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json create mode 100644 integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json diff --git a/integration-tests/transforms/0128-detect-language.hpl b/integration-tests/transforms/0128-detect-language.hpl new file mode 100644 index 00000000000..23f43378442 --- /dev/null +++ b/integration-tests/transforms/0128-detect-language.hpl @@ -0,0 +1,217 @@ + + + + + 0128-detect-language + Y + Detect language with the default options (every spoken language, high accuracy, one thread) and with a chosen set of languages, low accuracy mode and parallel jobs. Blank and null texts pass with UNKNOWN. + + + Normal + + + DETECT_LANGUAGE_FIELD + text + The corpus field accepts a variable + + + DETECT_LANGUAGE_THIRD + GERMAN + A language to choose accepts a variable + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + texts + detect, default options + Y + + + texts + detect, chosen languages + Y + + + detect, default options + validate + Y + + + detect, chosen languages + validate chosen + Y + + + + texts + DataGrid + English, French and German text, only spaces, and no text at all (null). + N + + 1 + + none + + + + + 1 + The children are playing in the garden while their parents prepare dinner in the kitchen. + + + 2 + Les enfants jouent dans le jardin pendant que leurs parents préparent le dîner dans la cuisine. + + + 3 + Die Kinder spielen im Garten, während ihre Eltern in der Küche das Abendessen vorbereiten. + + + 4 + + + + 5 + + + + + + -1 + -1 + N + id + String + + + -1 + -1 + N + text + String + + + + + 112 + 160 + + + + detect, default options + DetectLanguage + Every spoken language, high accuracy, one thread. + Y + + 1 + + none + + + text + N + N + + + + + 336 + 96 + + + + detect, chosen languages + DetectLanguage + English, French and German only, low accuracy mode, parallel jobs: the rows can come out in another order. + Y + + 1 + + none + + + ${DETECT_LANGUAGE_FIELD} + Y + Y + + + ENGLISH + + + FRENCH + + + ${DETECT_LANGUAGE_THIRD} + + + + + 336 + 224 + + + + validate + Dummy + + Y + + 1 + + none + + + + + 560 + 96 + + + + validate chosen + Dummy + + Y + + 1 + + none + + + + + 560 + 224 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv new file mode 100644 index 00000000000..782fe6e881f --- /dev/null +++ b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv @@ -0,0 +1,6 @@ +id,detected_language +1,ENGLISH +2,FRENCH +3,GERMAN +4,UNKNOWN +5,UNKNOWN diff --git a/integration-tests/transforms/datasets/golden-0128-detect-language.csv b/integration-tests/transforms/datasets/golden-0128-detect-language.csv new file mode 100644 index 00000000000..782fe6e881f --- /dev/null +++ b/integration-tests/transforms/datasets/golden-0128-detect-language.csv @@ -0,0 +1,6 @@ +id,detected_language +1,ENGLISH +2,FRENCH +3,GERMAN +4,UNKNOWN +5,UNKNOWN diff --git a/integration-tests/transforms/main-0128-detect-language.hwf b/integration-tests/transforms/main-0128-detect-language.hwf new file mode 100644 index 00000000000..9d1831237c8 --- /dev/null +++ b/integration-tests/transforms/main-0128-detect-language.hwf @@ -0,0 +1,79 @@ + + + + main-0128-detect-language + Y + + + + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 112 + 64 + + + + 0128-detect-language UNIT + + RunPipelineTests + + + + 0128-detect-language UNIT + + + N + 304 + 64 + + + + + + Start + 0128-detect-language UNIT + Y + Y + Y + + + + + + diff --git a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json new file mode 100644 index 00000000000..d9c08c0784e --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-0128-detect-language-chosen.csv", + "name": "golden-0128-detect-language-chosen", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "id", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "detected_language", + "field_format": "" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json new file mode 100644 index 00000000000..7c1acb1722c --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-0128-detect-language.csv", + "name": "golden-0128-detect-language", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "id", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "detected_language", + "field_format": "" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json b/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json new file mode 100644 index 00000000000..99ba566cdc7 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json @@ -0,0 +1,49 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "${HOP_UNIT_TESTS_FOLDER}", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "detected_language", + "data_set_field": "detected_language" + } + ], + "field_order": [ + "id" + ], + "data_set_name": "golden-0128-detect-language", + "transform_name": "validate" + }, + { + "field_mappings": [ + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "detected_language", + "data_set_field": "detected_language" + } + ], + "field_order": [ + "id" + ], + "data_set_name": "golden-0128-detect-language-chosen", + "transform_name": "validate chosen" + } + ], + "input_data_sets": [], + "name": "0128-detect-language UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0128-detect-language.hpl" +}