diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
index ace4d07dedc..6268a61e9f9 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
@@ -35,8 +35,39 @@ https://github.com/pemistahl/lingua?tab=readme-ov-file#3-which-languages-are-sup
|Option|Description
|Transform name|Name of the transform.
This name must be unique throughout the pipeline.
+|Corpus text field|The field with the text to examine.
+A variable is allowed.
+|Use Parallelism|Detect the language of several rows at the same time.
+The rows can leave the transform in another order than they came in.
+|Low accuracy mode|Use only the three-letter language models.
+This is faster and uses far less memory, but it is less accurate: short texts of a few words are often given the wrong language.
+Use it for longer texts.
+|Languages to detect|The languages the text can be in, by their Lingua name (`ENGLISH`, `FRENCH`, ...).
+A variable is allowed.
+Leave the list empty to detect every spoken language.
+List at least two languages.
|===
+== Output fields
+
+[options="header"]
+|===
+|Field|Description
+|detected_language|The name of the detected language, for example `ENGLISH`, or `UNKNOWN`.
+|detected_language_confidence|Lingua's score for that language, relative to the other languages.
+The detected language is the one with the highest score, so the value is always 1: it is not a probability and does not tell how sure the detection is.
+Empty when the language is `UNKNOWN`.
+|===
+
+A row with an empty text passes with `UNKNOWN` and no confidence.
+The language is also `UNKNOWN` when the two most likely languages score exactly the same.
+
+== Performance
+
+Detecting every spoken language loads the models of all 74 spoken languages: about 1.4 GB of memory, released when the transform finishes.
+When you know which languages the data can contain, list them: with five languages the transform is about 2.5 times as fast and needs about 175 MB.
+A text in a language that is not listed is given one of the listed languages, or `UNKNOWN`.
+
== Installation
This transform is not part of the default Hop client.
diff --git a/integration-tests/transforms/0128-detect-language.hpl b/integration-tests/transforms/0128-detect-language.hpl
new file mode 100644
index 00000000000..23f43378442
--- /dev/null
+++ b/integration-tests/transforms/0128-detect-language.hpl
@@ -0,0 +1,217 @@
+
+
+
+
+ 0128-detect-language
+ Y
+ Detect language with the default options (every spoken language, high accuracy, one thread) and with a chosen set of languages, low accuracy mode and parallel jobs. Blank and null texts pass with UNKNOWN.
+
+
+ Normal
+
+
+ DETECT_LANGUAGE_FIELD
+ text
+ The corpus field accepts a variable
+
+
+ DETECT_LANGUAGE_THIRD
+ GERMAN
+ A language to choose accepts a variable
+
+
+ N
+ 1000
+ 100
+ -
+ 2026/11/16 12:00:00.000
+ -
+ 2026/11/16 12:00:00.000
+
+
+
+
+
+ texts
+ detect, default options
+ Y
+
+
+ texts
+ detect, chosen languages
+ Y
+
+
+ detect, default options
+ validate
+ Y
+
+
+ detect, chosen languages
+ validate chosen
+ Y
+
+
+
+ texts
+ DataGrid
+ English, French and German text, only spaces, and no text at all (null).
+ N
+
+ 1
+
+ none
+
+
+
+
+ - 1
+ - The children are playing in the garden while their parents prepare dinner in the kitchen.
+
+
+ - 2
+ - Les enfants jouent dans le jardin pendant que leurs parents préparent le dîner dans la cuisine.
+
+
+ - 3
+ - Die Kinder spielen im Garten, während ihre Eltern in der Küche das Abendessen vorbereiten.
+
+
+ - 4
+ -
+
+
+ - 5
+
+
+
+
+
+ -1
+ -1
+ N
+ id
+ String
+
+
+ -1
+ -1
+ N
+ text
+ String
+
+
+
+
+ 112
+ 160
+
+
+
+ detect, default options
+ DetectLanguage
+ Every spoken language, high accuracy, one thread.
+ Y
+
+ 1
+
+ none
+
+
+ text
+ N
+ N
+
+
+
+
+ 336
+ 96
+
+
+
+ detect, chosen languages
+ DetectLanguage
+ English, French and German only, low accuracy mode, parallel jobs: the rows can come out in another order.
+ Y
+
+ 1
+
+ none
+
+
+ ${DETECT_LANGUAGE_FIELD}
+ Y
+ Y
+
+
+ ENGLISH
+
+
+ FRENCH
+
+
+ ${DETECT_LANGUAGE_THIRD}
+
+
+
+
+ 336
+ 224
+
+
+
+ validate
+ Dummy
+
+ Y
+
+ 1
+
+ none
+
+
+
+
+ 560
+ 96
+
+
+
+ validate chosen
+ Dummy
+
+ Y
+
+ 1
+
+ none
+
+
+
+
+ 560
+ 224
+
+
+
+
+
+
diff --git a/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
new file mode 100644
index 00000000000..782fe6e881f
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
@@ -0,0 +1,6 @@
+id,detected_language
+1,ENGLISH
+2,FRENCH
+3,GERMAN
+4,UNKNOWN
+5,UNKNOWN
diff --git a/integration-tests/transforms/datasets/golden-0128-detect-language.csv b/integration-tests/transforms/datasets/golden-0128-detect-language.csv
new file mode 100644
index 00000000000..782fe6e881f
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0128-detect-language.csv
@@ -0,0 +1,6 @@
+id,detected_language
+1,ENGLISH
+2,FRENCH
+3,GERMAN
+4,UNKNOWN
+5,UNKNOWN
diff --git a/integration-tests/transforms/main-0128-detect-language.hwf b/integration-tests/transforms/main-0128-detect-language.hwf
new file mode 100644
index 00000000000..9d1831237c8
--- /dev/null
+++ b/integration-tests/transforms/main-0128-detect-language.hwf
@@ -0,0 +1,79 @@
+
+
+
+ main-0128-detect-language
+ Y
+
+
+
+ -
+ 2026/11/16 12:00:00.000
+ -
+ 2026/11/16 12:00:00.000
+
+
+
+
+ Start
+
+ SPECIAL
+
+ 1
+ 12
+ 60
+ 0
+ 0
+ N
+ 0
+ 1
+ N
+ 112
+ 64
+
+
+
+ 0128-detect-language UNIT
+
+ RunPipelineTests
+
+
+
+ 0128-detect-language UNIT
+
+
+ N
+ 304
+ 64
+
+
+
+
+
+ Start
+ 0128-detect-language UNIT
+ Y
+ Y
+ Y
+
+
+
+
+
+
diff --git a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
new file mode 100644
index 00000000000..d9c08c0784e
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0128-detect-language-chosen.csv",
+ "name": "golden-0128-detect-language-chosen",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "detected_language",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
new file mode 100644
index 00000000000..7c1acb1722c
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0128-detect-language.csv",
+ "name": "golden-0128-detect-language",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "detected_language",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json b/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json
new file mode 100644
index 00000000000..99ba566cdc7
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json
@@ -0,0 +1,49 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "detected_language",
+ "data_set_field": "detected_language"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0128-detect-language",
+ "transform_name": "validate"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "detected_language",
+ "data_set_field": "detected_language"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0128-detect-language-chosen",
+ "transform_name": "validate chosen"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0128-detect-language UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0128-detect-language.hpl"
+}
diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
index bf569b32e74..611d2aac1d9 100644
--- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
+++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
@@ -18,7 +18,6 @@
package org.apache.hop.pipeline.transforms.language;
import static com.github.pemistahl.lingua.api.Language.UNKNOWN;
-import static com.github.pemistahl.lingua.api.LanguageDetectorBuilder.fromAllSpokenLanguages;
import static java.lang.System.arraycopy;
import static java.util.concurrent.ForkJoinPool.commonPool;
import static org.apache.commons.lang3.StringUtils.isBlank;
@@ -27,7 +26,9 @@
import com.github.pemistahl.lingua.api.Language;
import com.github.pemistahl.lingua.api.LanguageDetector;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
import java.util.SortedMap;
import java.util.concurrent.ExecutorService;
@@ -44,6 +45,7 @@ public class DetectLanguage extends BaseTransform PKG = DetectLanguage.class;
private static final String CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD =
"DetectLanguage.Exception.CouldnotFindField";
+ private static final String ERROR_CODE = "DetectLanguage001";
public DetectLanguage(
TransformMeta transformMeta,
@@ -71,15 +73,52 @@ public DetectLanguage(
private final Object lock = new Object();
- private final LanguageDetector detector =
- fromAllSpokenLanguages().withPreloadedLanguageModels().build();
+ /** The outcome of one detection: the language name and its confidence, null when unknown. */
+ record Detection(String language, Double confidence) {}
- private void awaitJobs() {
- awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ /**
+ * The Lingua builder for the configured options.
+ *
+ * @param languages the languages to choose from; empty means all spoken languages
+ * @param lowAccuracy use only the trigram models
+ */
+ static LanguageDetectorBuilder createBuilder(List languages, boolean lowAccuracy) {
+ LanguageDetectorBuilder builder =
+ languages.isEmpty()
+ ? LanguageDetectorBuilder.fromAllSpokenLanguages()
+ : LanguageDetectorBuilder.fromLanguages(languages.toArray(new Language[0]));
+ if (lowAccuracy) {
+ builder = builder.withLowAccuracyMode();
+ }
+ return builder.withPreloadedLanguageModels();
}
- private void awaitJobs(final long timeoutMillis) {
+ @Override
+ public boolean init() {
+ if (!super.init()) {
+ return false;
+ }
+ try {
+ data.detector = createBuilder(meta.resolveLanguages(this), meta.isLowAccuracy()).build();
+ return true;
+ } catch (Exception e) {
+ logError(BaseMessages.getString(PKG, "DetectLanguage.Error.DetectorNotBuilt"), e);
+ return false;
+ }
+ }
+
+ /**
+ * Waits for accepted asynchronous work, at most {@link #DRAIN_TIMEOUT_MILLIS}.
+ *
+ * @return true when every job finished, false when some were abandoned
+ */
+ private boolean awaitJobs() {
+ return awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ }
+
+ private boolean awaitJobs(final long timeoutMillis) {
boolean interrupted = false;
+ boolean drained;
final long deadline = System.currentTimeMillis() + timeoutMillis;
synchronized (lock) {
long remaining = timeoutMillis;
@@ -93,7 +132,8 @@ private void awaitJobs(final long timeoutMillis) {
}
remaining = deadline - System.currentTimeMillis();
}
- if (jobs > 0) {
+ drained = jobs == 0;
+ if (!drained) {
// A worker is stuck and no provider timeout applies. Do not block disposal forever: log and
// abandon the remaining jobs.
logBasic(
@@ -107,6 +147,7 @@ private void awaitJobs(final long timeoutMillis) {
if (interrupted) {
Thread.currentThread().interrupt();
}
+ return drained;
}
/**
@@ -150,8 +191,16 @@ private void finish() throws HopException {
@Override
public void dispose() {
// The common pool is shared. Drain our work, but never shut the pool down.
- awaitJobs();
- detector.unloadLanguageModels();
+ // After an error or a stop the transform ends without finish(), and parallel jobs can still be
+ // queued. They use the detector: unloading under them would make them fail on a missing
+ // detector, or load the models again with nothing left to unload them.
+ boolean drained = awaitJobs();
+ if (drained && data.detector != null) {
+ // Lingua keeps its models in static maps: release what this transform asked for. Another
+ // Detect language transform that is still running loads them again on demand.
+ data.detector.unloadLanguageModels();
+ data.detector = null;
+ }
final boolean shouldLog;
synchronized (lock) {
// Only log a failure we have not already surfaced through the run loop, so it is not logged
@@ -189,7 +238,9 @@ public boolean processRow() throws HopException {
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
- if (isEmpty(meta.getCorpusField())) {
+ // The field combo accepts variables
+ data.corpusField = resolve(meta.getCorpusField());
+ if (isEmpty(data.corpusField)) {
throw new HopException(
BaseMessages.getString(PKG, "DetectLanguage.Error.CorpusFieldMissing"));
}
@@ -198,7 +249,6 @@ public boolean processRow() throws HopException {
} // End If first
try {
- process(r);
if (isRowLevel()) {
logRowlevel(
BaseMessages.getString(
@@ -206,6 +256,7 @@ public boolean processRow() throws HopException {
"DetectLanguage.LineNumber",
getLinesRead() + " : " + getInputRowMeta().getString(r)));
}
+ process(r);
} catch (Exception e) {
// An infrastructure/worker failure must not be attributed to the current input row.
checkFailure();
@@ -216,8 +267,10 @@ public boolean processRow() throws HopException {
}
private void process(Object[] inputRow) throws HopException {
- String corpus = trim((String) inputRow[data.indexOfCorpusField]);
+ String corpus = trim(getInputRowMeta().getString(inputRow, data.indexOfCorpusField));
if (isBlank(corpus)) {
+ // Nothing to detect: the row passes with an unknown language and no confidence.
+ putDetection(inputRow, new Detection(UNKNOWN.name(), null));
return;
}
@@ -230,11 +283,13 @@ private void process(Object[] inputRow) throws HopException {
private void handleRowError(Object[] row, Exception e) throws HopException {
if (!getTransformMeta().isDoingErrorHandling()) {
- throw new HopException("Error processing row", e);
+ throw new HopException(
+ BaseMessages.getString(PKG, "DetectLanguage.ErrorInTransformRunning") + e.getMessage(),
+ e);
}
// Error output has the same serialization requirement as normal output.
synchronized (this) {
- putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "DetectLanguage001");
+ putError(getInputRowMeta(), row, 1, e.toString(), data.corpusField, ERROR_CODE);
}
}
@@ -257,11 +312,18 @@ private void processAsync(String text, Object[] inputRow) throws HopException {
() -> {
try {
try {
- processSync(text, inputRow);
+ // Stopped: nobody wants the row anymore, and dispose() waits for the queue to
+ // drain.
+ if (!isStopped() || isSafeStopped()) {
+ processSync(text, inputRow);
+ }
} catch (Exception e) {
handleRowError(inputRow, e);
}
} catch (Throwable e) {
+ // Throwable, not Exception: nobody reads the pool's Future, so an Error caught
+ // nowhere else would make the row disappear without a trace. An Error says nothing
+ // about the row: it always stops the transform.
synchronized (lock) {
if (failure == null) {
failure = e;
@@ -286,16 +348,14 @@ private void processAsync(String text, Object[] inputRow) throws HopException {
}
}
- @SuppressWarnings(
- "javabugs:S2259") // the probabilities come from the map's own keys and are never null
private void cacheIndexPositions() throws HopException {
if (data.indexOfCorpusField < 0) {
- data.indexOfCorpusField = data.previousRowMeta.indexOfValue(meta.getCorpusField());
+ data.indexOfCorpusField = data.previousRowMeta.indexOfValue(data.corpusField);
if (data.indexOfCorpusField < 0) {
// The field is unreachable!
throw new HopException(
BaseMessages.getString(
- PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD, meta.getCorpusField()));
+ PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD, data.corpusField));
}
}
if (data.indexOfDetectedLanguage < 0) {
@@ -321,9 +381,13 @@ private void cacheIndexPositions() throws HopException {
}
}
+ /**
+ * Detects the language of a text: the most likely language, unless the second one is as likely or
+ * closer than Lingua's minimum relative distance, in which case the language is unknown.
+ */
@SuppressWarnings(
"javabugs:S2259") // the probabilities come from the map's own keys and are never null
- private void processSync(String corpus, Object[] inputRow) throws HopTransformException {
+ static Detection detect(LanguageDetector detector, String corpus) {
SortedMap confidenceValues = detector.computeLanguageConfidenceValues(corpus);
Language detectedLanguage = UNKNOWN;
Double confidence = null;
@@ -348,11 +412,18 @@ private void processSync(String corpus, Object[] inputRow) throws HopTransformEx
}
}
}
+ return new Detection(detectedLanguage.name(), confidence);
+ }
+
+ private void processSync(String corpus, Object[] inputRow) throws HopTransformException {
+ putDetection(inputRow, detect(data.detector, corpus));
+ }
+ private void putDetection(Object[] inputRow, Detection detection) throws HopTransformException {
Object[] outputRow = new Object[inputRow.length + 2 + OVER_ALLOCATE_SIZE];
arraycopy(inputRow, 0, outputRow, 0, inputRow.length);
- outputRow[data.indexOfDetectedLanguage] = detectedLanguage.name();
- outputRow[data.indexOfDetectedLanguageConfidence] = confidence;
+ outputRow[data.indexOfDetectedLanguage] = detection.language();
+ outputRow[data.indexOfDetectedLanguageConfidence] = detection.confidence();
putRow(data.outputRowMeta, outputRow);
}
diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
new file mode 100644
index 00000000000..7b48e58e34f
--- /dev/null
+++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.language;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiTableColumn;
+import org.apache.hop.core.gui.plugin.GuiTableColumnType;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+/** One language the Detect language transform may answer. */
+@Getter
+@Setter
+public class DetectLanguageChoice {
+
+ /** The Lingua language name, for example ENGLISH. A variable is allowed. */
+ @HopMetadataProperty(
+ key = "name",
+ injectionKey = "LANGUAGE",
+ injectionKeyDescription = "DetectLanguage.Injection.Language")
+ @GuiTableColumn(
+ id = "name",
+ order = "10",
+ type = GuiTableColumnType.COMBO,
+ comboValuesMethod = "languageNames",
+ label = "i18n::DetectLanguage.Column.Language.Label",
+ variables = true,
+ width = 250)
+ private String name = "";
+
+ public DetectLanguageChoice() {}
+
+ public DetectLanguageChoice(String name) {
+ this.name = name;
+ }
+}
diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
index ef5d72bf98b..12f16dbfb34 100644
--- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
+++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.language;
+import com.github.pemistahl.lingua.api.LanguageDetector;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
@@ -30,6 +31,12 @@ public class DetectLanguageData extends BaseTransformData implements ITransformD
public IRowMeta outputRowMeta;
public int NrPrevFields;
+ /** The name of the corpus field, its variables resolved. */
+ public String corpusField;
+
+ /** Built in init() from the configured languages and accuracy mode. */
+ public LanguageDetector detector;
+
public DetectLanguageData() {
super();
indexOfCorpusField = -1;
diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
index c5a1fcb7664..372b679cee1 100644
--- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
+++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
@@ -14,48 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hop.pipeline.transforms.language;
-import static org.apache.hop.core.util.Utils.isEmpty;
-import static org.apache.hop.i18n.BaseMessages.getString;
-import static org.eclipse.swt.SWT.BORDER;
-import static org.eclipse.swt.SWT.CHECK;
-import static org.eclipse.swt.SWT.CURSOR_WAIT;
-import static org.eclipse.swt.SWT.READ_ONLY;
-import static org.eclipse.swt.SWT.RIGHT;
-
-import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.PipelineMeta;
-import org.apache.hop.pipeline.transform.ITransformDialog;
-import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
-import org.apache.hop.ui.core.dialog.ErrorDialog;
-import org.apache.hop.ui.core.widget.ComboItems;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
+import org.apache.hop.ui.core.widget.ComboVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
-import org.eclipse.swt.SWT;
-import org.eclipse.swt.custom.CCombo;
-import org.eclipse.swt.events.FocusEvent;
-import org.eclipse.swt.events.FocusListener;
-import org.eclipse.swt.graphics.Cursor;
-import org.eclipse.swt.layout.FormAttachment;
-import org.eclipse.swt.layout.FormData;
-import org.eclipse.swt.widgets.Button;
-import org.eclipse.swt.widgets.Label;
-import org.eclipse.swt.widgets.Listener;
+import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Shell;
-public class DetectLanguageDialog extends BaseTransformDialog implements ITransformDialog {
+public class DetectLanguageDialog extends BaseTransformDialog {
private static final Class> PKG = DetectLanguageDialog.class;
- private boolean gotPreviousFields = false;
-
- private CCombo wCorpusFieldName;
- private Button wParallelism;
private final DetectLanguageMeta input;
+ private GuiCompositeWidgets widgets;
public DetectLanguageDialog(
Shell parent,
@@ -69,82 +49,66 @@ public DetectLanguageDialog(
@Override
public String open() {
createShell(BaseMessages.getString(PKG, "DetectLanguageDialog.Shell.Title"));
-
buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build();
- Listener lsMod = e -> input.setChanged();
changed = input.hasChanged();
- // CorpusFieldName field
- Label wlCorpusFieldName = new Label(shell, RIGHT);
- wlCorpusFieldName.setText(getString(PKG, "DetectLanguageDialog.CorpusFieldName.Label"));
- PropsUi.setLook(wlCorpusFieldName);
- FormData fdlCorpusFieldName = new FormData();
- fdlCorpusFieldName.left = new FormAttachment(0, 0);
- fdlCorpusFieldName.right = new FormAttachment(middle, -margin);
- fdlCorpusFieldName.top = new FormAttachment(wSpacer, margin);
- wlCorpusFieldName.setLayoutData(fdlCorpusFieldName);
-
- wCorpusFieldName = new CCombo(shell, BORDER | READ_ONLY);
- PropsUi.setLook(wCorpusFieldName);
- wCorpusFieldName.addListener(SWT.Modify, lsMod);
- FormData fdCorpusFieldName = new FormData();
- fdCorpusFieldName.left = new FormAttachment(middle, 0);
- fdCorpusFieldName.top = new FormAttachment(wSpacer, margin);
- fdCorpusFieldName.right = new FormAttachment(100, 0);
- wCorpusFieldName.setLayoutData(fdCorpusFieldName);
- wCorpusFieldName.addFocusListener(
- new FocusListener() {
- @Override
- public void focusLost(FocusEvent e) {}
-
+ widgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ shell,
+ variables,
+ wTransformName,
+ wOk,
+ DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ input);
+ widgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
@Override
- public void focusGained(FocusEvent e) {
- Cursor busy = new Cursor(shell.getDisplay(), CURSOR_WAIT);
- shell.setCursor(busy);
- get();
- shell.setCursor(null);
- busy.dispose();
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) {
+ input.setChanged();
}
});
- // parallelism
- Label wlParallelism = new Label(shell, RIGHT);
- wlParallelism.setText(getString(PKG, "DetectLanguageDialog.Parallelism.Label"));
- PropsUi.setLook(wlParallelism);
- FormData fdlParallelism = new FormData();
- fdlParallelism.left = new FormAttachment(0, 0);
- fdlParallelism.top = new FormAttachment(wCorpusFieldName, margin);
- fdlParallelism.right = new FormAttachment(middle, -margin);
- wlParallelism.setLayoutData(fdlParallelism);
-
- wParallelism = new Button(shell, CHECK);
- wParallelism.setSelection(input.isParallelism());
- PropsUi.setLook(wParallelism);
- wParallelism.setToolTipText(getString(PKG, "DetectLanguageDialog.Parallelism.Tooltip"));
- FormData fdParallelism = new FormData();
- fdParallelism.left = new FormAttachment(middle, 0);
- fdParallelism.top = new FormAttachment(wCorpusFieldName, margin);
- fdParallelism.right = new FormAttachment(100, 0);
- wParallelism.setLayoutData(fdParallelism);
- wParallelism.addListener(SWT.Selection, lsMod);
-
- getData();
+ setFieldComboValues();
+ // Filling the field combo is not a change
input.setChanged(changed);
+
focusTransformName();
BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
-
return transformName;
}
- /** Copy information from the meta-data input to the dialog fields. */
- public void getData() {
-
- if (input.getCorpusField() != null) {
- wCorpusFieldName.setText(input.getCorpusField());
+ /**
+ * Stream field names cannot come from {@code comboValuesMethod}, which is handed only a log
+ * channel and a metadata provider, so the field combo is filled once the widgets exist. The
+ * configured field stays when the previous fields cannot be read.
+ */
+ private void setFieldComboValues() {
+ try {
+ IRowMeta fields = pipelineMeta.getPrevTransformFields(variables, transformName);
+ String[] names = fields == null ? new String[0] : fields.getFieldNames();
+ widgets.setComboValues(DetectLanguageMeta.WIDGET_CORPUS_FIELD, names);
+ // Setting items clears the widget's text, so put the configured field back afterwards.
+ if (!Utils.isEmpty(input.getCorpusField())) {
+ setComboText(DetectLanguageMeta.WIDGET_CORPUS_FIELD, input.getCorpusField());
+ }
+ } catch (Exception e) {
+ LogChannel.UI.logError(
+ BaseMessages.getString(PKG, "DetectLanguageDialog.FailedToGetFields.DialogMessage"), e);
}
+ }
- wParallelism.setSelection(input.isParallelism());
+ private void setComboText(String widgetId, String text) {
+ Control control = widgets.getWidgetsMap().get(widgetId);
+ if (control == null || control.isDisposed()) {
+ return;
+ }
+ if (control instanceof ComboVar comboVar) {
+ comboVar.setText(text);
+ } else if (control instanceof Combo combo) {
+ combo.setText(text);
+ }
}
private void cancel() {
@@ -154,32 +118,12 @@ private void cancel() {
}
private void ok() {
- if (isEmpty(wTransformName.getText())) {
+ if (Utils.isEmpty(wTransformName.getText())) {
return;
}
- input.setCorpusField(wCorpusFieldName.getText());
- input.setParallelism(wParallelism.getSelection());
-
- transformName = wTransformName.getText(); // return value
-
+ widgets.getWidgetsContents(input, DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID);
+ transformName = wTransformName.getText();
+ input.setChanged();
dispose();
}
-
- private void get() {
- if (!gotPreviousFields) {
- try {
- IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName);
- if (r != null) {
- ComboItems.setItemsKeepingText(wCorpusFieldName, r.getFieldNames());
- }
- gotPreviousFields = true;
- } catch (HopException ke) {
- new ErrorDialog(
- shell,
- getString(PKG, "DetectLanguageDialog.FailedToGetFields.DialogTitle"),
- getString(PKG, "DetectLanguageDialog.FailedToGetFields.DialogMessage"),
- ke);
- }
- }
- }
}
diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
index efb62eddbce..0f65485c4e0 100644
--- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
+++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
@@ -14,21 +14,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hop.pipeline.transforms.language;
import static org.apache.hop.core.ICheckResult.TYPE_RESULT_ERROR;
import static org.apache.hop.core.ICheckResult.TYPE_RESULT_OK;
+import static org.apache.hop.core.ICheckResult.TYPE_RESULT_WARNING;
import static org.apache.hop.core.util.Utils.isEmpty;
+import com.github.pemistahl.lingua.api.Language;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Locale;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaNumber;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -37,6 +50,9 @@
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
+@Getter
+@Setter
+@GuiPlugin
@Transform(
id = "DetectLanguage",
image = "detectlanguage.svg",
@@ -48,18 +64,88 @@
public class DetectLanguageMeta extends BaseTransformMeta {
private static final Class> PKG = DetectLanguageMeta.class;
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "DetectLanguage.Dialog";
+ public static final String WIDGET_CORPUS_FIELD = "corpusField";
+ public static final String WIDGET_PARALLELISM = "parallelism";
+ public static final String WIDGET_LOW_ACCURACY = "lowAccuracy";
+ public static final String WIDGET_LANGUAGES = "languages";
+
+ private static final String GROUP_OPTIONS = "i18n::DetectLanguage.Group.Options";
+ private static final String GROUP_LANGUAGES = "i18n::DetectLanguage.Group.Languages";
+
+ @GuiWidgetElement(
+ id = WIDGET_CORPUS_FIELD,
+ order = "0100",
+ type = GuiElementType.COMBO,
+ label = "i18n::DetectLanguageDialog.CorpusFieldName.Label",
+ toolTip = "i18n::DetectLanguageDialog.CorpusFieldName.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_OPTIONS,
+ groupOrder = "0100")
@HopMetadataProperty(
key = "corpusField",
injectionKey = "FIELD",
injectionKeyDescription = "DetectLanguage.Injection.CorpusField")
private String corpusField;
+ @GuiWidgetElement(
+ id = WIDGET_PARALLELISM,
+ order = "0200",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::DetectLanguageDialog.Parallelism.Label",
+ toolTip = "i18n::DetectLanguageDialog.Parallelism.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_OPTIONS,
+ groupOrder = "0100")
@HopMetadataProperty(
key = "parallelism",
injectionKey = "PARALLISM",
injectionKeyDescription = "DetectLanguage.Injection.Parallelism")
private boolean parallelism = false;
+ /**
+ * Lingua's low accuracy mode: only the trigram models are used. Faster and much lighter on
+ * memory, less accurate on short text. Off for pipelines saved before the option existed.
+ */
+ @GuiWidgetElement(
+ id = WIDGET_LOW_ACCURACY,
+ order = "0300",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::DetectLanguageDialog.LowAccuracy.Label",
+ toolTip = "i18n::DetectLanguageDialog.LowAccuracy.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_OPTIONS,
+ groupOrder = "0100")
+ @HopMetadataProperty(
+ key = "lowAccuracy",
+ injectionKey = "LOW_ACCURACY",
+ injectionKeyDescription = "DetectLanguage.Injection.LowAccuracy")
+ private boolean lowAccuracy = false;
+
+ /**
+ * The languages the transform may answer. Empty means every spoken language Lingua knows, which
+ * is what the transform did before the option existed.
+ */
+ @GuiWidgetElement(
+ id = WIDGET_LANGUAGES,
+ order = "0400",
+ type = GuiElementType.TABLE,
+ toolTip = "i18n::DetectLanguageDialog.Languages.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_LANGUAGES,
+ groupOrder = "0200",
+ tableRows = 8)
+ @HopMetadataProperty(
+ key = "language",
+ groupKey = "languages",
+ injectionGroupKey = "LANGUAGES",
+ injectionGroupDescription = "DetectLanguage.Injection.Languages")
+ private List languages = new ArrayList<>();
+
public DetectLanguageMeta() {
super();
}
@@ -67,6 +153,77 @@ public DetectLanguageMeta() {
@Override
public void setDefault() {
parallelism = false;
+ lowAccuracy = false;
+ languages = new ArrayList<>();
+ }
+
+ /** The names offered in the languages table: every language Lingua can detect. */
+ public List languageNames(ILogChannel log, IHopMetadataProvider metadataProvider) {
+ return Language.all().stream().map(Language::name).sorted().toList();
+ }
+
+ /**
+ * Resolves the configured languages.
+ *
+ * @return the languages to detect, without duplicates and in the configured order; empty when
+ * none is configured, which means all spoken languages
+ * @throws HopException when a name is not a Lingua language or when exactly one language is
+ * configured: detection needs at least two to choose from
+ */
+ public List resolveLanguages(IVariables variables) throws HopException {
+ return resolveLanguages(variables, null);
+ }
+
+ /**
+ * Resolves the configured languages, leaving out the names that still hold a variable.
+ *
+ * @param unresolved when not null, a name that still holds a variable after resolving is added
+ * here instead of failing: Verify does not know the values a pipeline only gets when it runs
+ */
+ List resolveLanguages(IVariables variables, List unresolved)
+ throws HopException {
+ List resolved = new ArrayList<>();
+ if (languages == null) {
+ return resolved;
+ }
+ for (DetectLanguageChoice choice : languages) {
+ String name = choice == null ? null : choice.getName();
+ if (variables != null) {
+ name = variables.resolve(name);
+ }
+ if (StringUtils.isBlank(name)) {
+ continue;
+ }
+ if (unresolved != null && StringUtil.containsVariableToken(name)) {
+ unresolved.add(name.trim());
+ continue;
+ }
+ Language language = findLanguage(name.trim());
+ if (language == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG, "DetectLanguage.Error.UnknownLanguage", name.trim()));
+ }
+ if (!resolved.contains(language)) {
+ resolved.add(language);
+ }
+ }
+ // With names still to be resolved, the number of languages is not known yet
+ if (resolved.size() == 1 && (unresolved == null || unresolved.isEmpty())) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "DetectLanguage.Error.SingleLanguage", resolved.get(0).name()));
+ }
+ return resolved;
+ }
+
+ private static Language findLanguage(String name) {
+ String upper = name.toUpperCase(Locale.ROOT);
+ for (Language language : Language.all()) {
+ if (language.name().equals(upper)) {
+ return language;
+ }
+ }
+ return null;
}
@Override
@@ -77,7 +234,6 @@ public void getFields(
TransformMeta nextTransform,
IVariables variables,
IHopMetadataProvider metadataProvider) {
-
// name = getParentTransformMeta().getName()
valueMetaString(r, name, "detected_language");
valueMetaNumber(r, name, "detected_language_confidence");
@@ -107,7 +263,6 @@ public void check(
IVariables variables,
IHopMetadataProvider metadataProvider) {
CheckResult cr;
-
if (isEmpty(corpusField)) {
cr =
new CheckResult(
@@ -123,6 +278,33 @@ public void check(
}
remarks.add(cr);
+ try {
+ List unresolved = new ArrayList<>();
+ List resolved = resolveLanguages(variables, unresolved);
+ if (!unresolved.isEmpty()) {
+ cr =
+ new CheckResult(
+ TYPE_RESULT_WARNING,
+ BaseMessages.getString(
+ PKG,
+ "DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime",
+ String.join(", ", unresolved)),
+ transformMeta);
+ } else {
+ cr =
+ new CheckResult(
+ TYPE_RESULT_OK,
+ resolved.isEmpty()
+ ? BaseMessages.getString(PKG, "DetectLanguageMeta.CheckResult.AllLanguages")
+ : BaseMessages.getString(
+ PKG, "DetectLanguageMeta.CheckResult.LanguagesOK", resolved.size()),
+ transformMeta);
+ }
+ } catch (HopException e) {
+ cr = new CheckResult(TYPE_RESULT_ERROR, e.getMessage().trim(), transformMeta);
+ }
+ remarks.add(cr);
+
// See if we have input streams leading to this transform!
if (input.length > 0) {
cr =
@@ -145,20 +327,4 @@ public void check(
public boolean supportsErrorHandling() {
return true;
}
-
- public String getCorpusField() {
- return corpusField;
- }
-
- public void setCorpusField(String corpusField) {
- this.corpusField = corpusField;
- }
-
- public boolean isParallelism() {
- return parallelism;
- }
-
- public void setParallelism(boolean parallelism) {
- this.parallelism = parallelism;
- }
}
diff --git a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
index be58a587f55..a319f4c8a0d 100644
--- a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
+++ b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
@@ -40,3 +40,19 @@ DetectLanguageMeta.CheckResult.ReceivingInfoFromOtherTransforms=Transform is rec
DetectLanguageMeta.CheckResult.ErrorOccurred=An error occurred:
DetectLanguageDialog.Parallelism.Label=Use Parallelism
DetectLanguageDialog.Parallelism.Tooltip=Process the rows in parallel
+DetectLanguageDialog.CorpusFieldName.Tooltip=The field with the text to examine
+DetectLanguageDialog.LowAccuracy.Label=Low accuracy mode
+DetectLanguageDialog.LowAccuracy.Tooltip=Use only the three-letter language models: faster and much lighter on memory, but less accurate. Meant for longer texts: short texts of a few words are often given the wrong language.
+DetectLanguageDialog.Languages.Tooltip=The languages the text can be in. Leave the list empty to detect every spoken language. A short list is faster and uses less memory; a text in a language that is not listed is given one of the listed languages or UNKNOWN.
+DetectLanguage.Group.Options=Options
+DetectLanguage.Group.Languages=Languages to detect
+DetectLanguage.Column.Language.Label=Language
+DetectLanguage.Injection.LowAccuracy=Low accuracy mode
+DetectLanguage.Injection.Languages=Languages to detect
+DetectLanguage.Injection.Language=Language name
+DetectLanguage.Error.UnknownLanguage=''{0}'' is not a language this transform can detect. Use a name from the list, for example ENGLISH.
+DetectLanguage.Error.SingleLanguage=Only one language is listed ({0}). List at least two languages, or leave the list empty to detect every spoken language.
+DetectLanguage.Error.DetectorNotBuilt=Unable to prepare the language detection
+DetectLanguageMeta.CheckResult.AllLanguages=Every spoken language is detected.
+DetectLanguageMeta.CheckResult.LanguagesOK={0} languages are detected.
+DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime=These languages are only known when the pipeline runs: {0}
diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
index 3806cae0642..75ca03e3046 100644
--- a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
+++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
@@ -23,18 +23,15 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.github.pemistahl.lingua.api.LanguageDetector;
-import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
import java.lang.reflect.Field;
import java.util.List;
import java.util.TreeMap;
@@ -48,12 +45,12 @@
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.mockito.MockedStatic;
@Timeout(10)
class DetectLanguageAsyncTest {
@@ -72,15 +69,14 @@ void setUp() throws Exception {
PipelineMeta pipelineMeta = new PipelineMeta();
transformMeta = spy(new TransformMeta("test", meta));
pipelineMeta.addTransform(transformMeta);
- try (MockedStatic factory =
- mockStatic(LanguageDetectorBuilder.class)) {
- LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class, RETURNS_SELF);
- LanguageDetector detector = mock(LanguageDetector.class);
- factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder);
- when(builder.build()).thenReturn(detector);
- when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>());
- transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null));
- }
+ LanguageDetector detector = mock(LanguageDetector.class);
+ when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>());
+ data.detector = detector;
+ data.corpusField = "input";
+ transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null));
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("input"));
+ transform.setInputRowMeta(inputRowMeta);
transform.first = false;
data.indexOfCorpusField = 0;
data.indexOfDetectedLanguage = 1;
diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
index e4cda78ed17..c13196035c5 100644
--- a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
+++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
@@ -14,15 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hop.pipeline.transforms.language;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.github.pemistahl.lingua.api.Language;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
+import org.w3c.dom.Node;
class DetectLanguageMetaTest {
@RegisterExtension
@@ -30,11 +43,170 @@ class DetectLanguageMetaTest {
@Test
void testLoadSave() throws HopException {
- List attributes = List.of("corpusField", "parallelism");
+ List attributes = List.of("corpusField", "parallelism", "lowAccuracy");
+ // the languages list has its own round trip test below
LoadSaveTester loadSaveTester =
- new LoadSaveTester<>(DetectLanguageMeta.class, attributes);
+ new LoadSaveTester<>(
+ DetectLanguageMeta.class,
+ attributes,
+ List.of("languages"),
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ null);
loadSaveTester.testSerialization();
}
+
+ /** A pipeline saved before the language list and the accuracy mode existed. */
+ @Test
+ void aTransformSavedBeforeTheNewOptionsKeepsItsBehavior() throws Exception {
+ DetectLanguageMeta meta =
+ load(
+ "commentY");
+
+ assertEquals("comment", meta.getCorpusField());
+ assertTrue(meta.isParallelism());
+ assertFalse(meta.isLowAccuracy());
+ assertTrue(meta.getLanguages().isEmpty());
+ // no languages listed: every spoken language, as before
+ assertTrue(meta.resolveLanguages(new Variables()).isEmpty());
+ }
+
+ @Test
+ void theNewOptionsSurviveAnXmlRoundTrip() throws Exception {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.setLowAccuracy(true);
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("FRENCH"));
+
+ DetectLanguageMeta copy = load("" + meta.getXml() + "");
+
+ assertEquals("comment", copy.getCorpusField());
+ assertFalse(copy.isParallelism());
+ assertTrue(copy.isLowAccuracy());
+ assertEquals(
+ List.of("ENGLISH", "FRENCH"),
+ copy.getLanguages().stream().map(DetectLanguageChoice::getName).toList());
+ }
+
+ @Test
+ void languagesResolveWithVariablesAnyCaseAndNoDuplicates() throws Exception {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("english"));
+ meta.getLanguages().add(new DetectLanguageChoice(" ${SECOND} "));
+ meta.getLanguages().add(new DetectLanguageChoice(""));
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ Variables variables = new Variables();
+ variables.setVariable("SECOND", "French");
+
+ assertEquals(List.of(Language.ENGLISH, Language.FRENCH), meta.resolveLanguages(variables));
+ }
+
+ @Test
+ void anUnknownLanguageIsRefused() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+
+ HopException e = assertThrows(HopException.class, () -> meta.resolveLanguages(new Variables()));
+ assertTrue(e.getMessage().contains("KLINGON"));
+ }
+
+ @Test
+ void aSingleLanguageIsRefused() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+
+ HopException e = assertThrows(HopException.class, () -> meta.resolveLanguages(new Variables()));
+ assertTrue(e.getMessage().contains("ENGLISH"));
+ }
+
+ @Test
+ void theLanguageListOffersEveryLinguaLanguage() {
+ List names = new DetectLanguageMeta().languageNames(null, null);
+
+ assertEquals(Language.all().size(), names.size());
+ assertTrue(names.contains("ENGLISH"));
+ assertFalse(names.contains("UNKNOWN"));
+ }
+
+ @Test
+ void checkReportsABadLanguageList() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+
+ List remarks = new ArrayList<>();
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("Detect");
+ meta.check(
+ remarks,
+ null,
+ transformMeta,
+ null,
+ new String[] {"in"},
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("KLINGON")));
+ }
+
+ /** A language from a variable that only gets a value when the pipeline runs is not an error. */
+ @Test
+ void checkLeavesLanguagesFromRuntimeVariablesToTheRun() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}"));
+
+ List remarks = new ArrayList<>();
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("Detect");
+ meta.check(
+ remarks,
+ null,
+ transformMeta,
+ null,
+ new String[] {"in"},
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertTrue(remarks.stream().noneMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR));
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("${SECOND_LANGUAGE}")));
+ }
+
+ @Test
+ void runningWithAnUnresolvedLanguageStillFails() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}"));
+
+ assertThrows(HopException.class, () -> meta.resolveLanguages(new Variables()));
+ }
+
+ private static DetectLanguageMeta load(String xml) throws Exception {
+ Node node = XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), "transform");
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.loadXml(node, new MemoryMetadataProvider());
+ return meta;
+ }
}
diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
new file mode 100644
index 00000000000..6f936af4e29
--- /dev/null
+++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.language;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import com.github.pemistahl.lingua.api.Language;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class DetectLanguageTest {
+
+ private static final String ENGLISH_TEXT =
+ "The weather is lovely today and we are going for a long walk in the park";
+ private static final String FRENCH_TEXT =
+ "Il fait très beau aujourd'hui et nous allons nous promener dans le parc";
+ private static final String GERMAN_TEXT =
+ "Das Wetter ist heute sehr schön und wir gehen im Park spazieren";
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper mockHelper;
+
+ /** Disposed after every test, also a failed one: the Lingua models live in static maps. */
+ private final List transforms = new ArrayList<>();
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>(
+ "Detect language", DetectLanguageMeta.class, DetectLanguageData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ when(mockHelper.transformMeta.getCopies(any())).thenReturn(1);
+ when(mockHelper.transformMeta.isPartitioned()).thenReturn(false);
+ when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(false);
+ }
+
+ @AfterEach
+ void tearDown() {
+ for (TestDetectLanguage transform : transforms) {
+ transform.dispose();
+ }
+ mockHelper.cleanUp();
+ }
+
+ /** Without options the detector is what the transform always built. */
+ @Test
+ void theDefaultDetectorCoversEverySpokenLanguageWithHighAccuracy() {
+ LanguageDetectorBuilder builder = DetectLanguage.createBuilder(List.of(), false);
+
+ assertEquals(Language.allSpokenOnes(), builder.getLanguages$lingua());
+ assertFalse(builder.isLowAccuracyModeEnabled$lingua());
+ assertTrue(builder.isEveryLanguageModelPreloaded$lingua());
+ }
+
+ @Test
+ void theConfiguredLanguagesAndAccuracyModeReachTheDetector() {
+ LanguageDetectorBuilder builder =
+ DetectLanguage.createBuilder(List.of(Language.ENGLISH, Language.FRENCH), true);
+
+ assertEquals(List.of(Language.ENGLISH, Language.FRENCH), builder.getLanguages$lingua());
+ assertTrue(builder.isLowAccuracyModeEnabled$lingua());
+ }
+
+ @Test
+ void rowsGetTheirLanguageAndBlankTextsPassAsUnknown() throws Exception {
+ DetectLanguage transform = createTransform(meta(false, false));
+ assertTrue(transform.init());
+
+ BlockingRowSet output =
+ run(transform, row(1L, ENGLISH_TEXT), row(2L, null), row(3L, FRENCH_TEXT), row(4L, " "));
+
+ Object[] english = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(1L, english[0]);
+ assertEquals("ENGLISH", english[2]);
+ assertNotNull(english[3]);
+
+ // a null or blank text used to drop the row
+ Object[] empty = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(2L, empty[0]);
+ assertEquals("UNKNOWN", empty[2]);
+ assertNull(empty[3]);
+
+ assertEquals("FRENCH", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+
+ Object[] blank = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(4L, blank[0]);
+ assertEquals("UNKNOWN", blank[2]);
+ assertNull(blank[3]);
+
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ }
+
+ @Test
+ void theCorpusFieldCanComeFromAVariable() throws Exception {
+ DetectLanguageMeta meta = meta(false, false);
+ meta.setCorpusField("${TEXT_FIELD}");
+ DetectLanguage transform = createTransform(meta);
+ assertTrue(transform.init());
+ transform.setVariable("TEXT_FIELD", "comment");
+
+ BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT));
+
+ assertEquals("GERMAN", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+ }
+
+ @Test
+ void lowAccuracyModeFillsBothFields() throws Exception {
+ DetectLanguage transform = createTransform(meta(false, true));
+ assertTrue(transform.init());
+
+ BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT));
+
+ Object[] german = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals("GERMAN", german[2]);
+ assertNotNull(german[3]);
+ }
+
+ @Test
+ void parallelModeWritesEveryRow() throws Exception {
+ DetectLanguage transform = createTransform(meta(true, false));
+ assertTrue(transform.init());
+
+ BlockingRowSet output =
+ run(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L, GERMAN_TEXT));
+
+ Map