Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,39 @@ https://github.com/pemistahl/lingua?tab=readme-ov-file#3-which-languages-are-sup
|Option|Description
|Transform name|Name of the transform.
This name must be unique throughout the pipeline.
|Corpus text field|The field with the text to examine.
A variable is allowed.
|Use Parallelism|Detect the language of several rows at the same time.
The rows can leave the transform in another order than they came in.
|Low accuracy mode|Use only the three-letter language models.
This is faster and uses far less memory, but it is less accurate: short texts of a few words are often given the wrong language.
Use it for longer texts.
|Languages to detect|The languages the text can be in, by their Lingua name (`ENGLISH`, `FRENCH`, ...).
A variable is allowed.
Leave the list empty to detect every spoken language.
List at least two languages.
|===

== Output fields

[options="header"]
|===
|Field|Description
|detected_language|The name of the detected language, for example `ENGLISH`, or `UNKNOWN`.
|detected_language_confidence|Lingua's score for that language, relative to the other languages.
The detected language is the one with the highest score, so the value is always 1: it is not a probability and does not tell how sure the detection is.
Empty when the language is `UNKNOWN`.
|===

A row with an empty text passes with `UNKNOWN` and no confidence.
The language is also `UNKNOWN` when the two most likely languages score exactly the same.

== Performance

Detecting every spoken language loads the models of all 74 spoken languages: about 1.4 GB of memory, released when the transform finishes.
When you know which languages the data can contain, list them: with five languages the transform is about 2.5 times as fast and needs about 175 MB.
A text in a language that is not listed is given one of the listed languages, or `UNKNOWN`.

== Installation

This transform is not part of the default Hop client.
Expand Down
217 changes: 217 additions & 0 deletions integration-tests/transforms/0128-detect-language.hpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--

Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

-->
<pipeline>
<info>
<name>0128-detect-language</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description>Detect language with the default options (every spoken language, high accuracy, one thread) and with a chosen set of languages, low accuracy mode and parallel jobs. Blank and null texts pass with UNKNOWN.</description>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
<parameter>
<name>DETECT_LANGUAGE_FIELD</name>
<default_value>text</default_value>
<description>The corpus field accepts a variable</description>
</parameter>
<parameter>
<name>DETECT_LANGUAGE_THIRD</name>
<default_value>GERMAN</default_value>
<description>A language to choose accepts a variable</description>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2026/11/16 12:00:00.000</created_date>
<modified_user>-</modified_user>
<modified_date>2026/11/16 12:00:00.000</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>texts</from>
<to>detect, default options</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>texts</from>
<to>detect, chosen languages</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>detect, default options</from>
<to>validate</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>detect, chosen languages</from>
<to>validate chosen</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>texts</name>
<type>DataGrid</type>
<description>English, French and German text, only spaces, and no text at all (null).</description>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<data>
<line>
<item>1</item>
<item>The children are playing in the garden while their parents prepare dinner in the kitchen.</item>
</line>
<line>
<item>2</item>
<item>Les enfants jouent dans le jardin pendant que leurs parents préparent le dîner dans la cuisine.</item>
</line>
<line>
<item>3</item>
<item>Die Kinder spielen im Garten, während ihre Eltern in der Küche das Abendessen vorbereiten.</item>
</line>
<line>
<item>4</item>
<item> </item>
</line>
<line>
<item>5</item>
<item/>
</line>
</data>
<fields>
<field>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<name>id</name>
<type>String</type>
</field>
<field>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<name>text</name>
<type>String</type>
</field>
</fields>
<attributes/>
<GUI>
<xloc>112</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>detect, default options</name>
<type>DetectLanguage</type>
<description>Every spoken language, high accuracy, one thread.</description>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<corpusField>text</corpusField>
<parallelism>N</parallelism>
<lowAccuracy>N</lowAccuracy>
<languages>
</languages>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>96</yloc>
</GUI>
</transform>
<transform>
<name>detect, chosen languages</name>
<type>DetectLanguage</type>
<description>English, French and German only, low accuracy mode, parallel jobs: the rows can come out in another order.</description>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<corpusField>${DETECT_LANGUAGE_FIELD}</corpusField>
<parallelism>Y</parallelism>
<lowAccuracy>Y</lowAccuracy>
<languages>
<language>
<name>ENGLISH</name>
</language>
<language>
<name>FRENCH</name>
</language>
<language>
<name>${DETECT_LANGUAGE_THIRD}</name>
</language>
</languages>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>validate</name>
<type>Dummy</type>
<description></description>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>560</xloc>
<yloc>96</yloc>
</GUI>
</transform>
<transform>
<name>validate chosen</name>
<type>Dummy</type>
<description></description>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>560</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
</pipeline>
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
id,detected_language
1,ENGLISH
2,FRENCH
3,GERMAN
4,UNKNOWN
5,UNKNOWN
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
id,detected_language
1,ENGLISH
2,FRENCH
3,GERMAN
4,UNKNOWN
5,UNKNOWN
79 changes: 79 additions & 0 deletions integration-tests/transforms/main-0128-detect-language.hwf
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--

Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

-->
<workflow>
<name>main-0128-detect-language</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<workflow_version/>
<created_user>-</created_user>
<created_date>2026/11/16 12:00:00.000</created_date>
<modified_user>-</modified_user>
<modified_date>2026/11/16 12:00:00.000</modified_date>
<parameters>
</parameters>
<actions>
<action>
<name>Start</name>
<description/>
<type>SPECIAL</type>
<attributes/>
<DayOfMonth>1</DayOfMonth>
<hour>12</hour>
<intervalMinutes>60</intervalMinutes>
<intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
<repeat>N</repeat>
<schedulerType>0</schedulerType>
<weekDay>1</weekDay>
<parallel>N</parallel>
<xloc>112</xloc>
<yloc>64</yloc>
<attributes_hac/>
</action>
<action>
<name>0128-detect-language UNIT</name>
<description/>
<type>RunPipelineTests</type>
<attributes/>
<test_names>
<test_name>
<name>0128-detect-language UNIT</name>
</test_name>
</test_names>
<parallel>N</parallel>
<xloc>304</xloc>
<yloc>64</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>Start</from>
<to>0128-detect-language UNIT</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
</notepads>
<attributes/>
</workflow>
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"base_filename": "golden-0128-detect-language-chosen.csv",
"name": "golden-0128-detect-language-chosen",
"description": "",
"dataset_fields": [
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
"field_name": "id",
"field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
"field_name": "detected_language",
"field_format": ""
}
],
"folder_name": ""
}
Loading
Loading