feat: EditorUtils use document's locale to splitting words

build.gradle

@@ -300,6 +300,7 @@ dependencies {
         }
         runtimeOnly(libs.language.detector)
         runtimeOnly(libs.dumont.hunspell)
+        implementation(libs.icu4j)
 
         // Lucene for tokenizers
         implementation(libs.bundles.lucene)
@@ -1701,7 +1702,7 @@ tasks.register('testAcceptance', Test) {
     classpath = sourceSets.testAcceptance.runtimeClasspath
     systemProperties = System.properties
     systemProperty 'java.util.logging.config.file', "${rootDir}/config/test/logger.properties"
-
+    dependsOn firstStepsEn
     dependsOn ':aligner:jar'
 }
 

gradle/libs.versions.toml

@@ -65,7 +65,7 @@ commons-lang3 = {group = "org.apache.commons", name = "commons-lang3", version.r
 commons-text = {group = "org.apache.commons", name = "commons-text", version.ref = "commons_text"}
 commons-validator = {group = "commons-validator", name = "commons-validator", version.ref = "commons_validator"}
 jsoup = {group = "org.jsoup", name = "jsoup", version.ref = "jsoup"}
-icj4j = {group = "com.ibm.icu", name = "icu4j", version.ref = "icu4j"}
+icu4j = {group = "com.ibm.icu", name = "icu4j", version.ref = "icu4j"}
 stax2-api = {group = "org.codehaus.woodstox", name = "stax2-api", version.ref = "stax2api"}
 woodstox-core = {group = "com.fasterxml.woodstox", name = "woodstox-core", version.ref = "woodstox"}
 languagetool-all = {group = "org.languagetool", name = "language-all", version.ref = "languagetool"}

language-modules/ja/build.gradle

@@ -25,7 +25,7 @@ dependencies {
             exclude module: 'icu4j'
         }
         implementation(dependencies.variantOf(libs.lucene.gosen) { classifier("ipadic") })
-        implementation(libs.icj4j)
+        compileOnly(libs.icu4j)
     }
 
     testImplementation(libs.junit4)
@@ -43,7 +43,7 @@ dependencies {
         exclude module: 'icu4j'
     }
     testRuntimeOnly(dependencies.variantOf(libs.lucene.gosen) { classifier("ipadic") })
-    testRuntimeOnly(libs.icj4j)
+    testRuntimeOnly(libs.icu4j)
 
     testImplementation(libs.assertj)
     testImplementation(testFixtures(project.rootProject))

src/org/omegat/gui/editor/EditorController.java

@@ -1947,6 +1947,7 @@ private void createAdditionalPanes() {
                     .setComponentOrientation(BiDiUtils.isRtl(language) ? ComponentOrientation.RIGHT_TO_LEFT
                             : ComponentOrientation.LEFT_TO_RIGHT);
             introPane.setEditable(false);
+            introPane.setName("IntroPane");
             DragTargetOverlay.apply(introPane, dropInfo);
             URI uri = Help.getHelpFileURI(OConsts.HELP_FIRST_STEPS_PREFIX, language, OConsts.HELP_FIRST_STEPS);
             if (uri != null) {
@@ -1958,6 +1959,7 @@ private void createAdditionalPanes() {
         emptyProjectPaneTitle = OStrings.getString("TF_INTRO_EMPTYPROJECT_FILENAME");
         emptyProjectPane = new JTextPane();
         emptyProjectPane.setEditable(false);
+        emptyProjectPane.setName("EmptyProjectPane");
         emptyProjectPane.setText(OStrings.getString("TF_INTRO_EMPTYPROJECT"));
         emptyProjectPane.setFont(mw.getApplicationFont());
         DragTargetOverlay.apply(emptyProjectPane, dropInfo);

src/org/omegat/gui/editor/EditorUtils.java

@@ -31,9 +31,13 @@
 import java.util.Locale;
 
 import javax.swing.text.BadLocationException;
+import javax.swing.text.Document;
+import javax.swing.text.Element;
 import javax.swing.text.JTextComponent;
 import javax.swing.text.Utilities;
 
+import com.ibm.icu.text.BreakIterator;
+
 import org.omegat.core.Core;
 import org.omegat.core.data.ProtectedPart;
 import org.omegat.core.data.SourceTextEntry;
@@ -61,15 +65,14 @@ private EditorUtils() {
      * Determines the start of a word for the given model location. This method
      * skips direction char.
      *
-     * TODO: change to use document's locale
-     *
-     * @param c
-     * @param offs
-     * @return
+     * @param c TextComponent of the editor area.
+     * @param offs offset of the text.
+     * @return position of word start on the text component.
      * @throws BadLocationException
+     *         when there is no line found in the text component.
      */
     public static int getWordStart(JTextComponent c, int offs) throws BadLocationException {
-        int result = Utilities.getWordStart(c, offs);
+        int result = getWordBoundary(c, offs, false);
         char ch = c.getDocument().getText(result, 1).charAt(0);
         if (isDirectionChar(ch)) {
             result++;
@@ -81,15 +84,14 @@ public static int getWordStart(JTextComponent c, int offs) throws BadLocationExc
      * Determines the end of a word for the given model location. This method
      * skips direction char.
      *
-     * TODO: change to use document's locale
-     *
-     * @param c
-     * @param offs
-     * @return
+     * @param c TextComponent of the editor area.
+     * @param offs offset of the text.
+     * @return position of the word end on the text component.
      * @throws BadLocationException
+     *         when there is no line found in the text component.
      */
     public static int getWordEnd(JTextComponent c, int offs) throws BadLocationException {
-        int result = Utilities.getWordEnd(c, offs);
+        int result = getWordBoundary(c, offs, true);
         if (result > 0) {
             char ch = c.getDocument().getText(result - 1, 1).charAt(0);
             if (isDirectionChar(ch)) {
@@ -99,6 +101,47 @@ public static int getWordEnd(JTextComponent c, int offs) throws BadLocationExcep
         return result;
     }
 
+    private static int getWordBoundary(JTextComponent c, int offs, boolean end) throws BadLocationException {
+        int result = offs;
+        Element line = Utilities.getParagraphElement(c, offs);
+        if (line == null) {
+            throw new BadLocationException("No word at " + offs, offs);
+        }
+        int lineStart = line.getStartOffset();
+        Document doc = c.getDocument();
+        int lineEnd = Math.min(line.getEndOffset(), doc.getLength());
+        if  (lineEnd - lineStart > 0) {
+            String lineString = doc.getText(lineStart, lineEnd - lineStart);
+            // Detection of target string locale.
+            // A default is UI component locale, and when OmegaT project
+            // is loaded, it uses a source or a target language as a
+            // processing locale.
+            Locale locale = c.getLocale();
+            if (c instanceof EditorTextArea3 && Core.getProject().isProjectLoaded()) {
+                if (((EditorTextArea3) c).isInActiveTranslation(offs)) {
+                    locale = Core.getProject().getProjectProperties().getTargetLanguage().getLocale();
+                } else {
+                    locale = Core.getProject().getProjectProperties().getSourceLanguage().getLocale();
+                }
+            }
+            result = lineStart + getWordBoundary(locale, lineString, offs - lineStart, end);
+        }
+        return result;
+    }
+
+    static int getWordBoundary(Locale locale, String lineString, int wordPosition, boolean end) {
+        BreakIterator words = com.ibm.icu.text.BreakIterator.getWordInstance(locale);
+        words.setText(lineString);
+        if (wordPosition >= words.last()) {
+            wordPosition = words.last() - 1;
+        }
+        if (end) {
+            return words.following(wordPosition);
+        }
+        words.following(wordPosition);
+        return words.previous();
+}
+
     /**
      * Check if char is direction char(u202A,u202B,u202C).
      *

test-acceptance/data/project_CN_JP/.gitignore

@@ -0,0 +1,3 @@
+project_stats.txt
+project_stats.json
+*.bak

test-acceptance/data/project_CN_JP/glossary/glossary.txt

@@ -0,0 +1,3 @@
+# Glossary in tab-separated format -*- coding: utf-8 -*-
+介绍      紹介
+中的      中心的な

test-acceptance/data/project_CN_JP/omegat.project

@@ -0,0 +1,33 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<omegat>
+  <project version="1.0">
+    <source_dir>source</source_dir>
+    <source_dir_excludes>
+      <mask>**/.svn/**</mask>
+      <mask>**/CVS/**</mask>
+      <mask>**/.cvs/**</mask>
+      <mask>**/.git/**</mask>
+      <mask>**/.hg/**</mask>
+      <mask>**/.repositories/**</mask>
+      <mask>**/desktop.ini</mask>
+      <mask>**/Thumbs.db</mask>
+      <mask>**/.DS_Store</mask>
+      <mask>**/~$*</mask>
+    </source_dir_excludes>
+    <target_dir>target</target_dir>
+    <tm_dir>tm</tm_dir>
+    <glossary_dir>glossary</glossary_dir>
+    <glossary_file>.-glossary.txt</glossary_file>
+    <dictionary_dir>dictionary</dictionary_dir>
+    <export_tm_dir></export_tm_dir>
+    <export_tm_levels></export_tm_levels>
+    <source_lang>zh-CN</source_lang>
+    <target_lang>ja-JP</target_lang>
+    <source_tok>org.omegat.tokenizer.LuceneSmartChineseTokenizer</source_tok>
+    <target_tok>org.omegat.tokenizer.LuceneJapaneseTokenizer</target_tok>
+    <sentence_seg>true</sentence_seg>
+    <support_default_translations>true</support_default_translations>
+    <remove_tags>true</remove_tags>
+    <external_command></external_command>
+  </project>
+</omegat>

test-acceptance/data/project_CN_JP/omegat/last_entry.properties

@@ -0,0 +1,4 @@
+#Thu Nov 07 21:30:29 JST 2024
+LAST_ENTRY_NUMBER=1
+LAST_ENTRY_SRC=\u592A\u5E73\u5BFA\u4E2D\u7684\u6587\u7B14\u5854
+LAST_ENTRY_FILE=source.txt

test-acceptance/data/project_CN_JP/omegat/project_save.tmx

@@ -0,0 +1,17 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!DOCTYPE tmx SYSTEM "tmx11.dtd">
+<tmx version="1.1">
+  <header creationtool="OmegaT" o-tmf="OmegaT TMX" adminlang="EN-US" datatype="plaintext" creationtoolversion="6.1.0_0_50ff299ad" segtype="sentence" srclang="zh-CN"/>
+  <body>
+<!-- Default translations -->
+    <tu>
+      <tuv lang="zh-CN">
+        <seg>太平寺中的文笔塔</seg>
+      </tuv>
+      <tuv lang="ja-JP" changeid="Hiroshi Miura" changedate="20241107T122621Z" creationid="Hiroshi Miura" creationdate="20241107T122621Z">
+        <seg>太平寺の中心的なペン塔</seg>
+      </tuv>
+    </tu>
+<!-- Alternative translations -->
+  </body>
+</tmx>

test-acceptance/data/project_CN_JP/source/source.txt

@@ -0,0 +1,5 @@
+太平寺中的文笔塔
+
+文筆塔原是江苏省常州市太平寺中的塔。太平寺始建于南北朝齐梁时期，是常州最古老的佛寺之一，今已不存。
+文笔塔为砖木结构，七级八面，每级4个拱门，中有旋梯。塔下有曲池、拱桥。
+“夕照塔影”为文笔胜景。现存塔为光绪末年（1905-1908年）重建

test-acceptance/src/org/omegat/gui/editor/EditorUtilsTest.java

@@ -0,0 +1,123 @@
+/**************************************************************************
+ OmegaT - Computer Assisted Translation (CAT) tool
+          with fuzzy matching, translation memory, keyword search,
+          glossaries, and translation leveraging into updated projects.
+
+ Copyright (C) 2024 Hiroshi Miura
+               Home page: https://www.omegat.org/
+               Support center: https://omegat.org/support
+
+ This file is part of OmegaT.
+
+ OmegaT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OmegaT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ **************************************************************************/
+
+package org.omegat.gui.editor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.Locale;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import javax.swing.SwingUtilities;
+import javax.swing.text.BadLocationException;
+import javax.swing.text.JTextComponent;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+
+import org.omegat.core.Core;
+import org.omegat.core.CoreEvents;
+import org.omegat.gui.main.ProjectUICommands;
+import org.omegat.gui.main.TestCoreGUI;
+import org.omegat.util.LocaleRule;
+import org.omegat.util.Preferences;
+
+/**
+ * @author Hiroshi Miura
+ */
+@RunWith(Enclosed.class)
+public class EditorUtilsTest {
+
+    public static class EditorUtilsFirstStepsTest extends TestCoreGUI {
+
+        @Rule
+        public final LocaleRule localeRule = new LocaleRule(new Locale("en"));
+
+        @Test
+        public void testEditorUtilsGetWordFirstSteps() throws BadLocationException {
+            int offs = 518;
+            JTextComponent editPane = window.panel("First Steps").textBox("IntroPane").target();
+            int posStart = EditorUtils.getWordStart(editPane, offs);
+            int posEnd = EditorUtils.getWordEnd(editPane, offs);
+            String word = editPane.getText(posStart, posEnd - posStart);
+            assertEquals("translation", word);
+            assertEquals(508, posStart);
+            assertEquals(519, posEnd);
+        }
+    }
+
+    public static class EditorUtilsLoadedProjectTest extends TestCoreGUI {
+
+        @Rule
+        public final LocaleRule localeRule = new LocaleRule(new Locale("en"));
+
+        @Rule
+        public final TemporaryFolder folder = TemporaryFolder.builder().assureDeletion().build();
+
+        @Test
+        public void testEditorUtilsGetWordLoadedProject() throws Exception {
+            // Prepare a sample project
+            File tmpDir = folder.newFolder("omegat-sample-project-");
+            File projSrc = new File("test-acceptance/data/project_CN_JP/");
+            FileUtils.copyDirectory(projSrc, tmpDir);
+            FileUtils.forceDeleteOnExit(tmpDir);
+            Preferences.setPreference(Preferences.PROJECT_FILES_SHOW_ON_LOAD, false);
+            // Load the project and wait a completion
+            CountDownLatch latch = new CountDownLatch(1);
+            CoreEvents.registerProjectChangeListener(eventType -> {
+                if (Core.getProject().isProjectLoaded()) {
+                    latch.countDown();
+                }
+            });
+            SwingUtilities.invokeAndWait(() -> ProjectUICommands.projectOpen(tmpDir, true));
+            try {
+                assertTrue(latch.await(5, TimeUnit.SECONDS));
+            } catch (InterruptedException ignored) {
+            }
+            //
+            final JTextComponent editPane = window.panel("Editor - source.txt").textBox().target();
+            // select word from a source text
+            int offs = 102;
+            int posStart = EditorUtils.getWordStart(editPane, offs);
+            int posEnd = EditorUtils.getWordEnd(editPane, offs);
+            String word = editPane.getText(posStart, posEnd - posStart);
+            assertEquals("太平寺", word);
+            // select word from a translation
+            offs = 109;
+            posStart = EditorUtils.getWordStart(editPane, offs);
+            posEnd = EditorUtils.getWordEnd(editPane, offs);
+            word = editPane.getText(posStart, posEnd - posStart);
+            assertEquals("太平寺", word);
+        }
+    }
+
+}