Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: introduce matchSource match template variable #1220

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
4 changes: 2 additions & 2 deletions config/checkstyle/suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@
lines="90,91,92,96,100,104,105,107,264,275,276,284,286,293,294,308,309,359,361,449,495,605,674,710"/>

<!-- matching -->
<suppress files="NearString\.java" checks="TypeName" lines="51-55"/>
<suppress files="NearString\.java" checks="ParameterNumber" lines="85-150"/>
<suppress files="NearString\.java" checks="TypeName" lines="50-70"/>
<suppress files="NearString\.java" checks="ParameterNumber"/>
<suppress files="LevenshteinDistance\.java" checks="LocalVariableName" lines="147,159"/>

<!-- Aligner, AlignFilePickerController, AlignPanelController -->
Expand Down
5 changes: 5 additions & 0 deletions src/org/omegat/Bundle.properties
miurahr marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -2947,3 +2947,8 @@ DICTIONARY_LOAD_FILE=Loaded dictionary from '{0}': {1} ms
DICTIONARY_LOAD_ERROR=Error load dictionary from '{0}': {1}
DICTIONARY_MANAGER_ERROR_SAVE_IGNORE=Error saving ignore words"
EDITOR_CONTROLLER_EXCEPTION=bad location exception when changing case

MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_TM=External TM
MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_FILES=Source Files
MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_MEMORY=Project
MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_TM_SUBSEG=Sub-segmented match
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_TM_SUBSEG=Sub-segmented match
MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_SUBSEGMENTS=Sub-segmented match

107 changes: 80 additions & 27 deletions src/org/omegat/core/matching/NearString.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,37 +49,81 @@
*/
public class NearString {
public enum MATCH_SOURCE {
MEMORY, TM, FILES
};
/** From project memory */
MEMORY,
/** From external TM in project tm/ folder */
TM,
/** From files */
miurahr marked this conversation as resolved.
Show resolved Hide resolved
miurahr marked this conversation as resolved.
Show resolved Hide resolved
FILES,
/** From sub-segmented match */
TM_SUBSEG
miurahr marked this conversation as resolved.
Show resolved Hide resolved
miurahr marked this conversation as resolved.
Show resolved Hide resolved
}

public enum SORT_KEY {
SCORE, SCORE_NO_STEM, ADJUSTED_SCORE
/** normal score */
SCORE,
/** score without stemming */
SCORE_NO_STEM,
/** adjusted score */
ADJUSTED_SCORE
}

public NearString(EntryKey key, ITMXEntry entry, MATCH_SOURCE comesFrom, boolean fuzzyMark,
Scores scores, byte[] nearData, String projName) {
this(key, entry.getSourceText(), entry.getTranslationText(), comesFrom, fuzzyMark, scores,
nearData, projName, entry.getCreator(), entry.getCreationDate(), entry.getChanger(),
/**
* Constructor.
*
* @param key
* entry key
* @param entry
* the TMX entry that has source text, translation text, creator,
* creation date, changer, change date and properties.
* @param comesFrom
* origin
* @param fuzzyMark
* fuzzy or not
* @param scores
* score values
* @param nearData
* similarity data.
* @param projName
* project name.
*/
public NearString(EntryKey key, ITMXEntry entry, MATCH_SOURCE comesFrom, boolean fuzzyMark, Scores scores,
byte[] nearData, String projName) {
this(key, entry.getSourceText(), entry.getTranslationText(), comesFrom, fuzzyMark, scores, nearData,
projName, entry.getCreator(), entry.getCreationDate(), entry.getChanger(),
entry.getChangeDate(), entry.getProperties());
}

/**
* Constructor, backward compatible.
* @param key entry key
* @param source source text
* @param translation translation text
* @param comesFrom origin
* @param fuzzyMark fuzzy or not
*
* @param key
* entry key
* @param source
* source text
* @param translation
* translation text
* @param comesFrom
* origin
* @param fuzzyMark
* fuzzy or not
* @param nearScore
* @param nearScoreNoStem
* @param adjustedScore
* @param nearData similarity data.
* @param projName project name.
* @param creator creator name
* @param creationDate creation date
* @param changer changer name
* @param changedDate changer date
* @param props properties of entry.
* @param nearData
* similarity data.
* @param projName
* project name.
* @param creator
* creator name
* @param creationDate
* creation date
* @param changer
* changer name
* @param changedDate
* changer date
* @param props
* properties of entry.
*/
@Deprecated
public NearString(EntryKey key, String source, String translation, MATCH_SOURCE comesFrom,
Expand Down Expand Up @@ -110,14 +154,23 @@ private NearString(EntryKey key, String source, String translation, MATCH_SOURCE

/**
* Merge NearString object.
* @param ns NearString to merge.
* @param key entry key.
* @param entry TMXEntry entry
* @param comesFrom origin
* @param fuzzyMark fuzzy or not
* @param scores similarity score
* @param nearData similarity data
* @param projName project name
*
* @param ns
* NearString to merge.
* @param key
* entry key.
* @param entry
* TMXEntry entry
* @param comesFrom
* origin
* @param fuzzyMark
* fuzzy or not
* @param scores
* similarity score
* @param nearData
* similarity data
* @param projName
* project name
* @return NearString merged.
*/
public static NearString merge(NearString ns, EntryKey key, ITMXEntry entry, MATCH_SOURCE comesFrom,
Expand Down
5 changes: 3 additions & 2 deletions src/org/omegat/core/statistics/FindMatches.java
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,8 @@ public List<NearString> search(String searchText, boolean requiresTranslation, b
for (ITMXEntry tmen : en.getValue().getEntries()) {
checkStopped(stop);
if (tmen.getSourceText() == null) {
// Not all TMX entries have a source; in that case there can
// be no meaningful match, so skip.
// Not all TMX entries have a source; skip it in
// the case, because of no meaningful.
continue;
}
if (requiresTranslation && tmen.getTranslationText() == null) {
Expand All @@ -279,6 +279,7 @@ public List<NearString> search(String searchText, boolean requiresTranslation, b
processEntry(null, tmen, en.getKey(), NearString.MATCH_SOURCE.TM, false, tmenPenalty);
}
}

// travel by all entries for check source file translations
for (SourceTextEntry ste : project.getAllEntries()) {
checkStopped(stop);
Expand Down
25 changes: 23 additions & 2 deletions src/org/omegat/gui/matches/MatchesVarExpansion.java
Original file line number Diff line number Diff line change
Expand Up @@ -90,19 +90,21 @@ public class MatchesVarExpansion extends VarExpansion<NearString> {
public static final String VAR_DIFF_REVERSED = "${diffReversed}";
public static final String VAR_SOURCE_LANGUAGE = "${sourceLanguage}";
public static final String VAR_TARGET_LANGUAGE = "${targetLanguage}";
public static final String VAR_MATCH_SOURCE = "${matchSource}";

private static final String[] MATCHES_VARIABLES = { VAR_ID, VAR_SOURCE_TEXT, VAR_DIFF, VAR_DIFF_REVERSED,
VAR_TARGET_TEXT, VAR_SCORE_BASE, VAR_SCORE_NOSTEM, VAR_SCORE_ADJUSTED, VAR_FILE_NAME_ONLY,
VAR_FILE_PATH, VAR_FILE_SHORT_PATH, VAR_INITIAL_CREATION_ID, VAR_INITIAL_CREATION_DATE,
VAR_CHANGED_ID, VAR_CHANGED_DATE, VAR_FUZZY_FLAG, VAR_SOURCE_LANGUAGE, VAR_TARGET_LANGUAGE };
VAR_CHANGED_ID, VAR_CHANGED_DATE, VAR_FUZZY_FLAG, VAR_SOURCE_LANGUAGE, VAR_TARGET_LANGUAGE,
VAR_MATCH_SOURCE };

public static List<String> getMatchesVariables() {
return Collections.unmodifiableList(Arrays.asList(MATCHES_VARIABLES));
}

public static final String DEFAULT_TEMPLATE = VAR_ID + ". " + VAR_FUZZY_FLAG + VAR_SOURCE_TEXT + "\n"
+ VAR_TARGET_TEXT + "\n" + "<" + VAR_SCORE_BASE + "/" + VAR_SCORE_NOSTEM + "/"
+ VAR_SCORE_ADJUSTED + "% " + VAR_FILE_PATH + ">";
+ VAR_SCORE_ADJUSTED + "%" + VAR_MATCH_SOURCE + VAR_FILE_PATH + ">";

public static final Pattern PATTERN_SINGLE_PROPERTY = Pattern.compile("@\\{(.+?)\\}");
public static final Pattern PATTERN_PROPERTY_GROUP = Pattern
Expand Down Expand Up @@ -222,6 +224,22 @@ private String getPropValue(List<TMXProp> props, String type) {
return null;
}

private String expandMatchSource(String localTemplate, NearString.MATCH_SOURCE comesFrom) {
switch (comesFrom) {
case TM:
return localTemplate.replace(VAR_MATCH_SOURCE, OStrings.getString("MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_TM") + " ");
case FILES:
return localTemplate.replace(VAR_MATCH_SOURCE, OStrings.getString("MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_FILES"));
case MEMORY:
return localTemplate.replace(VAR_MATCH_SOURCE, OStrings.getString("MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_MEMORY"));
case TM_SUBSEG:
miurahr marked this conversation as resolved.
Show resolved Hide resolved
return localTemplate.replace(VAR_MATCH_SOURCE, OStrings.getString(
"MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_TM_SUBSEG") + " ");
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_TM_SUBSEG") + " ");
"MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_SUBSEGMENTS") + " ");

default:
return localTemplate.replace(VAR_MATCH_SOURCE, "");
}
}

@Override
public String expandVariables(NearString match) {
// do not modify template directly, so that we can reuse for another
Expand Down Expand Up @@ -285,6 +303,9 @@ public String expandVariables(NearString match) {
} else {
localTemplate = localTemplate.replace(VAR_TARGET_TEXT, match.translation);
}

localTemplate = expandMatchSource(localTemplate, match.comesFrom);

return localTemplate;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ public void testFuzzyMatches() throws Exception {
window.textBox("matches_pane").requireNotEditable();
Pattern pattern = Pattern.compile("1. Error while reading MT results\\n"
+ "Erreur lors de la lecture des résultats de TA\\n"
+ "<\\d+/\\d+/\\d+%\\s*>");
+ "<\\d+/\\d+/\\d+%\\s*" + OStrings.getString(
"MATCHES_VAR_EXPANSION_MATCH_COMES_FROM_MEMORY") + "\\s*>");
window.textBox("matches_pane").requireText(pattern);
closeProject();
}
Expand Down
3 changes: 2 additions & 1 deletion test/src/org/omegat/gui/matches/MatchesVarExpansionTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.omegat.core.data.ProjectProperties;
import org.omegat.core.data.SourceTextEntry;
import org.omegat.core.matching.NearString;
import org.omegat.core.matching.NearString.MATCH_SOURCE;
import org.omegat.gui.editor.IEditor;
import org.omegat.gui.editor.IEditorFilter;
import org.omegat.gui.editor.IEditorSettings;
Expand Down Expand Up @@ -208,7 +209,7 @@ public NearString getMockNearString() {
entry.changeDate = 20020523;
entry.otherProperties = testProps;
NearString.Scores scores = new NearString.Scores(20, 40, 60);
return new NearString(null, entry, null, false, scores, null, "mock testing project");
return new NearString(null, entry, MATCH_SOURCE.TM, false, scores, null, "mock testing project");
};

private void setupProject(Language sourceLanguage, Language targetLanguage) {
Expand Down
Loading