Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include <stdio.h>
#include "gtest/gtest.h"
#include "mozilla/intl/LineBreaker.h"
#include "mozilla/intl/WordBreaker.h"
#include "mozilla/Preferences.h"
#include "mozilla/Span.h"
#include "nsISupports.h"
#include "nsServiceManagerUtils.h"
#include "nsString.h"
#include "nsTArray.h"
#include "nsXPCOM.h"
using mozilla::intl::LineBreaker;
using mozilla::intl::WordBreaker;
// Turn off clang-format to align the ruler comments to the test strings.
// clang-format off
static char teng0[] =
// 1 2 3 4 5 6 7
// 01234567890123456789012345678901234567890123456789012345678901234567890123456789
"hello world";
// clang-format on
static uint32_t lexp0[] = {5, 11};
static uint32_t wexp0[] = {5, 6, 11};
// clang-format off
static char teng1[] =
// 1 2 3 4 5 6 7
// 01234567890123456789012345678901234567890123456789012345678901234567890123456789
"This is a test to test(reasonable) line break. This 0.01123 = 45 x 48.";
// clang-format on
static uint32_t lexp1[] = {4, 7, 9, 14, 17, 34, 39, 40, 41,
42, 49, 54, 62, 64, 67, 69, 73};
static uint32_t wexp1[] = {4, 5, 7, 8, 9, 10, 14, 15, 17, 18, 22, 23,
33, 34, 35, 39, 43, 48, 49, 50, 54, 55, 56, 57,
62, 63, 64, 65, 67, 68, 69, 70, 72, 73};
// clang-format off
static char teng2[] =
// 1 2 3 4 5 6 7
// 01234567890123456789012345678901234567890123456789012345678901234567890123456789
"()((reasonab(l)e) line break. .01123=45x48.";
// clang-format on
static uint32_t lexp2[] = {17, 22, 23, 30, 44};
static uint32_t wexp2[] = {4, 12, 13, 14, 15, 16, 17, 18, 22,
24, 29, 30, 31, 32, 37, 38, 43, 44};
// clang-format off
static char teng3[] =
// 1 2 3 4 5 6 7
// 01234567890123456789012345678901234567890123456789012345678901234567890123456789
"It's a test to test(ronae ) line break....";
// clang-format on
static uint32_t lexp3[] = {4, 6, 11, 14, 25, 27, 32, 42};
static uint32_t wexp3[] = {2, 3, 4, 5, 6, 7, 11, 12, 14, 15,
19, 20, 25, 26, 27, 28, 32, 33, 38, 42};
static char ruler1[] =
" 1 2 3 4 5 6 7 ";
static char ruler2[] =
"0123456789012345678901234567890123456789012345678901234567890123456789012";
bool Check(const char* in, mozilla::Span<const uint32_t> out,
mozilla::Span<const uint32_t> res) {
const uint32_t outlen = out.Length();
const uint32_t i = res.Length();
bool ok = true;
if (i != outlen) {
ok = false;
printf("WARNING!!! return size wrong, expect %d but got %d \n", outlen, i);
}
for (uint32_t j = 0; j < i; j++) {
if (j < outlen) {
if (res[j] != out[j]) {
ok = false;
printf("[%d] expect %d but got %d\n", j, out[j], res[j]);
}
} else {
ok = false;
printf("[%d] additional %d\n", j, res[j]);
}
}
if (!ok) {
printf("string = \n%s\n", in);
printf("%s\n", ruler1);
printf("%s\n", ruler2);
printf("Expect = \n");
for (uint32_t j = 0; j < outlen; j++) {
printf("%d,", out[j]);
}
printf("\nResult = \n");
for (uint32_t j = 0; j < i; j++) {
printf("%d,", res[j]);
}
printf("\n");
}
return ok;
}
bool TestASCIILB(const char* in, mozilla::Span<const uint32_t> out) {
NS_ConvertASCIItoUTF16 input(in);
EXPECT_GT(input.Length(), 0u) << "Expect a non-empty input!";
nsTArray<uint32_t> result;
int32_t curr = 0;
while (true) {
curr = LineBreaker::Next(input.get(), input.Length(), curr);
if (curr == NS_LINEBREAKER_NEED_MORE_TEXT) {
break;
}
result.AppendElement(curr);
}
return Check(in, out, result);
}
bool TestASCIIWB(const char* in, mozilla::Span<const uint32_t> out) {
NS_ConvertASCIItoUTF16 input(in);
EXPECT_GT(input.Length(), 0u) << "Expect a non-empty input!";
nsTArray<uint32_t> result;
int32_t curr = 0;
while (true) {
curr = WordBreaker::Next(input.get(), input.Length(), curr);
if (curr == NS_WORDBREAKER_NEED_MORE_TEXT) {
break;
}
result.AppendElement(curr);
}
return Check(in, out, result);
}
TEST(LineBreak, LineBreaker)
{
ASSERT_TRUE(TestASCIILB(teng0, lexp0));
ASSERT_TRUE(TestASCIILB(teng1, lexp1));
ASSERT_TRUE(TestASCIILB(teng2, lexp2));
ASSERT_TRUE(TestASCIILB(teng3, lexp3));
}
TEST(WordBreak, WordBreaker)
{
ASSERT_TRUE(TestASCIIWB(teng0, wexp0));
ASSERT_TRUE(TestASCIIWB(teng1, wexp1));
ASSERT_TRUE(TestASCIIWB(teng2, wexp2));
ASSERT_TRUE(TestASCIIWB(teng3, wexp3));
}
// 012345678901234
static const char wb0[] = "T";
static const char wb1[] = "h";
static const char wb2[] = "";
static const char wb3[] = "is is a int";
static const char wb4[] = "";
static const char wb5[] = "";
static const char wb6[] = "ernationali";
static const char wb7[] = "zation work.";
static const char* wb[] = {wb0, wb1, wb2, wb3, wb4, wb5, wb6, wb7};
TEST(WordBreak, TestPrintWordWithBreak)
{
uint32_t numOfFragment = sizeof(wb) / sizeof(char*);
// This test generate the result string by appending '^' at every word break
// opportunity except the one at end of the text.
nsAutoString result;
for (uint32_t i = 0; i < numOfFragment; i++) {
NS_ConvertASCIItoUTF16 fragText(wb[i]);
int32_t cur = 0;
cur = WordBreaker::Next(fragText.get(), fragText.Length(), cur);
uint32_t start = 0;
while (cur != NS_WORDBREAKER_NEED_MORE_TEXT) {
result.Append(Substring(fragText, start, cur - start));
// Append '^' only if cur is within the fragText. We'll check the word
// break opportunity between fragText and nextFragText using
// BreakInBetween() below.
if (cur < static_cast<int32_t>(fragText.Length())) {
result.Append('^');
}
start = (cur >= 0 ? cur : cur - start);
cur = WordBreaker::Next(fragText.get(), fragText.Length(), cur);
}
if (i != numOfFragment - 1) {
NS_ConvertASCIItoUTF16 nextFragText(wb[i + 1]);
if (nextFragText.IsEmpty()) {
// If nextFragText is empty, there's no new possible word break
// opportunity.
continue;
}
const auto origFragLen = static_cast<int32_t>(fragText.Length());
fragText.Append(nextFragText);
bool canBreak =
origFragLen ==
WordBreaker::Next(fragText.get(), fragText.Length(), origFragLen - 1);
if (canBreak) {
result.Append('^');
}
}
}
ASSERT_STREQ("This^ ^is^ ^a^ ^internationalization^ ^work^.",
NS_ConvertUTF16toUTF8(result).get());
}
// This function searches a complete word starting from |offset| in wb[fragN].
// If it reaches the end of wb[fragN], and there is no word break opportunity
// between wb[fragN] and wb[fragN+1], it will continue the search in wb[fragN+1]
// until a word break.
void TestFindWordBreakFromPosition(uint32_t fragN, uint32_t offset,
const char* expected) {
uint32_t numOfFragment = sizeof(wb) / sizeof(char*);
NS_ConvertASCIItoUTF16 fragText(wb[fragN]);
mozilla::intl::WordRange res = WordBreaker::FindWord(fragText, offset);
nsAutoString result(Substring(fragText, res.mBegin, res.mEnd - res.mBegin));
if ((uint32_t)fragText.Length() <= res.mEnd) {
// if we hit the end of the fragment
nsAutoString curFragText = fragText;
for (uint32_t p = fragN + 1; p < numOfFragment; p++) {
NS_ConvertASCIItoUTF16 nextFragText(wb[p]);
if (nextFragText.IsEmpty()) {
// If nextFragText is empty, there's no new possible word break
// opportunity between curFragText and nextFragText.
continue;
}
const auto origFragLen = static_cast<int32_t>(curFragText.Length());
curFragText.Append(nextFragText);
bool canBreak = origFragLen == WordBreaker::Next(curFragText.get(),
curFragText.Length(),
origFragLen - 1);
if (canBreak) {
break;
}
mozilla::intl::WordRange r = WordBreaker::FindWord(nextFragText, 0);
result.Append(Substring(nextFragText, r.mBegin, r.mEnd - r.mBegin));
if ((uint32_t)nextFragText.Length() != r.mEnd) {
break;
}
}
}
ASSERT_STREQ(expected, NS_ConvertUTF16toUTF8(result).get())
<< "FindWordBreakFromPosition(" << fragN << ", " << offset << ")";
}
TEST(WordBreak, TestNextWordBreakWithComplexLanguage)
{
nsString fragText(u"\u0e40\u0e1b\u0e47\u0e19\u0e19\u0e31\u0e01");
int32_t offset = 0;
while (offset != NS_WORDBREAKER_NEED_MORE_TEXT) {
int32_t newOffset =
WordBreaker::Next(fragText.get(), fragText.Length(), offset);
ASSERT_NE(offset, newOffset);
offset = newOffset;
}
ASSERT_TRUE(true);
}
TEST(WordBreak, TestFindWordWithEmptyString)
{
mozilla::intl::WordRange expect{0, 0};
mozilla::intl::WordRange result = WordBreaker::FindWord(EmptyString(), 0);
ASSERT_EQ(expect.mBegin, result.mBegin);
ASSERT_EQ(expect.mEnd, result.mEnd);
}
TEST(WordBreak, TestNextWordBreakWithEmptyString)
{
char16_t empty[] = {};
ASSERT_EQ(NS_WORDBREAKER_NEED_MORE_TEXT, WordBreaker::Next(empty, 0, 0));
ASSERT_EQ(NS_WORDBREAKER_NEED_MORE_TEXT, WordBreaker::Next(empty, 0, 1));
}
TEST(WordBreak, TestFindWordBreakFromPosition)
{
TestFindWordBreakFromPosition(0, 0, "This");
TestFindWordBreakFromPosition(1, 0, "his");
TestFindWordBreakFromPosition(2, 0, "is");
TestFindWordBreakFromPosition(3, 0, "is");
TestFindWordBreakFromPosition(3, 1, "is");
TestFindWordBreakFromPosition(3, 9, " ");
TestFindWordBreakFromPosition(3, 10, "internationalization");
TestFindWordBreakFromPosition(4, 0, "ernationalization");
TestFindWordBreakFromPosition(5, 0, "ernationalization");
TestFindWordBreakFromPosition(6, 4, "ernationalization");
TestFindWordBreakFromPosition(6, 8, "ernationalization");
TestFindWordBreakFromPosition(7, 6, " ");
TestFindWordBreakFromPosition(7, 7, "work");
}
// Test for StopAtPunctuation option.
TEST(WordBreak, TestFindBreakWithStopAtPunctuation)
{
bool original =
mozilla::Preferences::GetBool("intl.icu4x.segmenter.enabled", true);
// Not UAX#29 rule
mozilla::Preferences::SetBool("intl.icu4x.segmenter.enabled", false);
nsString fragText(u"one.two");
mozilla::intl::WordRange result1 = WordBreaker::FindWord(fragText, 0);
ASSERT_EQ(0u, result1.mBegin);
ASSERT_EQ(3u, result1.mEnd);
mozilla::intl::WordRange result2 = WordBreaker::FindWord(fragText, 3);
ASSERT_EQ(3u, result2.mBegin);
ASSERT_EQ(4u, result2.mEnd);
mozilla::intl::WordRange result3 = WordBreaker::FindWord(fragText, 4);
ASSERT_EQ(4u, result3.mBegin);
ASSERT_EQ(7u, result3.mEnd);
// UAX#29 rule
mozilla::Preferences::SetBool("intl.icu4x.segmenter.enabled", true);
mozilla::intl::WordRange result4 = WordBreaker::FindWord(
fragText, 0, WordBreaker::FindWordOptions::StopAtPunctuation);
ASSERT_EQ(0u, result4.mBegin);
ASSERT_EQ(3u, result4.mEnd);
mozilla::intl::WordRange result5 = WordBreaker::FindWord(
fragText, 3, WordBreaker::FindWordOptions::StopAtPunctuation);
ASSERT_EQ(3u, result5.mBegin);
ASSERT_EQ(4u, result5.mEnd);
mozilla::intl::WordRange result6 = WordBreaker::FindWord(
fragText, 4, WordBreaker::FindWordOptions::StopAtPunctuation);
ASSERT_EQ(4u, result6.mBegin);
ASSERT_EQ(7u, result6.mEnd);
// Default (without StopAtPunctuation)
mozilla::intl::WordRange result7 = WordBreaker::FindWord(fragText, 0);
ASSERT_EQ(0u, result7.mBegin);
ASSERT_EQ(7u, result7.mEnd);
mozilla::intl::WordRange result8 = WordBreaker::FindWord(fragText, 3);
ASSERT_EQ(0u, result8.mBegin);
ASSERT_EQ(7u, result8.mEnd);
mozilla::intl::WordRange result9 = WordBreaker::FindWord(fragText, 4);
ASSERT_EQ(0u, result9.mBegin);
ASSERT_EQ(7u, result9.mEnd);
mozilla::Preferences::SetBool("intl.icu4x.segmenter.enabled", original);
}