Source code
Revision control
Copy as Markdown
Other Tools
use alloc::{
string::{String, ToString},
vec::Vec,
};
use crate::hir;
/// An inclusive range of codepoints from a generated file (hence the static
/// lifetime).
type Range = &'static [(char, char)];
/// An error that occurs when dealing with Unicode.
///
/// We don't impl the Error trait here because these always get converted
/// into other public errors. (This error type isn't exported.)
#[derive(Debug)]
pub enum Error {
PropertyNotFound,
PropertyValueNotFound,
// Not used when unicode-perl is enabled.
#[allow(dead_code)]
PerlClassNotFound,
}
/// An error that occurs when Unicode-aware simple case folding fails.
///
/// This error can occur when the case mapping tables necessary for Unicode
/// aware case folding are unavailable. This only occurs when the
/// `unicode-case` feature is disabled. (The feature is enabled by default.)
#[derive(Debug)]
pub struct CaseFoldError(());
#[cfg(feature = "std")]
impl std::error::Error for CaseFoldError {}
impl core::fmt::Display for CaseFoldError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
f,
"Unicode-aware case folding is not available \
(probably because the unicode-case feature is not enabled)"
)
}
}
/// An error that occurs when the Unicode-aware `\w` class is unavailable.
///
/// This error can occur when the data tables necessary for the Unicode aware
/// Perl character class `\w` are unavailable. This only occurs when the
/// `unicode-perl` feature is disabled. (The feature is enabled by default.)
#[derive(Debug)]
pub struct UnicodeWordError(());
#[cfg(feature = "std")]
impl std::error::Error for UnicodeWordError {}
impl core::fmt::Display for UnicodeWordError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
f,
"Unicode-aware \\w class is not available \
(probably because the unicode-perl feature is not enabled)"
)
}
}
/// A state oriented traverser of the simple case folding table.
///
/// A case folder can be constructed via `SimpleCaseFolder::new()`, which will
/// return an error if the underlying case folding table is unavailable.
///
/// After construction, it is expected that callers will use
/// `SimpleCaseFolder::mapping` by calling it with codepoints in strictly
/// increasing order. For example, calling it on `b` and then on `a` is illegal
/// and will result in a panic.
///
/// The main idea of this type is that it tries hard to make mapping lookups
/// fast by exploiting the structure of the underlying table, and the ordering
/// assumption enables this.
#[derive(Debug)]
pub struct SimpleCaseFolder {
/// The simple case fold table. It's a sorted association list, where the
/// keys are Unicode scalar values and the values are the corresponding
/// equivalence class (not including the key) of the "simple" case folded
/// Unicode scalar values.
table: &'static [(char, &'static [char])],
/// The last codepoint that was used for a lookup.
last: Option<char>,
/// The index to the entry in `table` corresponding to the smallest key `k`
/// such that `k > k0`, where `k0` is the most recent key lookup. Note that
/// in particular, `k0` may not be in the table!
next: usize,
}
impl SimpleCaseFolder {
/// Create a new simple case folder, returning an error if the underlying
/// case folding table is unavailable.
pub fn new() -> Result<SimpleCaseFolder, CaseFoldError> {
#[cfg(not(feature = "unicode-case"))]
{
Err(CaseFoldError(()))
}
#[cfg(feature = "unicode-case")]
{
Ok(SimpleCaseFolder {
table: crate::unicode_tables::case_folding_simple::CASE_FOLDING_SIMPLE,
last: None,
next: 0,
})
}
}
/// Return the equivalence class of case folded codepoints for the given
/// codepoint. The equivalence class returned never includes the codepoint
/// given. If the given codepoint has no case folded codepoints (i.e.,
/// no entry in the underlying case folding table), then this returns an
/// empty slice.
///
/// # Panics
///
/// This panics when called with a `c` that is less than or equal to the
/// previous call. In other words, callers need to use this method with
/// strictly increasing values of `c`.
pub fn mapping(&mut self, c: char) -> &'static [char] {
if let Some(last) = self.last {
assert!(
last < c,
"got codepoint U+{:X} which occurs before \
last codepoint U+{:X}",
u32::from(c),
u32::from(last),
);
}
self.last = Some(c);
if self.next >= self.table.len() {
return &[];
}
let (k, v) = self.table[self.next];
if k == c {
self.next += 1;
return v;
}
match self.get(c) {
Err(i) => {
self.next = i;
&[]
}
Ok(i) => {
// Since we require lookups to proceed
// in order, anything we find should be
// after whatever we thought might be
// next. Otherwise, the caller is either
// going out of order or we would have
// found our next key at 'self.next'.
assert!(i > self.next);
self.next = i + 1;
self.table[i].1
}
}
}
/// Returns true if and only if the given range overlaps with any region
/// of the underlying case folding table. That is, when true, there exists
/// at least one codepoint in the inclusive range `[start, end]` that has
/// a non-trivial equivalence class of case folded codepoints. Conversely,
/// when this returns false, all codepoints in the range `[start, end]`
/// correspond to the trivial equivalence class of case folded codepoints,
/// i.e., itself.
///
/// This is useful to call before iterating over the codepoints in the
/// range and looking up the mapping for each. If you know none of the
/// mappings will return anything, then you might be able to skip doing it
/// altogether.
///
/// # Panics
///
/// This panics when `end < start`.
pub fn overlaps(&self, start: char, end: char) -> bool {
use core::cmp::Ordering;
assert!(start <= end);
self.table
.binary_search_by(|&(c, _)| {
if start <= c && c <= end {
Ordering::Equal
} else if c > end {
Ordering::Greater
} else {
Ordering::Less
}
})
.is_ok()
}
/// Returns the index at which `c` occurs in the simple case fold table. If
/// `c` does not occur, then this returns an `i` such that `table[i-1].0 <
/// c` and `table[i].0 > c`.
fn get(&self, c: char) -> Result<usize, usize> {
self.table.binary_search_by_key(&c, |&(c1, _)| c1)
}
}
/// A query for finding a character class defined by Unicode. This supports
/// either use of a property name directly, or lookup by property value. The
/// former generally refers to Binary properties (see UTS#44, Table 8), but
/// as a special exception (see UTS#18, Section 1.2) both general categories
/// (an enumeration) and scripts (a catalog) are supported as if each of their
/// possible values were a binary property.
///
/// In all circumstances, property names and values are normalized and
/// canonicalized. That is, `GC == gc == GeneralCategory == general_category`.
///
/// The lifetime `'a` refers to the shorter of the lifetimes of property name
/// and property value.
#[derive(Debug)]
pub enum ClassQuery<'a> {
/// Return a class corresponding to a Unicode binary property, named by
/// a single letter.
OneLetter(char),
/// Return a class corresponding to a Unicode binary property.
///
/// Note that, by special exception (see UTS#18, Section 1.2), both
/// general category values and script values are permitted here as if
/// they were a binary property.
Binary(&'a str),
/// Return a class corresponding to all codepoints whose property
/// (identified by `property_name`) corresponds to the given value
/// (identified by `property_value`).
ByValue {
/// A property name.
property_name: &'a str,
/// A property value.
property_value: &'a str,
},
}
impl<'a> ClassQuery<'a> {
fn canonicalize(&self) -> Result<CanonicalClassQuery, Error> {
match *self {
ClassQuery::OneLetter(c) => self.canonical_binary(&c.to_string()),
ClassQuery::Binary(name) => self.canonical_binary(name),
ClassQuery::ByValue { property_name, property_value } => {
let property_name = symbolic_name_normalize(property_name);
let property_value = symbolic_name_normalize(property_value);
let canon_name = match canonical_prop(&property_name)? {
None => return Err(Error::PropertyNotFound),
Some(canon_name) => canon_name,
};
Ok(match canon_name {
"General_Category" => {
let canon = match canonical_gencat(&property_value)? {
None => return Err(Error::PropertyValueNotFound),
Some(canon) => canon,
};
CanonicalClassQuery::GeneralCategory(canon)
}
"Script" => {
let canon = match canonical_script(&property_value)? {
None => return Err(Error::PropertyValueNotFound),
Some(canon) => canon,
};
CanonicalClassQuery::Script(canon)
}
_ => {
let vals = match property_values(canon_name)? {
None => return Err(Error::PropertyValueNotFound),
Some(vals) => vals,
};
let canon_val =
match canonical_value(vals, &property_value) {
None => {
return Err(Error::PropertyValueNotFound)
}
Some(canon_val) => canon_val,
};
CanonicalClassQuery::ByValue {
property_name: canon_name,
property_value: canon_val,
}
}
})
}
}
}
fn canonical_binary(
&self,
name: &str,
) -> Result<CanonicalClassQuery, Error> {
let norm = symbolic_name_normalize(name);
// This is a special case where 'cf' refers to the 'Format' general
// category, but where the 'cf' abbreviation is also an abbreviation
// for the 'Case_Folding' property. But we want to treat it as
// a general category. (Currently, we don't even support the
// 'Case_Folding' property. But if we do in the future, users will be
// required to spell it out.)
//
// Also 'sc' refers to the 'Currency_Symbol' general category, but is
// also the abbreviation for the 'Script' property. So we avoid calling
// 'canonical_prop' for it too, which would erroneously normalize it
// to 'Script'.
//
// Another case: 'lc' is an abbreviation for the 'Cased_Letter'
// general category, but is also an abbreviation for the 'Lowercase_Mapping'
// property. We don't currently support the latter, so as with 'cf'
// above, we treat 'lc' as 'Cased_Letter'.
if norm != "cf" && norm != "sc" && norm != "lc" {
if let Some(canon) = canonical_prop(&norm)? {
return Ok(CanonicalClassQuery::Binary(canon));
}
}
if let Some(canon) = canonical_gencat(&norm)? {
return Ok(CanonicalClassQuery::GeneralCategory(canon));
}
if let Some(canon) = canonical_script(&norm)? {
return Ok(CanonicalClassQuery::Script(canon));
}
Err(Error::PropertyNotFound)
}
}
/// Like ClassQuery, but its parameters have been canonicalized. This also
/// differentiates binary properties from flattened general categories and
/// scripts.
#[derive(Debug, Eq, PartialEq)]
enum CanonicalClassQuery {
/// The canonical binary property name.
Binary(&'static str),
/// The canonical general category name.
GeneralCategory(&'static str),
/// The canonical script name.
Script(&'static str),
/// An arbitrary association between property and value, both of which
/// have been canonicalized.
///
/// Note that by construction, the property name of ByValue will never
/// be General_Category or Script. Those two cases are subsumed by the
/// eponymous variants.
ByValue {
/// The canonical property name.
property_name: &'static str,
/// The canonical property value.
property_value: &'static str,
},
}
/// Looks up a Unicode class given a query. If one doesn't exist, then
/// `None` is returned.
pub fn class(query: ClassQuery<'_>) -> Result<hir::ClassUnicode, Error> {
use self::CanonicalClassQuery::*;
match query.canonicalize()? {
Binary(name) => bool_property(name),
GeneralCategory(name) => gencat(name),
Script(name) => script(name),
ByValue { property_name: "Age", property_value } => {
let mut class = hir::ClassUnicode::empty();
for set in ages(property_value)? {
class.union(&hir_class(set));
}
Ok(class)
}
ByValue { property_name: "Script_Extensions", property_value } => {
script_extension(property_value)
}
ByValue {
property_name: "Grapheme_Cluster_Break",
property_value,
} => gcb(property_value),
ByValue { property_name: "Sentence_Break", property_value } => {
sb(property_value)
}
ByValue { property_name: "Word_Break", property_value } => {
wb(property_value)
}
_ => {
// What else should we support?
Err(Error::PropertyNotFound)
}
}
}
/// Returns a Unicode aware class for \w.
///
/// This returns an error if the data is not available for \w.
pub fn perl_word() -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-perl"))]
fn imp() -> Result<hir::ClassUnicode, Error> {
Err(Error::PerlClassNotFound)
}
#[cfg(feature = "unicode-perl")]
fn imp() -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::perl_word::PERL_WORD;
Ok(hir_class(PERL_WORD))
}
imp()
}
/// Returns a Unicode aware class for \s.
///
/// This returns an error if the data is not available for \s.
pub fn perl_space() -> Result<hir::ClassUnicode, Error> {
#[cfg(not(any(feature = "unicode-perl", feature = "unicode-bool")))]
fn imp() -> Result<hir::ClassUnicode, Error> {
Err(Error::PerlClassNotFound)
}
#[cfg(all(feature = "unicode-perl", not(feature = "unicode-bool")))]
fn imp() -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::perl_space::WHITE_SPACE;
Ok(hir_class(WHITE_SPACE))
}
#[cfg(feature = "unicode-bool")]
fn imp() -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::property_bool::WHITE_SPACE;
Ok(hir_class(WHITE_SPACE))
}
imp()
}
/// Returns a Unicode aware class for \d.
///
/// This returns an error if the data is not available for \d.
pub fn perl_digit() -> Result<hir::ClassUnicode, Error> {
#[cfg(not(any(feature = "unicode-perl", feature = "unicode-gencat")))]
fn imp() -> Result<hir::ClassUnicode, Error> {
Err(Error::PerlClassNotFound)
}
#[cfg(all(feature = "unicode-perl", not(feature = "unicode-gencat")))]
fn imp() -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::perl_decimal::DECIMAL_NUMBER;
Ok(hir_class(DECIMAL_NUMBER))
}
#[cfg(feature = "unicode-gencat")]
fn imp() -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::general_category::DECIMAL_NUMBER;
Ok(hir_class(DECIMAL_NUMBER))
}
imp()
}
/// Build a Unicode HIR class from a sequence of Unicode scalar value ranges.
pub fn hir_class(ranges: &[(char, char)]) -> hir::ClassUnicode {
let hir_ranges: Vec<hir::ClassUnicodeRange> = ranges
.iter()
.map(|&(s, e)| hir::ClassUnicodeRange::new(s, e))
.collect();
hir::ClassUnicode::new(hir_ranges)
}
/// Returns true only if the given codepoint is in the `\w` character class.
///
/// If the `unicode-perl` feature is not enabled, then this returns an error.
pub fn is_word_character(c: char) -> Result<bool, UnicodeWordError> {
#[cfg(not(feature = "unicode-perl"))]
fn imp(_: char) -> Result<bool, UnicodeWordError> {
Err(UnicodeWordError(()))
}
#[cfg(feature = "unicode-perl")]
fn imp(c: char) -> Result<bool, UnicodeWordError> {
use crate::{is_word_byte, unicode_tables::perl_word::PERL_WORD};
if u8::try_from(c).map_or(false, is_word_byte) {
return Ok(true);
}
Ok(PERL_WORD
.binary_search_by(|&(start, end)| {
use core::cmp::Ordering;
if start <= c && c <= end {
Ordering::Equal
} else if start > c {
Ordering::Greater
} else {
Ordering::Less
}
})
.is_ok())
}
imp(c)
}
/// A mapping of property values for a specific property.
///
/// The first element of each tuple is a normalized property value while the
/// second element of each tuple is the corresponding canonical property
/// value.
type PropertyValues = &'static [(&'static str, &'static str)];
fn canonical_gencat(
normalized_value: &str,
) -> Result<Option<&'static str>, Error> {
Ok(match normalized_value {
"any" => Some("Any"),
"assigned" => Some("Assigned"),
"ascii" => Some("ASCII"),
_ => {
let gencats = property_values("General_Category")?.unwrap();
canonical_value(gencats, normalized_value)
}
})
}
fn canonical_script(
normalized_value: &str,
) -> Result<Option<&'static str>, Error> {
let scripts = property_values("Script")?.unwrap();
Ok(canonical_value(scripts, normalized_value))
}
/// Find the canonical property name for the given normalized property name.
///
/// If no such property exists, then `None` is returned.
///
/// The normalized property name must have been normalized according to
/// UAX44 LM3, which can be done using `symbolic_name_normalize`.
///
/// If the property names data is not available, then an error is returned.
fn canonical_prop(
normalized_name: &str,
) -> Result<Option<&'static str>, Error> {
#[cfg(not(any(
feature = "unicode-age",
feature = "unicode-bool",
feature = "unicode-gencat",
feature = "unicode-perl",
feature = "unicode-script",
feature = "unicode-segment",
)))]
fn imp(_: &str) -> Result<Option<&'static str>, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(any(
feature = "unicode-age",
feature = "unicode-bool",
feature = "unicode-gencat",
feature = "unicode-perl",
feature = "unicode-script",
feature = "unicode-segment",
))]
fn imp(name: &str) -> Result<Option<&'static str>, Error> {
use crate::unicode_tables::property_names::PROPERTY_NAMES;
Ok(PROPERTY_NAMES
.binary_search_by_key(&name, |&(n, _)| n)
.ok()
.map(|i| PROPERTY_NAMES[i].1))
}
imp(normalized_name)
}
/// Find the canonical property value for the given normalized property
/// value.
///
/// The given property values should correspond to the values for the property
/// under question, which can be found using `property_values`.
///
/// If no such property value exists, then `None` is returned.
///
/// The normalized property value must have been normalized according to
/// UAX44 LM3, which can be done using `symbolic_name_normalize`.
fn canonical_value(
vals: PropertyValues,
normalized_value: &str,
) -> Option<&'static str> {
vals.binary_search_by_key(&normalized_value, |&(n, _)| n)
.ok()
.map(|i| vals[i].1)
}
/// Return the table of property values for the given property name.
///
/// If the property values data is not available, then an error is returned.
fn property_values(
canonical_property_name: &'static str,
) -> Result<Option<PropertyValues>, Error> {
#[cfg(not(any(
feature = "unicode-age",
feature = "unicode-bool",
feature = "unicode-gencat",
feature = "unicode-perl",
feature = "unicode-script",
feature = "unicode-segment",
)))]
fn imp(_: &'static str) -> Result<Option<PropertyValues>, Error> {
Err(Error::PropertyValueNotFound)
}
#[cfg(any(
feature = "unicode-age",
feature = "unicode-bool",
feature = "unicode-gencat",
feature = "unicode-perl",
feature = "unicode-script",
feature = "unicode-segment",
))]
fn imp(name: &'static str) -> Result<Option<PropertyValues>, Error> {
use crate::unicode_tables::property_values::PROPERTY_VALUES;
Ok(PROPERTY_VALUES
.binary_search_by_key(&name, |&(n, _)| n)
.ok()
.map(|i| PROPERTY_VALUES[i].1))
}
imp(canonical_property_name)
}
// This is only used in some cases, but small enough to just let it be dead
// instead of figuring out (and maintaining) the right set of features.
#[allow(dead_code)]
fn property_set(
name_map: &'static [(&'static str, Range)],
canonical: &'static str,
) -> Option<Range> {
name_map
.binary_search_by_key(&canonical, |x| x.0)
.ok()
.map(|i| name_map[i].1)
}
/// Returns an iterator over Unicode Age sets. Each item corresponds to a set
/// of codepoints that were added in a particular revision of Unicode. The
/// iterator yields items in chronological order.
///
/// If the given age value isn't valid or if the data isn't available, then an
/// error is returned instead.
fn ages(canonical_age: &str) -> Result<impl Iterator<Item = Range>, Error> {
#[cfg(not(feature = "unicode-age"))]
fn imp(_: &str) -> Result<impl Iterator<Item = Range>, Error> {
use core::option::IntoIter;
Err::<IntoIter<Range>, _>(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-age")]
fn imp(canonical_age: &str) -> Result<impl Iterator<Item = Range>, Error> {
use crate::unicode_tables::age;
const AGES: &[(&str, Range)] = &[
("V1_1", age::V1_1),
("V2_0", age::V2_0),
("V2_1", age::V2_1),
("V3_0", age::V3_0),
("V3_1", age::V3_1),
("V3_2", age::V3_2),
("V4_0", age::V4_0),
("V4_1", age::V4_1),
("V5_0", age::V5_0),
("V5_1", age::V5_1),
("V5_2", age::V5_2),
("V6_0", age::V6_0),
("V6_1", age::V6_1),
("V6_2", age::V6_2),
("V6_3", age::V6_3),
("V7_0", age::V7_0),
("V8_0", age::V8_0),
("V9_0", age::V9_0),
("V10_0", age::V10_0),
("V11_0", age::V11_0),
("V12_0", age::V12_0),
("V12_1", age::V12_1),
("V13_0", age::V13_0),
("V14_0", age::V14_0),
("V15_0", age::V15_0),
];
assert_eq!(AGES.len(), age::BY_NAME.len(), "ages are out of sync");
let pos = AGES.iter().position(|&(age, _)| canonical_age == age);
match pos {
None => Err(Error::PropertyValueNotFound),
Some(i) => Ok(AGES[..=i].iter().map(|&(_, classes)| classes)),
}
}
imp(canonical_age)
}
/// Returns the Unicode HIR class corresponding to the given general category.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given general category could not be found, or if the general
/// category data is not available, then an error is returned.
fn gencat(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-gencat"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-gencat")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::general_category::BY_NAME;
match name {
"ASCII" => Ok(hir_class(&[('\0', '\x7F')])),
"Any" => Ok(hir_class(&[('\0', '\u{10FFFF}')])),
"Assigned" => {
let mut cls = gencat("Unassigned")?;
cls.negate();
Ok(cls)
}
name => property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyValueNotFound),
}
}
match canonical_name {
"Decimal_Number" => perl_digit(),
name => imp(name),
}
}
/// Returns the Unicode HIR class corresponding to the given script.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given script could not be found, or if the script data is not
/// available, then an error is returned.
fn script(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-script"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-script")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::script::BY_NAME;
property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyValueNotFound)
}
imp(canonical_name)
}
/// Returns the Unicode HIR class corresponding to the given script extension.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given script extension could not be found, or if the script data is
/// not available, then an error is returned.
fn script_extension(
canonical_name: &'static str,
) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-script"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-script")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::script_extension::BY_NAME;
property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyValueNotFound)
}
imp(canonical_name)
}
/// Returns the Unicode HIR class corresponding to the given Unicode boolean
/// property.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given boolean property could not be found, or if the boolean
/// property data is not available, then an error is returned.
fn bool_property(
canonical_name: &'static str,
) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-bool"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-bool")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::property_bool::BY_NAME;
property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyNotFound)
}
match canonical_name {
"Decimal_Number" => perl_digit(),
"White_Space" => perl_space(),
name => imp(name),
}
}
/// Returns the Unicode HIR class corresponding to the given grapheme cluster
/// break property.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given property could not be found, or if the corresponding data is
/// not available, then an error is returned.
fn gcb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-segment"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-segment")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::grapheme_cluster_break::BY_NAME;
property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyValueNotFound)
}
imp(canonical_name)
}
/// Returns the Unicode HIR class corresponding to the given word break
/// property.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given property could not be found, or if the corresponding data is
/// not available, then an error is returned.
fn wb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-segment"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-segment")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::word_break::BY_NAME;
property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyValueNotFound)
}
imp(canonical_name)
}
/// Returns the Unicode HIR class corresponding to the given sentence
/// break property.
///
/// Name canonicalization is assumed to be performed by the caller.
///
/// If the given property could not be found, or if the corresponding data is
/// not available, then an error is returned.
fn sb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> {
#[cfg(not(feature = "unicode-segment"))]
fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
#[cfg(feature = "unicode-segment")]
fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> {
use crate::unicode_tables::sentence_break::BY_NAME;
property_set(BY_NAME, name)
.map(hir_class)
.ok_or(Error::PropertyValueNotFound)
}
imp(canonical_name)
}
/// Like symbolic_name_normalize_bytes, but operates on a string.
fn symbolic_name_normalize(x: &str) -> String {
let mut tmp = x.as_bytes().to_vec();
let len = symbolic_name_normalize_bytes(&mut tmp).len();
tmp.truncate(len);
// This should always succeed because `symbolic_name_normalize_bytes`
// guarantees that `&tmp[..len]` is always valid UTF-8.
//
// N.B. We could avoid the additional UTF-8 check here, but it's unlikely
// to be worth skipping the additional safety check. A benchmark must
// justify it first.
String::from_utf8(tmp).unwrap()
}
/// Normalize the given symbolic name in place according to UAX44-LM3.
///
/// A "symbolic name" typically corresponds to property names and property
/// value aliases. Note, though, that it should not be applied to property
/// string values.
///
/// The slice returned is guaranteed to be valid UTF-8 for all possible values
/// of `slice`.
///
fn symbolic_name_normalize_bytes(slice: &mut [u8]) -> &mut [u8] {
// I couldn't find a place in the standard that specified that property
// names/aliases had a particular structure (unlike character names), but
// we assume that it's ASCII only and drop anything that isn't ASCII.
let mut start = 0;
let mut starts_with_is = false;
if slice.len() >= 2 {
// Ignore any "is" prefix.
starts_with_is = slice[0..2] == b"is"[..]
|| slice[0..2] == b"IS"[..]
|| slice[0..2] == b"iS"[..]
|| slice[0..2] == b"Is"[..];
if starts_with_is {
start = 2;
}
}
let mut next_write = 0;
for i in start..slice.len() {
// VALIDITY ARGUMENT: To guarantee that the resulting slice is valid
// UTF-8, we ensure that the slice contains only ASCII bytes. In
// particular, we drop every non-ASCII byte from the normalized string.
let b = slice[i];
if b == b' ' || b == b'_' || b == b'-' {
continue;
} else if b'A' <= b && b <= b'Z' {
slice[next_write] = b + (b'a' - b'A');
next_write += 1;
} else if b <= 0x7F {
slice[next_write] = b;
next_write += 1;
}
}
// Special case: ISO_Comment has a 'isc' abbreviation. Since we generally
// ignore 'is' prefixes, the 'isc' abbreviation gets caught in the cross
// fire and ends up creating an alias for 'c' to 'ISO_Comment', but it
// is actually an alias for the 'Other' general category.
if starts_with_is && next_write == 1 && slice[0] == b'c' {
slice[0] = b'i';
slice[1] = b's';
slice[2] = b'c';
next_write = 3;
}
&mut slice[..next_write]
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "unicode-case")]
fn simple_fold_ok(c: char) -> impl Iterator<Item = char> {
SimpleCaseFolder::new().unwrap().mapping(c).iter().copied()
}
#[cfg(feature = "unicode-case")]
fn contains_case_map(start: char, end: char) -> bool {
SimpleCaseFolder::new().unwrap().overlaps(start, end)
}
#[test]
#[cfg(feature = "unicode-case")]
fn simple_fold_k() {
let xs: Vec<char> = simple_fold_ok('k').collect();
assert_eq!(xs, alloc::vec!['K', 'K']);
let xs: Vec<char> = simple_fold_ok('K').collect();
assert_eq!(xs, alloc::vec!['k', 'K']);
let xs: Vec<char> = simple_fold_ok('K').collect();
assert_eq!(xs, alloc::vec!['K', 'k']);
}
#[test]
#[cfg(feature = "unicode-case")]
fn simple_fold_a() {
let xs: Vec<char> = simple_fold_ok('a').collect();
assert_eq!(xs, alloc::vec!['A']);
let xs: Vec<char> = simple_fold_ok('A').collect();
assert_eq!(xs, alloc::vec!['a']);
}
#[test]
#[cfg(not(feature = "unicode-case"))]
fn simple_fold_disabled() {
assert!(SimpleCaseFolder::new().is_err());
}
#[test]
#[cfg(feature = "unicode-case")]
fn range_contains() {
assert!(contains_case_map('A', 'A'));
assert!(contains_case_map('Z', 'Z'));
assert!(contains_case_map('A', 'Z'));
assert!(contains_case_map('@', 'A'));
assert!(contains_case_map('Z', '['));
assert!(contains_case_map('☃', 'Ⰰ'));
assert!(!contains_case_map('[', '['));
assert!(!contains_case_map('[', '`'));
assert!(!contains_case_map('☃', '☃'));
}
#[test]
#[cfg(feature = "unicode-gencat")]
fn regression_466() {
use super::{CanonicalClassQuery, ClassQuery};
let q = ClassQuery::OneLetter('C');
assert_eq!(
q.canonicalize().unwrap(),
CanonicalClassQuery::GeneralCategory("Other")
);
}
#[test]
fn sym_normalize() {
let sym_norm = symbolic_name_normalize;
assert_eq!(sym_norm("Line_Break"), "linebreak");
assert_eq!(sym_norm("Line-break"), "linebreak");
assert_eq!(sym_norm("linebreak"), "linebreak");
assert_eq!(sym_norm("BA"), "ba");
assert_eq!(sym_norm("ba"), "ba");
assert_eq!(sym_norm("Greek"), "greek");
assert_eq!(sym_norm("isGreek"), "greek");
assert_eq!(sym_norm("IS_Greek"), "greek");
assert_eq!(sym_norm("isc"), "isc");
assert_eq!(sym_norm("is c"), "isc");
assert_eq!(sym_norm("is_c"), "isc");
}
#[test]
fn valid_utf8_symbolic() {
let mut x = b"abc\xFFxyz".to_vec();
let y = symbolic_name_normalize_bytes(&mut x);
assert_eq!(y, b"abcxyz");
}
}