0x1949 Team - FAZEMRX - MANAGER
Edit File: Subs-Charset.php
<?php /** * Simple Machines Forum (SMF) * * @package SMF * @author Simple Machines https://www.simplemachines.org * @copyright 2023 Simple Machines and individual contributors * @license https://www.simplemachines.org/about/smf/license.php BSD * * @version 2.1.4 */ if (!defined('SMF')) die('No direct access...'); require_once($sourcedir . '/Unicode/Metadata.php'); /** * Converts the given UTF-8 string into lowercase. * Equivalent to mb_strtolower($string, 'UTF-8'), except that we can keep the * output consistent across PHP versions and up to date with the latest version * of Unicode. * * @param string $string The string * @return string The lowercase version of $string */ function utf8_strtolower($string) { return utf8_convert_case($string, 'lower'); } /** * Convert the given UTF-8 string to uppercase. * Equivalent to mb_strtoupper($string, 'UTF-8'), except that we can keep the * output consistent across PHP versions and up to date with the latest version * of Unicode. * * @param string $string The string * @return string The uppercase version of $string */ function utf8_strtoupper($string) { return utf8_convert_case($string, 'upper'); } /** * Casefolds the given UTF-8 string. * Equivalent to mb_convert_case($string, MB_CASE_FOLD, 'UTF-8'), except that * we can keep the output consistent across PHP versions and up to date with * the latest version of Unicode. * * @param string $string The string * @return string The uppercase version of $string */ function utf8_casefold($string) { return utf8_convert_case($string, 'fold'); } /** * Converts the case of the given UTF-8 string. * * @param string $string The string. * @param string $case One of 'upper', 'lower', 'fold', 'title', 'ucfirst', or 'ucwords'. * @param bool $simple If true, use simple maps instead of full maps. Default: false. * @return string A version of $string converted to the specified case. */ function utf8_convert_case($string, $case, $simple = false) { global $sourcedir, $txt; $simple = !empty($simple); $lang = empty($txt['lang_locale']) ? '' : substr($txt['lang_locale'], 0, 2); // The main case conversion logic if (in_array($case, array('upper', 'lower', 'fold'))) { $chars = preg_split('/(.)/su', $string, 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); if ($chars === false) return false; switch ($case) { case 'upper': require_once($sourcedir . '/Unicode/CaseUpper.php'); $substitutions = $simple ? utf8_strtoupper_simple_maps() : utf8_strtoupper_maps(); // Turkish & Azeri conditional casing, part 1. if (in_array($lang, array('tr', 'az'))) $substitutions['i'] = 'İ'; break; case 'lower': require_once($sourcedir . '/Unicode/CaseLower.php'); $substitutions = $simple ? utf8_strtolower_simple_maps() : utf8_strtolower_maps(); // Turkish & Azeri conditional casing, part 1. if (in_array($lang, array('tr', 'az'))) { $substitutions['İ'] = 'i'; $substitutions['I' . "\xCC\x87"] = 'i'; $substitutions['I'] = 'ı'; } break; case 'fold': require_once($sourcedir . '/Unicode/CaseFold.php'); $substitutions = $simple ? utf8_casefold_simple_maps() : utf8_casefold_maps(); break; } foreach ($chars as &$char) $char = isset($substitutions[$char]) ? $substitutions[$char] : $char; $string = implode('', $chars); } elseif (in_array($case, array('title', 'ucfirst', 'ucwords'))) { require_once($sourcedir . '/Unicode/RegularExpressions.php'); require_once($sourcedir . '/Unicode/CaseUpper.php'); require_once($sourcedir . '/Unicode/CaseTitle.php'); $prop_classes = utf8_regex_properties(); $upper = $simple ? utf8_strtoupper_simple_maps() : utf8_strtoupper_maps(); // Turkish & Azeri conditional casing, part 1. if (in_array($lang, array('tr', 'az'))) $upper['i'] = 'İ'; $title = array_merge($upper, $simple ? utf8_titlecase_simple_maps() : utf8_titlecase_maps()); switch ($case) { case 'title': $string = utf8_convert_case($string, 'lower', $simple); $regex = '/(?:^|[^\w' . $prop_classes['Case_Ignorable'] . '])\K(\p{L})/u'; break; case 'ucwords': $regex = '/(?:^|[^\w' . $prop_classes['Case_Ignorable'] . '])\K(\p{L})(?=[' . $prop_classes['Case_Ignorable'] . ']*(?:(?<upper>\p{Lu})|\w?))/u'; break; case 'ucfirst': $regex = '/^[^\w' . $prop_classes['Case_Ignorable'] . ']*\K(\p{L})(?=[' . $prop_classes['Case_Ignorable'] . ']*(?:(?<upper>\p{Lu})|\w?))/u'; break; } $string = preg_replace_callback( $regex, function($matches) use ($upper, $title) { // If second letter is uppercase, use uppercase for first letter. // Otherwise, use titlecase for first letter. $case = !empty($matches['upper']) ? 'upper' : 'title'; $matches[1] = isset($$case[$matches[1]]) ? $$case[$matches[1]] : $matches[1]; return $matches[1]; }, $string ); } // If casefolding, we're done. if ($case === 'fold') return $string; // Handle conditional casing situations... $substitutions = array(); $replacements = array(); // Greek conditional casing, part 1: Fix lowercase sigma. // Note that this rule doesn't depend on $txt['lang_locale']. if ($case !== 'upper' && strpos($string, 'ς') !== false || strpos($string, 'σ') !== false) { require_once($sourcedir . '/Unicode/RegularExpressions.php'); $prop_classes = utf8_regex_properties(); // First, convert all lowercase sigmas to regular form. $substitutions['ς'] = 'σ'; // Then convert any at the end of words to final form. $replacements['/\Bσ([' . $prop_classes['Case_Ignorable'] . ']*)(?!\p{L})/u'] = 'ς$1'; } // Greek conditional casing, part 2: No accents on uppercase strings. if ($lang === 'el' && $case === 'upper') { // Composed forms. $substitutions += array( 'Ά' => 'Α', 'Ἀ' => 'Α', 'Ἁ' => 'Α', 'Ὰ' => 'Α', 'Ᾰ' => 'Α', 'Ᾱ' => 'Α', 'Α' => 'Α', 'Α' => 'Α', 'Ἂ' => 'Α', 'Ἃ' => 'Α', 'Ἄ' => 'Α', 'Ἅ' => 'Α', 'Ἆ' => 'Α', 'Ἇ' => 'Α', 'Ὰ' => 'Α', 'Ά' => 'Α', 'Α' => 'Α', 'Ἀ' => 'Α', 'Ἁ' => 'Α', 'Ἂ' => 'Α', 'Ἃ' => 'Α', 'Ἄ' => 'Α', 'Ἅ' => 'Α', 'Ἆ' => 'Α', 'Ἇ' => 'Α', 'Έ' => 'Ε', 'Ἐ' => 'Ε', 'Ἑ' => 'Ε', 'Ὲ' => 'Ε', 'Ἒ' => 'Ε', 'Ἓ' => 'Ε', 'Ἔ' => 'Ε', 'Ἕ' => 'Ε', 'Ή' => 'Η', 'Ἠ' => 'Η', 'Ἡ' => 'Η', 'Ὴ' => 'Η', 'Η' => 'Η', 'Η' => 'Η', 'Ἢ' => 'Η', 'Ἣ' => 'Η', 'Ἤ' => 'Η', 'Ἥ' => 'Η', 'Ἦ' => 'Η', 'Ἧ' => 'Η', 'Ἠ' => 'Η', 'Ἡ' => 'Η', 'Ὴ' => 'Η', 'Ή' => 'Η', 'Η' => 'Η', 'Ἢ' => 'Η', 'Ἣ' => 'Η', 'Ἤ' => 'Η', 'Ἥ' => 'Η', 'Ἦ' => 'Η', 'Ἧ' => 'Η', 'Ί' => 'Ι', 'Ἰ' => 'Ι', 'Ἱ' => 'Ι', 'Ὶ' => 'Ι', 'Ῐ' => 'Ι', 'Ῑ' => 'Ι', 'Ι' => 'Ι', 'Ϊ' => 'Ι', 'Ι' => 'Ι', 'Ἲ' => 'Ι', 'Ἳ' => 'Ι', 'Ἴ' => 'Ι', 'Ἵ' => 'Ι', 'Ἶ' => 'Ι', 'Ἷ' => 'Ι', 'Ι' => 'Ι', 'Ι' => 'Ι', 'Ό' => 'Ο', 'Ὀ' => 'Ο', 'Ὁ' => 'Ο', 'Ὸ' => 'Ο', 'Ὂ' => 'Ο', 'Ὃ' => 'Ο', 'Ὄ' => 'Ο', 'Ὅ' => 'Ο', 'Ῥ' => 'Ρ', 'Ύ' => 'Υ', 'Υ' => 'Υ', 'Ὑ' => 'Υ', 'Ὺ' => 'Υ', 'Ῠ' => 'Υ', 'Ῡ' => 'Υ', 'Υ' => 'Υ', 'Ϋ' => 'Υ', 'Υ' => 'Υ', 'Υ' => 'Υ', 'Ὓ' => 'Υ', 'Υ' => 'Υ', 'Ὕ' => 'Υ', 'Υ' => 'Υ', 'Ὗ' => 'Υ', 'Υ' => 'Υ', 'Υ' => 'Υ', 'Υ' => 'Υ', 'Ώ' => 'Ω', 'Ὠ' => 'Ω', 'Ὡ' => 'Ω', 'Ὼ' => 'Ω', 'Ω' => 'Ω', 'Ω' => 'Ω', 'Ὢ' => 'Ω', 'Ὣ' => 'Ω', 'Ὤ' => 'Ω', 'Ὥ' => 'Ω', 'Ὦ' => 'Ω', 'Ὧ' => 'Ω', 'Ὠ' => 'Ω', 'Ὡ' => 'Ω', 'Ώ' => 'Ω', 'Ω' => 'Ω', 'Ὢ' => 'Ω', 'Ὣ' => 'Ω', 'Ὤ' => 'Ω', 'Ὥ' => 'Ω', 'Ὦ' => 'Ω', 'Ὧ' => 'Ω', ); // Individual Greek diacritics. $substitutions += array( "\xCC\x80" => '', "\xCC\x81" => '', "\xCC\x84" => '', "\xCC\x86" => '', "\xCC\x88" => '', "\xCC\x93" => '', "\xCC\x94" => '', "\xCD\x82" => '', "\xCD\x83" => '', "\xCD\x84" => '', "\xCD\x85" => '', "\xCD\xBA" => '', "\xCE\x84" => '', "\xCE\x85" => '', "\xE1\xBE\xBD" => '', "\xE1\xBE\xBF" => '', "\xE1\xBF\x80" => '', "\xE1\xBF\x81" => '', "\xE1\xBF\x8D" => '', "\xE1\xBF\x8E" => '', "\xE1\xBF\x8F" => '', "\xE1\xBF\x9D" => '', "\xE1\xBF\x9E" => '', "\xE1\xBF\x9F" => '', "\xE1\xBF\xAD" => '', "\xE1\xBF\xAE" => '', "\xE1\xBF\xAF" => '', "\xE1\xBF\xBD" => '', "\xE1\xBF\xBE" => '', ); } // Turkish & Azeri conditional casing, part 2. if ($case !== 'upper' && in_array($lang, array('tr', 'az'))) { // Remove unnecessary "COMBINING DOT ABOVE" after i $substitutions['i' . "\xCC\x87"] = 'i'; } // Lithuanian conditional casing. if ($lang === 'lt') { // Force a dot above lowercase i and j with accents by inserting // the "COMBINING DOT ABOVE" character. // Note: some fonts handle this incorrectly and show two dots, // but that's a bug in those fonts and cannot be fixed here. if ($case !== 'upper') $replacements['/(i\x{328}?|\x{12F}|j)([\x{300}\x{301}\x{303}])/u'] = '$1' . "\xCC\x87" . '$2'; // Remove "COMBINING DOT ABOVE" after uppercase I and J. if ($case !== 'lower') $replacements['/(I\x{328}?|\x{12E}|J)\x{307}/u'] = '$1'; } // Dutch has a special titlecase rule. if ($lang === 'nl' && $case === 'title') { $replacements['/\bIj/u'] = 'IJ'; } // Now perform whatever conditional casing fixes we need. if (!empty($substitutions)) $string = strtr($string, $substitutions); if (!empty($replacements)) $string = preg_replace(array_keys($replacements), $replacements, $string); return $string; } /** * Normalizes UTF-8 via Canonical Decomposition. * * @param string $string A UTF-8 string * @return string The decomposed version of $string */ function utf8_normalize_d($string) { $string = (string) $string; if (is_callable('IntlChar::getUnicodeVersion') && version_compare(implode('.', IntlChar::getUnicodeVersion()), SMF_UNICODE_VERSION, '>=')) { if (is_callable('normalizer_is_normalized') && normalizer_is_normalized($string, Normalizer::FORM_D)) return $string; if (is_callable('normalizer_normalize')) return normalizer_normalize($string, Normalizer::FORM_D); } if (utf8_is_normalized($string, 'd')) return $string; $chars = preg_split('/(.)/su', $string, 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); if ($chars === false) return false; return implode('', utf8_decompose($chars, false)); } /** * Normalizes UTF-8 via Compatibility Decomposition. * * @param string $string A UTF-8 string. * @return string The decomposed version of $string. */ function utf8_normalize_kd($string) { $string = (string) $string; if (is_callable('IntlChar::getUnicodeVersion') && version_compare(implode('.', IntlChar::getUnicodeVersion()), SMF_UNICODE_VERSION, '>=')) { if (is_callable('normalizer_is_normalized') && normalizer_is_normalized($string, Normalizer::FORM_KD)) return $string; if (is_callable('normalizer_normalize')) return normalizer_normalize($string, Normalizer::FORM_KD); } if (utf8_is_normalized($string, 'kd')) return $string; $chars = preg_split('/(.)/su', $string, 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); if ($chars === false) return false; return implode('', utf8_decompose($chars, true)); } /** * Normalizes UTF-8 via Canonical Decomposition then Canonical Composition. * * @param string $string A UTF-8 string * @return string The composed version of $string */ function utf8_normalize_c($string) { $string = (string) $string; if (is_callable('IntlChar::getUnicodeVersion') && version_compare(implode('.', IntlChar::getUnicodeVersion()), SMF_UNICODE_VERSION, '>=')) { if (is_callable('normalizer_is_normalized') && normalizer_is_normalized($string, Normalizer::FORM_C)) return $string; if (is_callable('normalizer_normalize')) return normalizer_normalize($string, Normalizer::FORM_C); } if (utf8_is_normalized($string, 'c')) return $string; $chars = preg_split('/(.)/su', $string, 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); if ($chars === false) return false; return implode('', utf8_compose(utf8_decompose($chars, false))); } /** * Normalizes UTF-8 via Compatibility Decomposition then Canonical Composition. * * @param string $string The string * @return string The composed version of $string */ function utf8_normalize_kc($string) { $string = (string) $string; if (is_callable('IntlChar::getUnicodeVersion') && version_compare(implode('.', IntlChar::getUnicodeVersion()), SMF_UNICODE_VERSION, '>=')) { if (is_callable('normalizer_is_normalized') && normalizer_is_normalized($string, Normalizer::FORM_KC)) return $string; if (is_callable('normalizer_normalize')) return normalizer_normalize($string, Normalizer::FORM_KC); } if (utf8_is_normalized($string, 'kc')) return $string; $chars = preg_split('/(.)/su', $string, 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); if ($chars === false) return false; return implode('', utf8_compose(utf8_decompose($chars, true))); } /** * Casefolds UTF-8 via Compatibility Composition Casefolding. * Used by idn_to_ascii polyfill in Subs-Compat.php * * @param string $string The string * @return string The casefolded version of $string */ function utf8_normalize_kc_casefold($string) { global $sourcedir; $string = (string) $string; if (utf8_is_normalized($string, 'kc_casefold')) return $string; $chars = preg_split('/(.)/su', $string, 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); if ($chars === false) return false; $chars = utf8_decompose($chars, true); require_once($sourcedir . '/Unicode/CaseFold.php'); require_once($sourcedir . '/Unicode/DefaultIgnorables.php'); $substitutions = utf8_casefold_maps(); $ignorables = array_flip(utf8_default_ignorables()); foreach ($chars as &$char) { if (isset($substitutions[$char])) $char = $substitutions[$char]; elseif (isset($ignorables[$char])) $char = ''; } return implode('', utf8_compose($chars)); } /** * Checks whether a string is already normalized to a given form. * * @param string|array $string A string of UTF-8 characters. * @param string $form One of 'd', 'c', 'kd', 'kc', or 'kc_casefold' * @return bool Whether the string is already normalized to the given form. */ function utf8_is_normalized($string, $form) { global $sourcedir; // Check whether string contains characters that are disallowed in this form. switch ($form) { case 'd': $prop = 'NFD_QC'; break; case 'kd': $prop = 'NFKD_QC'; break; case 'c': $prop = 'NFC_QC'; break; case 'kc': $prop = 'NFKC_QC'; break; case 'kc_casefold': $prop = 'Changes_When_NFKC_Casefolded'; break; default: return false; break; } require_once($sourcedir . '/Unicode/QuickCheck.php'); $qc = utf8_regex_quick_check(); if (preg_match('/[' . $qc[$prop] . ']/u', $string)) return false; // Check whether all combining marks are in canonical order. // Note: Because PCRE's Unicode data might be outdated compared to ours, // this regex checks for marks and anything PCRE thinks is not a character. // That means the more thorough checks will occasionally be performed on // strings that don't need them, but building and running a perfect regex // would be more expensive in the vast majority of cases, so meh. if (preg_match_all('/([\p{M}\p{Cn}])/u', $string, $matches, PREG_OFFSET_CAPTURE)) { require_once($sourcedir . '/Unicode/CombiningClasses.php'); $combining_classes = utf8_combining_classes(); $last_pos = 0; $last_len = 0; $last_ccc = 0; foreach ($matches[1] as $match) { $char = $match[0]; $pos = $match[1]; $ccc = isset($combining_classes[$char]) ? $combining_classes[$char] : 0; // Not in canonical order, so return false. if ($pos === $last_pos + $last_len && $ccc > 0 && $last_ccc > $ccc) return false; $last_pos = $pos; $last_len = strlen($char); $last_ccc = $ccc; } } // If we get here, the string is normalized correctly. return true; } /** * Helper function for utf8_normalize_d and utf8_normalize_kd. * * @param array $chars Array of Unicode characters * @param bool $compatibility If true, perform compatibility decomposition. Default false. * @return array Array of decomposed Unicode characters. */ function utf8_decompose($chars, $compatibility = false) { global $sourcedir; if (!empty($compatibility)) { require_once($sourcedir . '/Unicode/DecompositionCompatibility.php'); $substitutions = utf8_normalize_kd_maps(); foreach ($chars as &$char) $char = isset($substitutions[$char]) ? $substitutions[$char] : $char; } require_once($sourcedir . '/Unicode/DecompositionCanonical.php'); require_once($sourcedir . '/Unicode/CombiningClasses.php'); $substitutions = utf8_normalize_d_maps(); $combining_classes = utf8_combining_classes(); // Replace characters with decomposed forms. for ($i=0; $i < count($chars); $i++) { // Hangul characters. // See "Hangul Syllable Decomposition" in the Unicode standard, ch. 3.12. if ($chars[$i] >= "\xEA\xB0\x80" && $chars[$i] <= "\xED\x9E\xA3") { if (!function_exists('mb_ord')) require_once($sourcedir . '/Subs-Compat.php'); $s = mb_ord($chars[$i]); $sindex = $s - 0xAC00; $l = (int) (0x1100 + $sindex / (21 * 28)); $v = (int) (0x1161 + ($sindex % (21 * 28)) / 28); $t = $sindex % 28; $chars[$i] = implode('', array(mb_chr($l), mb_chr($v), $t ? mb_chr(0x11A7 + $t) : '')); } // Everything else. elseif (isset($substitutions[$chars[$i]])) $chars[$i] = $substitutions[$chars[$i]]; } // Must re-split the string before sorting. $chars = preg_split('/(.)/su', implode('', $chars), 0, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY); // Sort characters into canonical order. for ($i = 1; $i < count($chars); $i++) { if (empty($combining_classes[$chars[$i]]) || empty($combining_classes[$chars[$i - 1]])) continue; if ($combining_classes[$chars[$i - 1]] > $combining_classes[$chars[$i]]) { $temp = $chars[$i]; $chars[$i] = $chars[$i - 1]; $chars[$i -1] = $temp; // Backtrack and check again. if ($i > 1) $i -= 2; } } return $chars; } /** * Helper function for utf8_normalize_c and utf8_normalize_kc. * * @param array $chars Array of decomposed Unicode characters * @return array Array of composed Unicode characters. */ function utf8_compose($chars) { global $sourcedir; require_once($sourcedir . '/Unicode/Composition.php'); require_once($sourcedir . '/Unicode/CombiningClasses.php'); $substitutions = utf8_compose_maps(); $combining_classes = utf8_combining_classes(); for ($c = 0; $c < count($chars); $c++) { // Singleton replacements. if (isset($substitutions[$chars[$c]])) $chars[$c] = $substitutions[$chars[$c]]; // Hangul characters. // See "Hangul Syllable Composition" in the Unicode standard, ch. 3.12. if ($chars[$c] >= "\xE1\x84\x80" && $chars[$c] <= "\xE1\x84\x92" && isset($chars[$c + 1]) && $chars[$c + 1] >= "\xE1\x85\xA1" && $chars[$c + 1] <= "\xE1\x85\xB5") { if (!function_exists('mb_ord')) require_once($sourcedir . '/Subs-Compat.php'); $l_part = $chars[$c]; $v_part = $chars[$c + 1]; $t_part = null; $l_index = mb_ord($l_part) - 0x1100; $v_index = mb_ord($v_part) - 0x1161; $lv_index = $l_index * 588 + $v_index * 28; $s = 0xAC00 + $lv_index; if (isset($chars[$c + 2]) && $chars[$c + 2] >= "\xE1\x86\xA8" && $chars[$c + 2] <= "\xE1\x87\x82") { $t_part = $chars[$c + 2]; $t_index = mb_ord($t_part) - 0x11A7; $s += $t_index; } $chars[$c] = mb_chr($s); $chars[++$c] = null; if (isset($t_part)) $chars[++$c] = null; continue; } if ($c > 0) { $ccc = isset($combining_classes[$chars[$c]]) ? $combining_classes[$chars[$c]] : 0; // Find the preceding starter character. $l = $c - 1; while ($l > 0 && (!isset($chars[$l]) || (!empty($combining_classes[$chars[$l]]) && $combining_classes[$chars[$l]] < $ccc))) $l--; // Is there a composed form for this combination? if (isset($substitutions[$chars[$l] . $chars[$c]])) { // Replace the starter character with the composed character. $chars[$l] = $substitutions[$chars[$l] . $chars[$c]]; // Unset the current combining character. $chars[$c] = null; } } } return $chars; } /** * Helper function for sanitize_chars() that deals with invisible characters. * * This function deals with control characters, private use characters, * non-characters, and characters that are invisible by definition in the * Unicode standard. It does not deal with characters that are supposed to be * visible according to the Unicode standard, and makes no attempt to compensate * for possibly incomplete Unicode support in text rendering engines on client * devices. * * @param string $string The string to sanitize. * @param int $level Controls how invisible formatting characters are handled. * 0: Allow valid formatting characters. Use for sanitizing text in posts. * 1: Allow necessary formatting characters. Use for sanitizing usernames. * 2: Disallow all formatting characters. Use for internal comparisions * only, such as in the word censor, search contexts, etc. * @param string $substitute Replacement string for the invalid characters. * @return string The sanitized string. */ function utf8_sanitize_invisibles($string, $level, $substitute) { global $sourcedir; $string = (string) $string; $level = min(max((int) $level, 0), 2); $substitute = (string) $substitute; require_once($sourcedir . '/Unicode/RegularExpressions.php'); $prop_classes = utf8_regex_properties(); // We never want non-whitespace control characters $disallowed[] = '[^\P{Cc}\t\r\n]'; // We never want private use characters or non-characters. // Use our own version of \p{Cn} in order to avoid possible inconsistencies // between our data and whichever version of PCRE happens to be installed // on this server. Unlike \p{Cc} and \p{Co}, which never change, the value // of \p{Cn} changes with every new version of Unicode. $disallowed[] = '[\p{Co}' . $prop_classes['Cn'] . ']'; // Several more things we never want: $disallowed[] = '[' . implode('', array( // Soft Hyphen. '\x{AD}', // Khmer Vowel Inherent AQ and Khmer Vowel Inherent AA. // Unicode Standard ch. 16 says: "they are insufficient for [their] // purpose and should be considered errors in the encoding." '\x{17B4}-\x{17B5}', // Invisible math characters. '\x{2061}-\x{2064}', // Deprecated formatting characters. '\x{206A}-\x{206F}', // Zero Width No-Break Space, a.k.a. Byte Order Mark. '\x{FEFF}', // Annotation characters and Object Replacement Character. '\x{FFF9}-\x{FFFC}', )) . ']'; switch ($level) { case 2: $disallowed[] = '[' . implode('', array( // Combining Grapheme Character. '\x{34F}', // Zero Width Non-Joiner. '\x{200C}', // Zero Width Joiner. '\x{200D}', // All variation selectors. $prop_classes['Variation_Selector'], // Tag characters. '\x{E0000}-\x{E007F}', )) . ']'; // no break case 1: $disallowed[] = '[' . implode('', array( // Zero Width Space. '\x{200B}', // Word Joiner. '\x{2060}', // "Bidi_Control" characters. // Disallowing means that all characters will behave according // to their default bidirectional text properties. $prop_classes['Bidi_Control'], // Hangul filler characters. // Used as placeholders in incomplete ideographs. '\x{115F}\x{1160}\x{3164}\x{FFA0}', // Shorthand formatting characters. '\x{1BCA0}-\x{1BCA3}', // Musical formatting characters. '\x{1D173}-\x{1D17A}', )) . ']'; break; default: // Zero Width Space only allowed in certain scripts. $disallowed[] = '(?<![\p{Thai}\p{Myanmar}\p{Khmer}\p{Hiragana}\p{Katakana}])\x{200B}'; // Word Joiner disallowed inside words. (Yes, \w is Unicode safe.) $disallowed[] = '(?<=\w)\x{2060}(?=\w)'; // Hangul Choseong Filler and Hangul Jungseong Filler must followed // by more Hangul Jamo characters. $disallowed[] = '[\x{115F}\x{1160}](?![\x{1100}-\x{11FF}\x{A960}-\x{A97F}\x{D7B0}-\x{D7FF}])'; // Hangul Filler for Hangul compatibility chars. $disallowed[] = '\x{3164}(?![\x{3130}-\x{318F}])'; // Halfwidth Hangul Filler for halfwidth Hangul compatibility chars. $disallowed[] = '\x{FFA0}(?![\x{FFA1}-\x{FFDC}])'; // Shorthand formatting characters only with other shorthand chars. $disallowed[] = '[\x{1BCA0}-\x{1BCA3}](?![\x{1BC00}-\x{1BC9F}])'; $disallowed[] = '(?<![\x{1BC00}-\x{1BC9F}])[\x{1BCA0}-\x{1BCA3}]'; // Musical formatting characters only with other musical chars. $disallowed[] = '[\x{1D173}\x{1D175}\x{1D177}\x{1D179}](?![\x{1D100}-\x{1D1FF}])'; $disallowed[] = '(?<![\x{1D100}-\x{1D1FF}])[\x{1D174}\x{1D176}\x{1D178}\x{1D17A}]'; break; } if ($level < 2) { /* Combining Grapheme Character has two uses: to override standard search and collation behaviours, which we never want to allow, and to ensure correct behaviour of combining marks in a few exceptional cases, which is legitimate and should be allowed. This means we can simply test whether it is followed by a combining mark in order to determine whether to allow it. */ $disallowed[] = '\x{34F}(?!\p{M})'; // Tag characters not allowed inside words. $disallowed[] = '(?<=\w)[\x{E0000}-\x{E007F}](?=\w)'; } $string = preg_replace('/' . implode('|', $disallowed) . '/u', $substitute, $string); // Are we done yet? if (!preg_match('/[' . $prop_classes['Join_Control'] . $prop_classes['Regional_Indicator'] . $prop_classes['Emoji'] . $prop_classes['Variation_Selector'] . ']/u', $string)) return $string; // String must be in Normalization Form C for the following checks to work. $string = utf8_normalize_c($string); $placeholders = array(); // Use placeholders to preserve known emoji from further processing. // Regex source is https://unicode.org/reports/tr51/#EBNF_and_Regex $string = preg_replace_callback( '/' . // Flag emojis '[' . $prop_classes['Regional_Indicator'] . ']{2}' . // Or '|' . // Emoji characters '[' . $prop_classes['Emoji'] . ']' . // Possibly followed by modifiers of various sorts '(' . '[' . $prop_classes['Emoji_Modifier'] . ']' . '|' . '\x{FE0F}\x{20E3}?' . '|' . '[\x{E0020}-\x{E007E}]+\x{E007F}' . ')?' . // Possibly concatenated with Zero Width Joiner and more emojis // (e.g. the "family" emoji sequences) '(' . '\x{200D}[' . $prop_classes['Emoji'] . ']' . '(' . '[' . $prop_classes['Emoji_Modifier'] . ']' . '|' . '\x{FE0F}\x{20E3}?' . '|' . '[\x{E0020}-\x{E007E}]+\x{E007F}' . ')?' . ')*' . '/u', function ($matches) use (&$placeholders) { // Skip lone ASCII characters that are not actully part of an emoji sequence. // This can happen because the digits 0-9 and the '*' and '#' characters are // the base characters for the "Emoji_Keycap_Sequence" emojis. if (strlen($matches[0]) === 1) return $matches[0]; $placeholders[$matches[0]] = "\xEE\xB3\x9B" . md5($matches[0]) . "\xEE\xB3\x9C"; return $placeholders[$matches[0]]; }, $string ); // Get rid of any unsanctioned variation selectors. if (preg_match('/[' . $prop_classes['Variation_Selector'] . ']/u', $string)) { /* Unicode gives pre-defined lists of sanctioned variation sequences and says any use of variation selectors outside those sequences is unsanctioned. */ $patterns = array('/[' . $prop_classes['Ideographic'] . ']\K[\x{E0100}-\x{E01EF}]/u'); foreach (utf8_regex_variation_selectors() as $variation_selector => $allowed_base_chars) $patterns[] = '/[' . $allowed_base_chars . ']\K[' . $variation_selector . ']/u'; // Use placeholders for sanctioned variation selectors. $string = preg_replace_callback( $patterns, function ($matches) use (&$placeholders) { $placeholders[$matches[0]] = "\xEE\xB3\x9B" . md5($matches[0]) . "\xEE\xB3\x9C"; return $placeholders[$matches[0]]; }, $string ); // Remove any unsanctioned variation selectors. $string = preg_replace('/[' . $prop_classes['Variation_Selector'] . ']/u', $substitute, $string); } // Join controls are only allowed inside words in special circumstances. // See https://unicode.org/reports/tr31/#Layout_and_Format_Control_Characters if (preg_match('/[' . $prop_classes['Join_Control'] . ']/u', $string)) { // Zero Width Non-Joiner (U+200C) $zwnj = "\xE2\x80\x8C"; // Zero Width Joiner (U+200D) $zwj = "\xE2\x80\x8D"; $placeholders[$zwnj] = "\xEE\x80\x8C"; $placeholders[$zwj] = "\xEE\x80\x8D"; // When not in strict mode, allow ZWJ at word boundaries. if ($level === 0) $string = preg_replace('/\b\x{200D}|\x{200D}\b/u', $placeholders[$zwj], $string); // Tests for Zero Width Joiner and Zero Width Non-Joiner. $joining_type_classes = utf8_regex_joining_type(); $indic_classes = utf8_regex_indic(); foreach (array_merge($joining_type_classes, $indic_classes) as $script => $classes) { // Cursive scripts like Arabic use ZWNJ in certain contexts. // For these scripts, use test A1 for allowing ZWNJ. // https://unicode.org/reports/tr31/#A1 if (isset($joining_type_classes[$script])) { $lj = !empty($classes['Left_Joining']) ? $classes['Left_Joining'] : ''; $rj = !empty($classes['Right_Joining']) ? $classes['Right_Joining'] : ''; $t = !empty($classes['Transparent']) ? '[' . $classes['Transparent'] . ']*' : ''; if (!empty($classes['Dual_Joining'])) { $lj .= $classes['Dual_Joining']; $rj .= $classes['Dual_Joining']; } $pattern = '[' . $lj . ']' . $t . $zwnj . $t . '[' . $rj . ']'; } // Indic scripts with viramas use ZWNJ and ZWJ in certain contexts. // For these scripts, use tests A2 and B for allowing ZWNJ and ZWJ. // https://unicode.org/reports/tr31/#A2 // https://unicode.org/reports/tr31/#B else { // A letter that is part of this particular script. $letter = '[' . $classes['Letter'] . ']'; // Zero or more non-spacing marks used in this script. $nonspacing_marks = '[' . $classes['Nonspacing_Mark'] . ']*'; // Zero or more non-spacing combining marks used in this script. $nonspacing_combining_marks = '[' . $classes['Nonspacing_Combining_Mark'] . ']*'; // ZWNJ must be followed by another letter in the same script. $zwnj_pattern = '\x{200C}(?=' . $nonspacing_combining_marks . $letter . ')'; // ZWJ must NOT be followed by a vowel dependent character in this // script or by any character from a different script. $zwj_pattern = '\x{200D}(?!' . (!empty($classes['Vowel_Dependent']) ? '[' . $classes['Vowel_Dependent'] . ']|' : '') . '[^' . $classes['All'] . '])'; // Now build the pattern for this script. $pattern = $letter . $nonspacing_marks . '[' . $classes['Virama'] . ']' . $nonspacing_combining_marks . '\K' . (!empty($zwj_pattern) ? '(?:' . $zwj_pattern . '|' . $zwnj_pattern . ')' : $zwnj_pattern); } // Do the thing. $string = preg_replace_callback( '/' . $pattern . '/u', function ($matches) use ($placeholders) { return strtr($matches[0], $placeholders); }, $string ); // Did we catch 'em all? if (strpos($string, $zwnj) === false && strpos($string, $zwj) === false) break; } // Apart from the exceptions above, ZWNJ and ZWJ are not allowed. $string = str_replace(array($zwj, $zwnj), $substitute, $string); } // Revert placeholders back to original characters. $string = strtr($string, array_flip($placeholders)); return $string; } ?>