joomla/cms
29
0
Fork 0
mirror of https://github.com/joomla/joomla-cms.git synced 2024-08-26 13:09:46 +00:00
Code Packages Releases Activity

Merged JString updates from branch in order to support system without mb_string - Added PHPUTF8 library.

Browse Source 
git-svn-id: http://joomlacode.org/svn/joomla/development/trunk@13746 6f6e1ebd-4c2b-0410-823f-f34bde69bce9
This commit is contained in:
Ian MacLennan 2009-12-14 22:58:26 +00:00
parent b24f277554
commit aefdffe91e
28 changed files with 3374 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
installation/CHANGELOG
View File

@ -27,6 +27,10 @@ $ -> Language fix or change
- -> Removed
! -> Note
14-Dec-2009 Ian MacLennan
^ Modified JString to use phputf8 to support systems without mb_string
14-Dec-2009 Andrew Eddie
# Fixed bug in installation and diff SQL. TEXT fields can't have defaults.

264
libraries/joomla/utilities/string.php
View File

@ -7,7 +7,7 @@
* @license GNU General Public License version 2 or later; see LICENSE.txt
*/
// No direct access
// Check to ensure this file is within the rest of the framework
defined('JPATH_BASE') or die;
/**
@ -31,7 +31,7 @@ if (function_exists('iconv') || ((!strtoupper(substr(PHP_OS, 0, 3)) === 'WIN' &&
/**
* String handling class for utf-8 data
* Wraps the mb library
* Wraps the phputf8 library
* All functions assume the validity of utf-8 strings.
*
* @static
@ -55,14 +55,11 @@ abstract class JString
*/
public static function strpos($str, $search, $offset = FALSE)
{
if (strlen($str) && strlen($search))
{
if ($offset === FALSE) {
return mb_strpos($str, $search);
} else {
return mb_strpos($str, $search, $offset);
}
} else return FALSE;
if ( $offset === FALSE ) {
return utf8_strpos($str, $search);
} else {
return utf8_strpos($str, $search, $offset);
}
}
/**
@ -78,29 +75,7 @@ abstract class JString
*/
public static function strrpos($str, $search, $offset = false)
{
if ($offset === FALSE)
{
# Emulate behaviour of strrpos rather than raising warning
if (empty($str)) {
return FALSE;
}
return mb_strrpos($str, $search);
}
else
{
if (!is_int($offset)) {
trigger_error('strrpos expects parameter 3 to be long',E_USER_WARNING);
return FALSE;
}
$str = mb_substr($str, $offset);
if (FALSE !== ($pos = mb_strrpos($str, $search))) {
return $pos + $offset;
}
return FALSE;
}
return utf8_strrpos($str, $search);
}
/**
@ -118,9 +93,9 @@ abstract class JString
public static function substr($str, $offset, $length = FALSE)
{
if ($length === FALSE) {
return mb_substr($str, $offset);
return utf8_substr($str, $offset);
} else {
return mb_substr($str, $offset, $length);
return utf8_substr($str, $offset, $length);
}
}
@ -138,7 +113,7 @@ abstract class JString
* @see http://www.php.net/strtolower
*/
public static function strtolower($str){
return mb_strtolower($str);
return utf8_strtolower($str);
}
/**
@ -155,7 +130,7 @@ abstract class JString
* @see http://www.php.net/strtoupper
*/
public static function strtoupper($str){
return mb_strtoupper($str);
return utf8_strtoupper($str);
}
/**
@ -168,7 +143,7 @@ abstract class JString
* @see http://www.php.net/strlen
*/
public static function strlen($str){
return mb_strlen($str);
return utf8_strlen($str);
}
/**
@ -185,48 +160,11 @@ abstract class JString
*/
public static function str_ireplace($search, $replace, $str, $count = NULL)
{
if (!is_array($search))
{
$slen = strlen($search);
$lendif = strlen($replace) - $slen;
if ($slen == 0) {
return $str;
}
$search = JString::strtolower($search);
$search = preg_quote($search, '/');
$lstr = JString::strtolower($str);
$i = 0;
$matched = 0;
while (preg_match('/(.*)'.$search.'/Us',$lstr, $matches)) {
if ($i === $count) {
break;
}
$mlen = strlen($matches[0]);
$lstr = substr($lstr, $mlen);
$str = substr_replace($str, $replace, $matched+strlen($matches[1]), $slen);
$matched += $mlen + $lendif;
$i++;
}
return $str;
jimport('phputf8.str_ireplace');
if ( $count === FALSE ) {
return utf8_ireplace($search, $replace, $str);
} else {
foreach (array_keys($search) as $k)
{
if (is_array($replace))
{
if (array_key_exists($k,$replace)) {
$str = JString::str_ireplace($search[$k], $replace[$k], $str, $count);
} else {
$str = JString::str_ireplace($search[$k], '', $str, $count);
}
} else {
$str = JString::str_ireplace($search[$k], $replace, $str, $count);
}
}
return $str;
return utf8_ireplace($search, $replace, $str, $count);
}
}
@ -243,17 +181,8 @@ abstract class JString
*/
public static function str_split($str, $split_len = 1)
{
if (!preg_match('/^[0-9]+$/',$split_len) || $split_len < 1) {
return FALSE;
}
$len = JString::strlen($str);
if ($len <= $split_len) {
return array($str);
}
preg_match_all('/.{'.$split_len.'}|[^\x00]{1,'.$split_len.'}$/us', $str, $ar);
return $ar[0];
jimport('phputf8.str_split');
return utf8_str_split($str, $split_len);
}
/**
@ -269,9 +198,8 @@ abstract class JString
*/
public static function strcasecmp($str1, $str2)
{
$strX = JString::strtolower($strX);
$strY = JString::strtolower($strY);
return strcmp($strX, $strY);
jimport('phputf8.strcasecmp');
return utf8_strcasecmp($str1, $str2);
}
/**
@ -289,23 +217,14 @@ abstract class JString
*/
public static function strcspn($str, $mask, $start = NULL, $length = NULL)
{
if (empty($mask) || strlen($mask) == 0) {
return NULL;
jimport('phputf8.strcspn');
if ( $start === FALSE && $length === FALSE ) {
return utf8_strcspn($str, $mask);
} else if ( $length === FALSE ) {
return utf8_strcspn($str, $mask, $start);
} else {
return utf8_strcspn($str, $mask, $start, $length);
}
$mask = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$mask);
if ($start !== NULL || $length !== NULL) {
$str = JString::substr($str, $start, $length);
}
preg_match('/^[^'.$mask.']+/u',$str, $matches);
if (isset($matches[0])) {
return mb_strlen($matches[0]);
}
return 0;
}
/**
@ -323,19 +242,8 @@ abstract class JString
*/
public static function stristr($str, $search)
{
if (strlen($search) == 0) {
return $str;
}
$lstr = JString::strtolower($str);
$lsearch = JString::strtolower($search);
preg_match('|^(.*)'.preg_quote($lsearch).'|Us',$lstr, $matches);
if (count($matches) == 2) {
return substr($str, strlen($matches[1]));
}
return FALSE;
jimport('phputf8.stristr');
return utf8_stristr($str, $search);
}
/**
@ -350,8 +258,8 @@ abstract class JString
*/
public static function strrev($str)
{
preg_match_all('/./us', $str, $ar);
return join('',array_reverse($ar[0]));
jimport('phputf8.strrev');
return utf8_strrev($str);
}
/**
@ -368,19 +276,14 @@ abstract class JString
*/
public static function strspn($str, $mask, $start = NULL, $length = NULL)
{
$mask = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$mask);
if ($start !== NULL || $length !== NULL) {
$str = JString::substr($str, $start, $length);
jimport('phputf8.strspn');
if ( $start === FALSE && $length === FALSE ) {
return utf8_strspn($str, $mask);
} else if ( $length === FALSE ) {
return utf8_strspn($str, $mask, $start);
} else {
return utf8_strspn($str, $mask, $start, $length);
}
preg_match('/^['.$mask.']+/u',$str, $matches);
if (isset($matches[0])) {
return JString::strlen($matches[0]);
}
return 0;
}
/**
@ -397,13 +300,12 @@ abstract class JString
*/
public static function substr_replace($str, $repl, $start, $length = NULL)
{
preg_match_all('/./us', $str, $ar);
preg_match_all('/./us', $repl, $rar);
if ($length === NULL) {
$length = JString::strlen($str);
// loaded by library loader
if ( $length === FALSE ) {
return utf8_substr_replace($str, $repl, $start);
} else {
return utf8_substr_replace($str, $repl, $start, $length);
}
array_splice($ar[0], $start, $length, $rar[0]);
return join('',$ar[0]);
}
/**
@ -422,12 +324,12 @@ abstract class JString
*/
public static function ltrim($str, $charlist = FALSE)
{
if ($charlist === FALSE) return ltrim($str);
//quote charlist for use in a characterclass
$charlist = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$charlist);
return preg_replace('/^['.$charlist.']+/u','',$str);
jimport('phputf8.trim');
if ( $charlist === FALSE ) {
return utf8_ltrim( $str );
} else {
return utf8_ltrim( $str, $charlist );
}
}
/**
@ -446,14 +348,12 @@ abstract class JString
*/
public static function rtrim($str, $charlist = FALSE)
{
if ($charlist === FALSE) {
return rtrim($str);
}
//quote charlist for use in a characterclass
$charlist = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$charlist);
return preg_replace('/['.$charlist.']+$/u','',$str);
jimport('phputf8.trim');
if ( $charlist === FALSE ) {
return utf8_rtrim($str);
} else {
return utf8_rtrim( $str, $charlist );
}
}
/**
@ -472,11 +372,12 @@ abstract class JString
*/
public static function trim($str, $charlist = FALSE)
{
if ($charlist === FALSE) {
return trim($str);
jimport('phputf8.trim');
if ( $charlist === FALSE ) {
return utf8_trim( $str );
} else {
return utf8_trim( $str, $charlist );
}
return JString::ltrim(JString::rtrim($str, $charlist), $charlist);
}
/**
@ -491,18 +392,8 @@ abstract class JString
*/
public static function ucfirst($str)
{
switch (JString::strlen($str)) {
case 0:
return '';
break;
case 1:
return JString::strtoupper($str);
break;
default:
preg_match('/^(.{1})(.*)$/us', $str, $matches);
return JString::strtoupper($matches[1]).$matches[2];
break;
}
jimport('phputf8.ucfirst');
return utf8_ucfirst($str);
}
/**
@ -517,28 +408,8 @@ abstract class JString
*/
public static function ucwords($str)
{
// Note: [\x0c\x09\x0b\x0a\x0d\x20] matches;
// form feeds, horizontal tabs, vertical tabs, linefeeds and carriage returns
// This corresponds to the definition of a "word" defined at http://www.php.net/ucwords
$pattern = '/(^|([\x0c\x09\x0b\x0a\x0d\x20]+))([^\x0c\x09\x0b\x0a\x0d\x20]{1})[^\x0c\x09\x0b\x0a\x0d\x20]*/u';
return preg_replace_callback($pattern, 'JString::ucwords_callback',$str);
}
/**
* Callback function for preg_replace_callback call in JString::ucwords
* You don't need to call this yourself
* @param array of matches corresponding to a single word
* @return string with first char of the word in uppercase
* @see ucwords
* @see strtoupper
*/
public static function ucwords_callback($matches)
{
$leadingws = $matches[2];
$ucfirst = JString::strtoupper($matches[3]);
$ucword = JString::substr_replace(ltrim($matches[0]),$ucfirst,0,1);
return $leadingws . $ucword;
jimport('phputf8.ucwords');
return utf8_ucwords($str);
}
/**
@ -553,8 +424,7 @@ abstract class JString
*/
public static function transcode($source, $from_encoding, $to_encoding)
{
if (is_string($source))
{
if (is_string($source)) {
/*
* "//TRANSLIT" is appendd to the $to_encoding to ensure that when iconv comes
* across a character that cannot be represented in the target charset, it can
@ -571,6 +441,7 @@ abstract class JString
* @author <hsivonen@iki.fi>
* @param string UTF-8 encoded string
* @return boolean true if valid
* @since 1.6
* @see http://hsivonen.iki.fi/php-utf8/
* @see compliant
*/
@ -705,6 +576,7 @@ abstract class JString
* @see http://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
* @param string UTF-8 string to check
* @return boolean TRUE if string is valid UTF-8
* @since 1.6
*/
public static function compliant($str)
{

504
libraries/phputf8/LICENSE Normal file
View File

@ -0,0 +1,504 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 2.1, February 1999
Copyright (C) 1991, 1999 Free Software Foundation, Inc.
51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
[This is the first released version of the Lesser GPL. It also counts
as the successor of the GNU Library Public License, version 2, hence
the version number 2.1.]
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
Licenses are intended to guarantee your freedom to share and change
free software--to make sure the software is free for all its users.
This license, the Lesser General Public License, applies to some
specially designated software packages--typically libraries--of the
Free Software Foundation and other authors who decide to use it. You
can use it too, but we suggest you first think carefully about whether
this license or the ordinary General Public License is the better
strategy to use in any particular case, based on the explanations below.
When we speak of free software, we are referring to freedom of use,
not price. Our General Public Licenses are designed to make sure that
you have the freedom to distribute copies of free software (and charge
for this service if you wish); that you receive source code or can get
it if you want it; that you can change the software and use pieces of
it in new free programs; and that you are informed that you can do
these things.
To protect your rights, we need to make restrictions that forbid
distributors to deny you these rights or to ask you to surrender these
rights. These restrictions translate to certain responsibilities for
you if you distribute copies of the library or if you modify it.
For example, if you distribute copies of the library, whether gratis
or for a fee, you must give the recipients all the rights that we gave
you. You must make sure that they, too, receive or can get the source
code. If you link other code with the library, you must provide
complete object files to the recipients, so that they can relink them
with the library after making changes to the library and recompiling
it. And you must show them these terms so they know their rights.
We protect your rights with a two-step method: (1) we copyright the
library, and (2) we offer you this license, which gives you legal
permission to copy, distribute and/or modify the library.
To protect each distributor, we want to make it very clear that
there is no warranty for the free library. Also, if the library is
modified by someone else and passed on, the recipients should know
that what they have is not the original version, so that the original
author's reputation will not be affected by problems that might be
introduced by others.
Finally, software patents pose a constant threat to the existence of
any free program. We wish to make sure that a company cannot
effectively restrict the users of a free program by obtaining a
restrictive license from a patent holder. Therefore, we insist that
any patent license obtained for a version of the library must be
consistent with the full freedom of use specified in this license.
Most GNU software, including some libraries, is covered by the
ordinary GNU General Public License. This license, the GNU Lesser
General Public License, applies to certain designated libraries, and
is quite different from the ordinary General Public License. We use
this license for certain libraries in order to permit linking those
libraries into non-free programs.
When a program is linked with a library, whether statically or using
a shared library, the combination of the two is legally speaking a
combined work, a derivative of the original library. The ordinary
General Public License therefore permits such linking only if the
entire combination fits its criteria of freedom. The Lesser General
Public License permits more lax criteria for linking other code with
the library.
We call this license the "Lesser" General Public License because it
does Less to protect the user's freedom than the ordinary General
Public License. It also provides other free software developers Less
of an advantage over competing non-free programs. These disadvantages
are the reason we use the ordinary General Public License for many
libraries. However, the Lesser license provides advantages in certain
special circumstances.
For example, on rare occasions, there may be a special need to
encourage the widest possible use of a certain library, so that it becomes
a de-facto standard. To achieve this, non-free programs must be
allowed to use the library. A more frequent case is that a free
library does the same job as widely used non-free libraries. In this
case, there is little to gain by limiting the free library to free
software only, so we use the Lesser General Public License.
In other cases, permission to use a particular library in non-free
programs enables a greater number of people to use a large body of
free software. For example, permission to use the GNU C Library in
non-free programs enables many more people to use the whole GNU
operating system, as well as its variant, the GNU/Linux operating
system.
Although the Lesser General Public License is Less protective of the
users' freedom, it does ensure that the user of a program that is
linked with the Library has the freedom and the wherewithal to run
that program using a modified version of the Library.
The precise terms and conditions for copying, distribution and
modification follow. Pay close attention to the difference between a
"work based on the library" and a "work that uses the library". The
former contains code derived from the library, whereas the latter must
be combined with the library in order to run.
GNU LESSER GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License Agreement applies to any software library or other
program which contains a notice placed by the copyright holder or
other authorized party saying it may be distributed under the terms of
this Lesser General Public License (also called "this License").
Each licensee is addressed as "you".
A "library" means a collection of software functions and/or data
prepared so as to be conveniently linked with application programs
(which use some of those functions and data) to form executables.
The "Library", below, refers to any such software library or work
which has been distributed under these terms. A "work based on the
Library" means either the Library or any derivative work under
copyright law: that is to say, a work containing the Library or a
portion of it, either verbatim or with modifications and/or translated
straightforwardly into another language. (Hereinafter, translation is
included without limitation in the term "modification".)
"Source code" for a work means the preferred form of the work for
making modifications to it. For a library, complete source code means
all the source code for all modules it contains, plus any associated
interface definition files, plus the scripts used to control compilation
and installation of the library.
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running a program using the Library is not restricted, and output from
such a program is covered only if its contents constitute a work based
on the Library (independent of the use of the Library in a tool for
writing it). Whether that is true depends on what the Library does
and what the program that uses the Library does.
1. You may copy and distribute verbatim copies of the Library's
complete source code as you receive it, in any medium, provided that
you conspicuously and appropriately publish on each copy an
appropriate copyright notice and disclaimer of warranty; keep intact
all the notices that refer to this License and to the absence of any
warranty; and distribute a copy of this License along with the
Library.
You may charge a fee for the physical act of transferring a copy,
and you may at your option offer warranty protection in exchange for a
fee.
2. You may modify your copy or copies of the Library or any portion
of it, thus forming a work based on the Library, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) The modified work must itself be a software library.
b) You must cause the files modified to carry prominent notices
stating that you changed the files and the date of any change.
c) You must cause the whole of the work to be licensed at no
charge to all third parties under the terms of this License.
d) If a facility in the modified Library refers to a function or a
table of data to be supplied by an application program that uses
the facility, other than as an argument passed when the facility
is invoked, then you must make a good faith effort to ensure that,
in the event an application does not supply such function or
table, the facility still operates, and performs whatever part of
its purpose remains meaningful.
(For example, a function in a library to compute square roots has
a purpose that is entirely well-defined independent of the
application. Therefore, Subsection 2d requires that any
application-supplied function or table used by this function must
be optional: if the application does not supply it, the square
root function must still compute square roots.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Library,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Library, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote
it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Library.
In addition, mere aggregation of another work not based on the Library
with the Library (or with a work based on the Library) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may opt to apply the terms of the ordinary GNU General Public
License instead of this License to a given copy of the Library. To do
this, you must alter all the notices that refer to this License, so
that they refer to the ordinary GNU General Public License, version 2,
instead of to this License. (If a newer version than version 2 of the
ordinary GNU General Public License has appeared, then you can specify
that version instead if you wish.) Do not make any other change in
these notices.
Once this change is made in a given copy, it is irreversible for
that copy, so the ordinary GNU General Public License applies to all
subsequent copies and derivative works made from that copy.
This option is useful when you wish to copy part of the code of
the Library into a program that is not a library.
4. You may copy and distribute the Library (or a portion or
derivative of it, under Section 2) in object code or executable form
under the terms of Sections 1 and 2 above provided that you accompany
it with the complete corresponding machine-readable source code, which
must be distributed under the terms of Sections 1 and 2 above on a
medium customarily used for software interchange.
If distribution of object code is made by offering access to copy
from a designated place, then offering equivalent access to copy the
source code from the same place satisfies the requirement to
distribute the source code, even though third parties are not
compelled to copy the source along with the object code.
5. A program that contains no derivative of any portion of the
Library, but is designed to work with the Library by being compiled or
linked with it, is called a "work that uses the Library". Such a
work, in isolation, is not a derivative work of the Library, and
therefore falls outside the scope of this License.
However, linking a "work that uses the Library" with the Library
creates an executable that is a derivative of the Library (because it
contains portions of the Library), rather than a "work that uses the
library". The executable is therefore covered by this License.
Section 6 states terms for distribution of such executables.
When a "work that uses the Library" uses material from a header file
that is part of the Library, the object code for the work may be a
derivative work of the Library even though the source code is not.
Whether this is true is especially significant if the work can be
linked without the Library, or if the work is itself a library. The
threshold for this to be true is not precisely defined by law.
If such an object file uses only numerical parameters, data
structure layouts and accessors, and small macros and small inline
functions (ten lines or less in length), then the use of the object
file is unrestricted, regardless of whether it is legally a derivative
work. (Executables containing this object code plus portions of the
Library will still fall under Section 6.)
Otherwise, if the work is a derivative of the Library, you may
distribute the object code for the work under the terms of Section 6.
Any executables containing that work also fall under Section 6,
whether or not they are linked directly with the Library itself.
6. As an exception to the Sections above, you may also combine or
link a "work that uses the Library" with the Library to produce a
work containing portions of the Library, and distribute that work
under terms of your choice, provided that the terms permit
modification of the work for the customer's own use and reverse
engineering for debugging such modifications.
You must give prominent notice with each copy of the work that the
Library is used in it and that the Library and its use are covered by
this License. You must supply a copy of this License. If the work
during execution displays copyright notices, you must include the
copyright notice for the Library among them, as well as a reference
directing the user to the copy of this License. Also, you must do one
of these things:
a) Accompany the work with the complete corresponding
machine-readable source code for the Library including whatever
changes were used in the work (which must be distributed under
Sections 1 and 2 above); and, if the work is an executable linked
with the Library, with the complete machine-readable "work that
uses the Library", as object code and/or source code, so that the
user can modify the Library and then relink to produce a modified
executable containing the modified Library. (It is understood
that the user who changes the contents of definitions files in the
Library will not necessarily be able to recompile the application
to use the modified definitions.)
b) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (1) uses at run time a
copy of the library already present on the user's computer system,
rather than copying library functions into the executable, and (2)
will operate properly with a modified version of the library, if
the user installs one, as long as the modified version is
interface-compatible with the version that the work was made with.
c) Accompany the work with a written offer, valid for at
least three years, to give the same user the materials
specified in Subsection 6a, above, for a charge no more
than the cost of performing this distribution.
d) If distribution of the work is made by offering access to copy
from a designated place, offer equivalent access to copy the above
specified materials from the same place.
e) Verify that the user has already received a copy of these
materials or that you have already sent this user a copy.
For an executable, the required form of the "work that uses the
Library" must include any data and utility programs needed for
reproducing the executable from it. However, as a special exception,
the materials to be distributed need not include anything that is
normally distributed (in either source or binary form) with the major
components (compiler, kernel, and so on) of the operating system on
which the executable runs, unless that component itself accompanies
the executable.
It may happen that this requirement contradicts the license
restrictions of other proprietary libraries that do not normally
accompany the operating system. Such a contradiction means you cannot
use both them and the Library together in an executable that you
distribute.
7. You may place library facilities that are a work based on the
Library side-by-side in a single library together with other library
facilities not covered by this License, and distribute such a combined
library, provided that the separate distribution of the work based on
the Library and of the other library facilities is otherwise
permitted, and provided that you do these two things:
a) Accompany the combined library with a copy of the same work
based on the Library, uncombined with any other library
facilities. This must be distributed under the terms of the
Sections above.
b) Give prominent notice with the combined library of the fact
that part of it is a work based on the Library, and explaining
where to find the accompanying uncombined form of the same work.
8. You may not copy, modify, sublicense, link with, or distribute
the Library except as expressly provided under this License. Any
attempt otherwise to copy, modify, sublicense, link with, or
distribute the Library is void, and will automatically terminate your
rights under this License. However, parties who have received copies,
or rights, from you under this License will not have their licenses
terminated so long as such parties remain in full compliance.
9. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Library or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Library (or any work based on the
Library), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Library or works based on it.
10. Each time you redistribute the Library (or any work based on the
Library), the recipient automatically receives a license from the
original licensor to copy, distribute, link with or modify the Library
subject to these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties with
this License.
11. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Library at all. For example, if a patent
license would not permit royalty-free redistribution of the Library by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Library.
If any portion of this section is held invalid or unenforceable under any
particular circumstance, the balance of the section is intended to apply,
and the section as a whole is intended to apply in other circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
12. If the distribution and/or use of the Library is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Library under this License may add
an explicit geographical distribution limitation excluding those countries,
so that distribution is permitted only in or among countries not thus
excluded. In such case, this License incorporates the limitation as if
written in the body of this License.
13. The Free Software Foundation may publish revised and/or new
versions of the Lesser General Public License from time to time.
Such new versions will be similar in spirit to the present version,
but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Library
specifies a version number of this License which applies to it and
"any later version", you have the option of following the terms and
conditions either of that version or of any later version published by
the Free Software Foundation. If the Library does not specify a
license version number, you may choose any version ever published by
the Free Software Foundation.
14. If you wish to incorporate parts of the Library into other free
programs whose distribution conditions are incompatible with these,
write to the author to ask for permission. For software which is
copyrighted by the Free Software Foundation, write to the Free
Software Foundation; we sometimes make exceptions for this. Our
decision will be guided by the two goals of preserving the free status
of all derivatives of our free software and of promoting the sharing
and reuse of software generally.
NO WARRANTY
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Libraries
If you develop a new library, and you want it to be of the greatest
possible use to the public, we recommend making it free software that
everyone can redistribute and change. You can do so by permitting
redistribution under these terms (or, alternatively, under the terms of the
ordinary General Public License).
To apply these terms, attach the following notices to the library. It is
safest to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least the
"copyright" line and a pointer to where the full notice is found.
<one line to give the library's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Also add information on how to contact you by electronic and paper mail.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the library, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the
library `Frob' (a library for tweaking knobs) written by James Random Hacker.
<signature of Ty Coon>, 1 April 1990
Ty Coon, President of Vice
That's all there is to it!

82
libraries/phputf8/README Normal file
View File

@ -0,0 +1,82 @@
++PHP UTF-8++
Version 0.5
++DOCUMENTATION++
Documentation in progress in ./docs dir
http://www.phpwact.org/php/i18n/charsets
http://www.phpwact.org/php/i18n/utf-8
Important Note: DO NOT use these functions without understanding WHY
you are using them. In particular, do not blindly replace all use of PHP's
string functions which functions found here - most of the time you will
not need to, and you will be introducing a significant performance
overhead to your application. You can get a good idea of when to use what
from reading: http://www.phpwact.org/php/i18n/utf-8
Important Note: For sake of performance most of the functions here are
not "defensive" (e.g. there is not extensive parameter checking, well
formed UTF-8 is assumed). This is particularily relevant when is comes to
catching badly formed UTF-8 - you should screen input on the "outer
perimeter" with help from functions in the utf8_validation.php and
utf8_bad.php files.
Important Note: this library treats ALL ASCII characters as valid, including ASCII control characters. But if you use some ASCII control characters in XML, it will render the XML ill-formed. Don't be a bozo: http://hsivonen.iki.fi/producing-xml/#controlchar
++BUGS / SUPPORT / FEATURE REQUESTS ++
Please report bugs to:
http://sourceforge.net/tracker/?group_id=142846&atid=753842
- if you are able, please submit a failing unit test
(http://www.lastcraft.com/simple_test.php) with your bug report.
For feature requests / faster implementation of functions found here,
please drop them in via the RFE tracker: http://sourceforge.net/tracker/?group_id=142846&atid=753845
Particularily interested in faster implementations!
For general support / help, use:
http://sourceforge.net/tracker/?group_id=142846&atid=753843
In the VERY WORST case, you can email me: hfuecks gmail com - I tend to be slow to respond though so be warned.
Important Note: when reporting bugs, please provide the following
information;
PHP version, whether the iconv extension is loaded (in PHP5 it's
there by default), whether the mbstring extension is loaded. The
following PHP script can be used to determine this information;
<?php
print "PHP Version: " .phpversion()."<br>";
if ( extension_loaded('mbstring') ) {
print "mbstring available<br>";
} else {
print "mbstring not available<br>";
}
if ( extension_loaded('iconv') ) {
print "iconv available<br>";
} else {
print "iconv not available<br>";
}
?>
++LICENSING++
Parts of the code in this library come from other places, under different
licenses.
The authors involved have been contacted (see below). Attribution for
which code came from elsewhere can be found in the source code itself.
+Andreas Gohr / Chris Smith - Dokuwiki
There is a fair degree of collaboration / exchange of ideas and code
beteen Dokuwiki's UTF-8 library;
http://dev.splitbrain.org/view/darcs/dokuwiki/inc/utf8.php
and phputf8. Although Dokuwiki is released under GPL, its UTF-8
library is released under LGPL, hence no conflict with phputf8
+Henri Sivonen (http://hsivonen.iki.fi/php-utf8/ /
http://hsivonen.iki.fi/php-utf8/) has also given permission for his
code to be released under the terms of the LGPL. He ported a Unicode / UTF-8
converter from the Mozilla codebase to PHP, which is re-used in phputf8

1
libraries/phputf8/index.html Normal file
View File

@ -0,0 +1 @@
<html><body></body></html>

140
libraries/phputf8/mbstring/core.php Normal file
View File

@ -0,0 +1,140 @@
<?php
/**
* @version $Id: core.php,v 1.5 2006/02/28 22:12:25 harryf Exp $
* @package utf8
* @subpackage strings
*/
/**
* Define UTF8_CORE as required
*/
if ( !defined('UTF8_CORE') ) {
define('UTF8_CORE',TRUE);
}
//--------------------------------------------------------------------
/**
* Wrapper round mb_strlen
* Assumes you have mb_internal_encoding to UTF-8 already
* Note: this function does not count bad bytes in the string - these
* are simply ignored
* @param string UTF-8 string
* @return int number of UTF-8 characters in string
* @package utf8
* @subpackage strings
*/
function utf8_strlen($str){
return mb_strlen($str);
}
//--------------------------------------------------------------------
/**
* Assumes mbstring internal encoding is set to UTF-8
* Wrapper around mb_strpos
* Find position of first occurrence of a string
* @param string haystack
* @param string needle (you should validate this with utf8_is_valid)
* @param integer offset in characters (from left)
* @return mixed integer position or FALSE on failure
* @package utf8
* @subpackage strings
*/
function utf8_strpos($str, $search, $offset = FALSE){
if ( $offset === FALSE ) {
return mb_strpos($str, $search);
} else {
return mb_strpos($str, $search, $offset);
}
}
//--------------------------------------------------------------------
/**
* Assumes mbstring internal encoding is set to UTF-8
* Wrapper around mb_strrpos
* Find position of last occurrence of a char in a string
* @param string haystack
* @param string needle (you should validate this with utf8_is_valid)
* @param integer (optional) offset (from left)
* @return mixed integer position or FALSE on failure
* @package utf8
* @subpackage strings
*/
function utf8_strrpos($str, $search, $offset = FALSE){
if ( $offset === FALSE ) {
# Emulate behaviour of strrpos rather than raising warning
if ( empty($str) ) {
return FALSE;
}
return mb_strrpos($str, $search);
} else {
if ( !is_int($offset) ) {
trigger_error('utf8_strrpos expects parameter 3 to be long',E_USER_WARNING);
return FALSE;
}
$str = mb_substr($str, $offset);
if ( FALSE !== ( $pos = mb_strrpos($str, $search) ) ) {
return $pos + $offset;
}
return FALSE;
}
}
//--------------------------------------------------------------------
/**
* Assumes mbstring internal encoding is set to UTF-8
* Wrapper around mb_substr
* Return part of a string given character offset (and optionally length)
* @param string
* @param integer number of UTF-8 characters offset (from left)
* @param integer (optional) length in UTF-8 characters from offset
* @return mixed string or FALSE if failure
* @package utf8
* @subpackage strings
*/
function utf8_substr($str, $offset, $length = FALSE){
if ( $length === FALSE ) {
return mb_substr($str, $offset);
} else {
return mb_substr($str, $offset, $length);
}
}
//--------------------------------------------------------------------
/**
* Assumes mbstring internal encoding is set to UTF-8
* Wrapper around mb_strtolower
* Make a string lowercase
* Note: The concept of a characters "case" only exists is some alphabets
* such as Latin, Greek, Cyrillic, Armenian and archaic Georgian - it does
* not exist in the Chinese alphabet, for example. See Unicode Standard
* Annex #21: Case Mappings
* @param string
* @return mixed either string in lowercase or FALSE is UTF-8 invalid
* @package utf8
* @subpackage strings
*/
function utf8_strtolower($str){
return mb_strtolower($str);
}
//--------------------------------------------------------------------
/**
* Assumes mbstring internal encoding is set to UTF-8
* Wrapper around mb_strtoupper
* Make a string uppercase
* Note: The concept of a characters "case" only exists is some alphabets
* such as Latin, Greek, Cyrillic, Armenian and archaic Georgian - it does
* not exist in the Chinese alphabet, for example. See Unicode Standard
* Annex #21: Case Mappings
* @param string
* @return mixed either string in lowercase or FALSE is UTF-8 invalid
* @package utf8
* @subpackage strings
*/
function utf8_strtoupper($str){
return mb_strtoupper($str);
}

438
libraries/phputf8/native/core.php Normal file
View File

@ -0,0 +1,438 @@
<?php
/**
* @version $Id: core.php,v 1.9 2007/08/12 01:11:33 harryf Exp $
* @package utf8
* @subpackage strings
*/
/**
* Define UTF8_CORE as required
*/
if ( !defined('UTF8_CORE') ) {
define('UTF8_CORE',TRUE);
}
//--------------------------------------------------------------------
/**
* Unicode aware replacement for strlen(). Returns the number
* of characters in the string (not the number of bytes), replacing
* multibyte characters with a single byte equivalent
* utf8_decode() converts characters that are not in ISO-8859-1
* to '?', which, for the purpose of counting, is alright - It's
* much faster than iconv_strlen
* Note: this function does not count bad UTF-8 bytes in the string
* - these are simply ignored
* @author <chernyshevsky at hotmail dot com>
* @link http://www.php.net/manual/en/function.strlen.php
* @link http://www.php.net/manual/en/function.utf8-decode.php
* @param string UTF-8 string
* @return int number of UTF-8 characters in string
* @package utf8
* @subpackage strings
*/
function utf8_strlen($str){
return strlen(utf8_decode($str));
}
//--------------------------------------------------------------------
/**
* UTF-8 aware alternative to strpos
* Find position of first occurrence of a string
* Note: This will get alot slower if offset is used
* Note: requires utf8_strlen amd utf8_substr to be loaded
* @param string haystack
* @param string needle (you should validate this with utf8_is_valid)
* @param integer offset in characters (from left)
* @return mixed integer position or FALSE on failure
* @see http://www.php.net/strpos
* @see utf8_strlen
* @see utf8_substr
* @package utf8
* @subpackage strings
*/
function utf8_strpos($str, $needle, $offset = NULL) {
if ( is_null($offset) ) {
$ar = explode($needle, $str, 2);
if ( count($ar) > 1 ) {
return utf8_strlen($ar[0]);
}
return FALSE;
} else {
if ( !is_int($offset) ) {
trigger_error('utf8_strpos: Offset must be an integer',E_USER_ERROR);
return FALSE;
}
$str = utf8_substr($str, $offset);
if ( FALSE !== ( $pos = utf8_strpos($str, $needle) ) ) {
return $pos + $offset;
}
return FALSE;
}
}
//--------------------------------------------------------------------
/**
* UTF-8 aware alternative to strrpos
* Find position of last occurrence of a char in a string
* Note: This will get alot slower if offset is used
* Note: requires utf8_substr and utf8_strlen to be loaded
* @param string haystack
* @param string needle (you should validate this with utf8_is_valid)
* @param integer (optional) offset (from left)
* @return mixed integer position or FALSE on failure
* @see http://www.php.net/strrpos
* @see utf8_substr
* @see utf8_strlen
* @package utf8
* @subpackage strings
*/
function utf8_strrpos($str, $needle, $offset = NULL) {
if ( is_null($offset) ) {
$ar = explode($needle, $str);
if ( count($ar) > 1 ) {
// Pop off the end of the string where the last match was made
array_pop($ar);
$str = join($needle,$ar);
return utf8_strlen($str);
}
return FALSE;
} else {
if ( !is_int($offset) ) {
trigger_error('utf8_strrpos expects parameter 3 to be long',E_USER_WARNING);
return FALSE;
}
$str = utf8_substr($str, $offset);
if ( FALSE !== ( $pos = utf8_strrpos($str, $needle) ) ) {
return $pos + $offset;
}
return FALSE;
}
}
//--------------------------------------------------------------------
/**
* UTF-8 aware alternative to substr
* Return part of a string given character offset (and optionally length)
*
* Note arguments: comparied to substr - if offset or length are
* not integers, this version will not complain but rather massages them
* into an integer.
*
* Note on returned values: substr documentation states false can be
* returned in some cases (e.g. offset > string length)
* mb_substr never returns false, it will return an empty string instead.
* This adopts the mb_substr approach
*
* Note on implementation: PCRE only supports repetitions of less than
* 65536, in order to accept up to MAXINT values for offset and length,
* we'll repeat a group of 65535 characters when needed.
*
* Note on implementation: calculating the number of characters in the
* string is a relatively expensive operation, so we only carry it out when
* necessary. It isn't necessary for +ve offsets and no specified length
*
* @author Chris Smith<chris@jalakai.co.uk>
* @param string
* @param integer number of UTF-8 characters offset (from left)
* @param integer (optional) length in UTF-8 characters from offset
* @return mixed string or FALSE if failure
* @package utf8
* @subpackage strings
*/
function utf8_substr($str, $offset, $length = NULL) {
// generates E_NOTICE
// for PHP4 objects, but not PHP5 objects
$str = (string)$str;
$offset = (int)$offset;
if (!is_null($length)) $length = (int)$length;
// handle trivial cases
if ($length === 0) return '';
if ($offset < 0 && $length < 0 && $length < $offset)
return '';
// normalise negative offsets (we could use a tail
// anchored pattern, but they are horribly slow!)
if ($offset < 0) {
// see notes
$strlen = strlen(utf8_decode($str));
$offset = $strlen + $offset;
if ($offset < 0) $offset = 0;
}
$Op = '';
$Lp = '';
// establish a pattern for offset, a
// non-captured group equal in length to offset
if ($offset > 0) {
$Ox = (int)($offset/65535);
$Oy = $offset%65535;
if ($Ox) {
$Op = '(?:.{65535}){'.$Ox.'}';
}
$Op = '^(?:'.$Op.'.{'.$Oy.'})';
} else {
// offset == 0; just anchor the pattern
$Op = '^';
}
// establish a pattern for length
if (is_null($length)) {
// the rest of the string
$Lp = '(.*)$';
} else {
if (!isset($strlen)) {
// see notes
$strlen = strlen(utf8_decode($str));
}
// another trivial case
if ($offset > $strlen) return '';
if ($length > 0) {
// reduce any length that would
// go passed the end of the string
$length = min($strlen-$offset, $length);
$Lx = (int)( $length / 65535 );
$Ly = $length % 65535;
// negative length requires a captured group
// of length characters
if ($Lx) $Lp = '(?:.{65535}){'.$Lx.'}';
$Lp = '('.$Lp.'.{'.$Ly.'})';
} else if ($length < 0) {
if ( $length < ($offset - $strlen) ) {
return '';
}
$Lx = (int)((-$length)/65535);
$Ly = (-$length)%65535;
// negative length requires ... capture everything
// except a group of -length characters
// anchored at the tail-end of the string
if ($Lx) $Lp = '(?:.{65535}){'.$Lx.'}';
$Lp = '(.*)(?:'.$Lp.'.{'.$Ly.'})$';
}
}
if (!preg_match( '#'.$Op.$Lp.'#us',$str, $match )) {
return '';
}
return $match[1];
}
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to strtolower
* Make a string lowercase
* Note: The concept of a characters "case" only exists is some alphabets
* such as Latin, Greek, Cyrillic, Armenian and archaic Georgian - it does
* not exist in the Chinese alphabet, for example. See Unicode Standard
* Annex #21: Case Mappings
* Note: requires utf8_to_unicode and utf8_from_unicode
* @author Andreas Gohr <andi@splitbrain.org>
* @param string
* @return mixed either string in lowercase or FALSE is UTF-8 invalid
* @see http://www.php.net/strtolower
* @see utf8_to_unicode
* @see utf8_from_unicode
* @see http://www.unicode.org/reports/tr21/tr21-5.html
* @see http://dev.splitbrain.org/view/darcs/dokuwiki/inc/utf8.php
* @package utf8
* @subpackage strings
*/
function utf8_strtolower($string){
static $UTF8_UPPER_TO_LOWER = NULL;
if ( is_null($UTF8_UPPER_TO_LOWER) ) {
$UTF8_UPPER_TO_LOWER = array(
0x0041=>0x0061, 0x03A6=>0x03C6, 0x0162=>0x0163, 0x00C5=>0x00E5, 0x0042=>0x0062,
0x0139=>0x013A, 0x00C1=>0x00E1, 0x0141=>0x0142, 0x038E=>0x03CD, 0x0100=>0x0101,
0x0490=>0x0491, 0x0394=>0x03B4, 0x015A=>0x015B, 0x0044=>0x0064, 0x0393=>0x03B3,
0x00D4=>0x00F4, 0x042A=>0x044A, 0x0419=>0x0439, 0x0112=>0x0113, 0x041C=>0x043C,
0x015E=>0x015F, 0x0143=>0x0144, 0x00CE=>0x00EE, 0x040E=>0x045E, 0x042F=>0x044F,
0x039A=>0x03BA, 0x0154=>0x0155, 0x0049=>0x0069, 0x0053=>0x0073, 0x1E1E=>0x1E1F,
0x0134=>0x0135, 0x0427=>0x0447, 0x03A0=>0x03C0, 0x0418=>0x0438, 0x00D3=>0x00F3,
0x0420=>0x0440, 0x0404=>0x0454, 0x0415=>0x0435, 0x0429=>0x0449, 0x014A=>0x014B,
0x0411=>0x0431, 0x0409=>0x0459, 0x1E02=>0x1E03, 0x00D6=>0x00F6, 0x00D9=>0x00F9,
0x004E=>0x006E, 0x0401=>0x0451, 0x03A4=>0x03C4, 0x0423=>0x0443, 0x015C=>0x015D,
0x0403=>0x0453, 0x03A8=>0x03C8, 0x0158=>0x0159, 0x0047=>0x0067, 0x00C4=>0x00E4,
0x0386=>0x03AC, 0x0389=>0x03AE, 0x0166=>0x0167, 0x039E=>0x03BE, 0x0164=>0x0165,
0x0116=>0x0117, 0x0108=>0x0109, 0x0056=>0x0076, 0x00DE=>0x00FE, 0x0156=>0x0157,
0x00DA=>0x00FA, 0x1E60=>0x1E61, 0x1E82=>0x1E83, 0x00C2=>0x00E2, 0x0118=>0x0119,
0x0145=>0x0146, 0x0050=>0x0070, 0x0150=>0x0151, 0x042E=>0x044E, 0x0128=>0x0129,
0x03A7=>0x03C7, 0x013D=>0x013E, 0x0422=>0x0442, 0x005A=>0x007A, 0x0428=>0x0448,
0x03A1=>0x03C1, 0x1E80=>0x1E81, 0x016C=>0x016D, 0x00D5=>0x00F5, 0x0055=>0x0075,
0x0176=>0x0177, 0x00DC=>0x00FC, 0x1E56=>0x1E57, 0x03A3=>0x03C3, 0x041A=>0x043A,
0x004D=>0x006D, 0x016A=>0x016B, 0x0170=>0x0171, 0x0424=>0x0444, 0x00CC=>0x00EC,
0x0168=>0x0169, 0x039F=>0x03BF, 0x004B=>0x006B, 0x00D2=>0x00F2, 0x00C0=>0x00E0,
0x0414=>0x0434, 0x03A9=>0x03C9, 0x1E6A=>0x1E6B, 0x00C3=>0x00E3, 0x042D=>0x044D,
0x0416=>0x0436, 0x01A0=>0x01A1, 0x010C=>0x010D, 0x011C=>0x011D, 0x00D0=>0x00F0,
0x013B=>0x013C, 0x040F=>0x045F, 0x040A=>0x045A, 0x00C8=>0x00E8, 0x03A5=>0x03C5,
0x0046=>0x0066, 0x00DD=>0x00FD, 0x0043=>0x0063, 0x021A=>0x021B, 0x00CA=>0x00EA,
0x0399=>0x03B9, 0x0179=>0x017A, 0x00CF=>0x00EF, 0x01AF=>0x01B0, 0x0045=>0x0065,
0x039B=>0x03BB, 0x0398=>0x03B8, 0x039C=>0x03BC, 0x040C=>0x045C, 0x041F=>0x043F,
0x042C=>0x044C, 0x00DE=>0x00FE, 0x00D0=>0x00F0, 0x1EF2=>0x1EF3, 0x0048=>0x0068,
0x00CB=>0x00EB, 0x0110=>0x0111, 0x0413=>0x0433, 0x012E=>0x012F, 0x00C6=>0x00E6,
0x0058=>0x0078, 0x0160=>0x0161, 0x016E=>0x016F, 0x0391=>0x03B1, 0x0407=>0x0457,
0x0172=>0x0173, 0x0178=>0x00FF, 0x004F=>0x006F, 0x041B=>0x043B, 0x0395=>0x03B5,
0x0425=>0x0445, 0x0120=>0x0121, 0x017D=>0x017E, 0x017B=>0x017C, 0x0396=>0x03B6,
0x0392=>0x03B2, 0x0388=>0x03AD, 0x1E84=>0x1E85, 0x0174=>0x0175, 0x0051=>0x0071,
0x0417=>0x0437, 0x1E0A=>0x1E0B, 0x0147=>0x0148, 0x0104=>0x0105, 0x0408=>0x0458,
0x014C=>0x014D, 0x00CD=>0x00ED, 0x0059=>0x0079, 0x010A=>0x010B, 0x038F=>0x03CE,
0x0052=>0x0072, 0x0410=>0x0430, 0x0405=>0x0455, 0x0402=>0x0452, 0x0126=>0x0127,
0x0136=>0x0137, 0x012A=>0x012B, 0x038A=>0x03AF, 0x042B=>0x044B, 0x004C=>0x006C,
0x0397=>0x03B7, 0x0124=>0x0125, 0x0218=>0x0219, 0x00DB=>0x00FB, 0x011E=>0x011F,
0x041E=>0x043E, 0x1E40=>0x1E41, 0x039D=>0x03BD, 0x0106=>0x0107, 0x03AB=>0x03CB,
0x0426=>0x0446, 0x00DE=>0x00FE, 0x00C7=>0x00E7, 0x03AA=>0x03CA, 0x0421=>0x0441,
0x0412=>0x0432, 0x010E=>0x010F, 0x00D8=>0x00F8, 0x0057=>0x0077, 0x011A=>0x011B,
0x0054=>0x0074, 0x004A=>0x006A, 0x040B=>0x045B, 0x0406=>0x0456, 0x0102=>0x0103,
0x039B=>0x03BB, 0x00D1=>0x00F1, 0x041D=>0x043D, 0x038C=>0x03CC, 0x00C9=>0x00E9,
0x00D0=>0x00F0, 0x0407=>0x0457, 0x0122=>0x0123,
);
}
$uni = utf8_to_unicode($string);
if ( !$uni ) {
return FALSE;
}
$cnt = count($uni);
for ($i=0; $i < $cnt; $i++){
if ( isset($UTF8_UPPER_TO_LOWER[$uni[$i]]) ) {
$uni[$i] = $UTF8_UPPER_TO_LOWER[$uni[$i]];
}
}
return utf8_from_unicode($uni);
}
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to strtoupper
* Make a string uppercase
* Note: The concept of a characters "case" only exists is some alphabets
* such as Latin, Greek, Cyrillic, Armenian and archaic Georgian - it does
* not exist in the Chinese alphabet, for example. See Unicode Standard
* Annex #21: Case Mappings
* Note: requires utf8_to_unicode and utf8_from_unicode
* @author Andreas Gohr <andi@splitbrain.org>
* @param string
* @return mixed either string in lowercase or FALSE is UTF-8 invalid
* @see http://www.php.net/strtoupper
* @see utf8_to_unicode
* @see utf8_from_unicode
* @see http://www.unicode.org/reports/tr21/tr21-5.html
* @see http://dev.splitbrain.org/view/darcs/dokuwiki/inc/utf8.php
* @package utf8
* @subpackage strings
*/
function utf8_strtoupper($string){
static $UTF8_LOWER_TO_UPPER = NULL;
if ( is_null($UTF8_LOWER_TO_UPPER) ) {
$UTF8_LOWER_TO_UPPER = array(
0x0061=>0x0041, 0x03C6=>0x03A6, 0x0163=>0x0162, 0x00E5=>0x00C5, 0x0062=>0x0042,
0x013A=>0x0139, 0x00E1=>0x00C1, 0x0142=>0x0141, 0x03CD=>0x038E, 0x0101=>0x0100,
0x0491=>0x0490, 0x03B4=>0x0394, 0x015B=>0x015A, 0x0064=>0x0044, 0x03B3=>0x0393,
0x00F4=>0x00D4, 0x044A=>0x042A, 0x0439=>0x0419, 0x0113=>0x0112, 0x043C=>0x041C,
0x015F=>0x015E, 0x0144=>0x0143, 0x00EE=>0x00CE, 0x045E=>0x040E, 0x044F=>0x042F,
0x03BA=>0x039A, 0x0155=>0x0154, 0x0069=>0x0049, 0x0073=>0x0053, 0x1E1F=>0x1E1E,
0x0135=>0x0134, 0x0447=>0x0427, 0x03C0=>0x03A0, 0x0438=>0x0418, 0x00F3=>0x00D3,
0x0440=>0x0420, 0x0454=>0x0404, 0x0435=>0x0415, 0x0449=>0x0429, 0x014B=>0x014A,
0x0431=>0x0411, 0x0459=>0x0409, 0x1E03=>0x1E02, 0x00F6=>0x00D6, 0x00F9=>0x00D9,
0x006E=>0x004E, 0x0451=>0x0401, 0x03C4=>0x03A4, 0x0443=>0x0423, 0x015D=>0x015C,
0x0453=>0x0403, 0x03C8=>0x03A8, 0x0159=>0x0158, 0x0067=>0x0047, 0x00E4=>0x00C4,
0x03AC=>0x0386, 0x03AE=>0x0389, 0x0167=>0x0166, 0x03BE=>0x039E, 0x0165=>0x0164,
0x0117=>0x0116, 0x0109=>0x0108, 0x0076=>0x0056, 0x00FE=>0x00DE, 0x0157=>0x0156,
0x00FA=>0x00DA, 0x1E61=>0x1E60, 0x1E83=>0x1E82, 0x00E2=>0x00C2, 0x0119=>0x0118,
0x0146=>0x0145, 0x0070=>0x0050, 0x0151=>0x0150, 0x044E=>0x042E, 0x0129=>0x0128,
0x03C7=>0x03A7, 0x013E=>0x013D, 0x0442=>0x0422, 0x007A=>0x005A, 0x0448=>0x0428,
0x03C1=>0x03A1, 0x1E81=>0x1E80, 0x016D=>0x016C, 0x00F5=>0x00D5, 0x0075=>0x0055,
0x0177=>0x0176, 0x00FC=>0x00DC, 0x1E57=>0x1E56, 0x03C3=>0x03A3, 0x043A=>0x041A,
0x006D=>0x004D, 0x016B=>0x016A, 0x0171=>0x0170, 0x0444=>0x0424, 0x00EC=>0x00CC,
0x0169=>0x0168, 0x03BF=>0x039F, 0x006B=>0x004B, 0x00F2=>0x00D2, 0x00E0=>0x00C0,
0x0434=>0x0414, 0x03C9=>0x03A9, 0x1E6B=>0x1E6A, 0x00E3=>0x00C3, 0x044D=>0x042D,
0x0436=>0x0416, 0x01A1=>0x01A0, 0x010D=>0x010C, 0x011D=>0x011C, 0x00F0=>0x00D0,
0x013C=>0x013B, 0x045F=>0x040F, 0x045A=>0x040A, 0x00E8=>0x00C8, 0x03C5=>0x03A5,
0x0066=>0x0046, 0x00FD=>0x00DD, 0x0063=>0x0043, 0x021B=>0x021A, 0x00EA=>0x00CA,
0x03B9=>0x0399, 0x017A=>0x0179, 0x00EF=>0x00CF, 0x01B0=>0x01AF, 0x0065=>0x0045,
0x03BB=>0x039B, 0x03B8=>0x0398, 0x03BC=>0x039C, 0x045C=>0x040C, 0x043F=>0x041F,
0x044C=>0x042C, 0x00FE=>0x00DE, 0x00F0=>0x00D0, 0x1EF3=>0x1EF2, 0x0068=>0x0048,
0x00EB=>0x00CB, 0x0111=>0x0110, 0x0433=>0x0413, 0x012F=>0x012E, 0x00E6=>0x00C6,
0x0078=>0x0058, 0x0161=>0x0160, 0x016F=>0x016E, 0x03B1=>0x0391, 0x0457=>0x0407,
0x0173=>0x0172, 0x00FF=>0x0178, 0x006F=>0x004F, 0x043B=>0x041B, 0x03B5=>0x0395,
0x0445=>0x0425, 0x0121=>0x0120, 0x017E=>0x017D, 0x017C=>0x017B, 0x03B6=>0x0396,
0x03B2=>0x0392, 0x03AD=>0x0388, 0x1E85=>0x1E84, 0x0175=>0x0174, 0x0071=>0x0051,
0x0437=>0x0417, 0x1E0B=>0x1E0A, 0x0148=>0x0147, 0x0105=>0x0104, 0x0458=>0x0408,
0x014D=>0x014C, 0x00ED=>0x00CD, 0x0079=>0x0059, 0x010B=>0x010A, 0x03CE=>0x038F,
0x0072=>0x0052, 0x0430=>0x0410, 0x0455=>0x0405, 0x0452=>0x0402, 0x0127=>0x0126,
0x0137=>0x0136, 0x012B=>0x012A, 0x03AF=>0x038A, 0x044B=>0x042B, 0x006C=>0x004C,
0x03B7=>0x0397, 0x0125=>0x0124, 0x0219=>0x0218, 0x00FB=>0x00DB, 0x011F=>0x011E,
0x043E=>0x041E, 0x1E41=>0x1E40, 0x03BD=>0x039D, 0x0107=>0x0106, 0x03CB=>0x03AB,
0x0446=>0x0426, 0x00FE=>0x00DE, 0x00E7=>0x00C7, 0x03CA=>0x03AA, 0x0441=>0x0421,
0x0432=>0x0412, 0x010F=>0x010E, 0x00F8=>0x00D8, 0x0077=>0x0057, 0x011B=>0x011A,
0x0074=>0x0054, 0x006A=>0x004A, 0x045B=>0x040B, 0x0456=>0x0406, 0x0103=>0x0102,
0x03BB=>0x039B, 0x00F1=>0x00D1, 0x043D=>0x041D, 0x03CC=>0x038C, 0x00E9=>0x00C9,
0x00F0=>0x00D0, 0x0457=>0x0407, 0x0123=>0x0122,
);
}
$uni = utf8_to_unicode($string);
if ( !$uni ) {
return FALSE;
}
$cnt = count($uni);
for ($i=0; $i < $cnt; $i++){
if( isset($UTF8_LOWER_TO_UPPER[$uni[$i]]) ) {
$uni[$i] = $UTF8_LOWER_TO_UPPER[$uni[$i]];
}
}
return utf8_from_unicode($uni);
}

92
libraries/phputf8/ord.php Normal file
View File

@ -0,0 +1,92 @@
<?php
/**
* @version $Id: ord.php,v 1.4 2006/09/11 15:22:54 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to ord
* Returns the unicode ordinal for a character
* @param string UTF-8 encoded character
* @return int unicode ordinal for the character
* @see http://www.php.net/ord
* @see http://www.php.net/manual/en/function.ord.php#46267
*/
function utf8_ord($chr) {
$ord0 = ord($chr);
if ( $ord0 >= 0 && $ord0 <= 127 ) {
return $ord0;
}
if ( !isset($chr{1}) ) {
trigger_error('Short sequence - at least 2 bytes expected, only 1 seen');
return FALSE;
}
$ord1 = ord($chr{1});
if ( $ord0 >= 192 && $ord0 <= 223 ) {
return ( $ord0 - 192 ) * 64
+ ( $ord1 - 128 );
}
if ( !isset($chr{2}) ) {
trigger_error('Short sequence - at least 3 bytes expected, only 2 seen');
return FALSE;
}
$ord2 = ord($chr{2});
if ( $ord0 >= 224 && $ord0 <= 239 ) {
return ($ord0-224)*4096
+ ($ord1-128)*64
+ ($ord2-128);
}
if ( !isset($chr{3}) ) {
trigger_error('Short sequence - at least 4 bytes expected, only 3 seen');
return FALSE;
}
$ord3 = ord($chr{3});
if ($ord0>=240 && $ord0<=247) {
return ($ord0-240)*262144
+ ($ord1-128)*4096
+ ($ord2-128)*64
+ ($ord3-128);
}
if ( !isset($chr{4}) ) {
trigger_error('Short sequence - at least 5 bytes expected, only 4 seen');
return FALSE;
}
$ord4 = ord($chr{4});
if ($ord0>=248 && $ord0<=251) {
return ($ord0-248)*16777216
+ ($ord1-128)*262144
+ ($ord2-128)*4096
+ ($ord3-128)*64
+ ($ord4-128);
}
if ( !isset($chr{5}) ) {
trigger_error('Short sequence - at least 6 bytes expected, only 5 seen');
return FALSE;
}
if ($ord0>=252 && $ord0<=253) {
return ($ord0-252) * 1073741824
+ ($ord1-128)*16777216
+ ($ord2-128)*262144
+ ($ord3-128)*4096
+ ($ord4-128)*64
+ (ord($c{5})-128);
}
if ( $ord0 >= 254 && $ord0 <= 255 ) {
trigger_error('Invalid UTF-8 with surrogate ordinal '.$ord0);
return FALSE;
}
}

80
libraries/phputf8/str_ireplace.php Normal file
View File

@ -0,0 +1,80 @@
<?php
/**
* @version $Id: str_ireplace.php,v 1.2 2007/08/12 01:20:46 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to str_ireplace
* Case-insensitive version of str_replace
* Note: requires utf8_strtolower
* Note: it's not fast and gets slower if $search / $replace is array
* Notes: it's based on the assumption that the lower and uppercase
* versions of a UTF-8 character will have the same length in bytes
* which is currently true given the hash table to strtolower
* @param string
* @return string
* @see http://www.php.net/str_ireplace
* @see utf8_strtolower
* @package utf8
* @subpackage strings
*/
function utf8_ireplace($search, $replace, $str, $count = NULL){
if ( !is_array($search) ) {
$slen = strlen($search);
if ( $slen == 0 ) {
return $str;
}
$lendif = strlen($replace) - strlen($search);
$search = utf8_strtolower($search);
$search = preg_quote($search);
$lstr = utf8_strtolower($str);
$i = 0;
$matched = 0;
while ( preg_match('/(.*)'.$search.'/Us',$lstr, $matches) ) {
if ( $i === $count ) {
break;
}
$mlen = strlen($matches[0]);
$lstr = substr($lstr, $mlen);
$str = substr_replace($str, $replace, $matched+strlen($matches[1]), $slen);
$matched += $mlen + $lendif;
$i++;
}
return $str;
} else {
foreach ( array_keys($search) as $k ) {
if ( is_array($replace) ) {
if ( array_key_exists($k,$replace) ) {
$str = utf8_ireplace($search[$k], $replace[$k], $str, $count);
} else {
$str = utf8_ireplace($search[$k], '', $str, $count);
}
} else {
$str = utf8_ireplace($search[$k], $replace, $str, $count);
}
}
return $str;
}
}

57
libraries/phputf8/str_pad.php Normal file
View File

@ -0,0 +1,57 @@
<?php
/**
* @version $Id: str_pad.php,v 1.1 2006/09/03 09:25:13 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* Replacement for str_pad. $padStr may contain multi-byte characters.
*
* @author Oliver Saunders <oliver (a) osinternetservices.com>
* @param string $input
* @param int $length
* @param string $padStr
* @param int $type ( same constants as str_pad )
* @return string
* @see http://www.php.net/str_pad
* @see utf8_substr
* @package utf8
* @subpackage strings
*/
function utf8_str_pad($input, $length, $padStr = ' ', $type = STR_PAD_RIGHT) {
$inputLen = utf8_strlen($input);
if ($length <= $inputLen) {
return $input;
}
$padStrLen = utf8_strlen($padStr);
$padLen = $length - $inputLen;
if ($type == STR_PAD_RIGHT) {
$repeatTimes = ceil($padLen / $padStrLen);
return utf8_substr($input . str_repeat($padStr, $repeatTimes), 0, $length);
}
if ($type == STR_PAD_LEFT) {
$repeatTimes = ceil($padLen / $padStrLen);
return utf8_substr(str_repeat($padStr, $repeatTimes), 0, floor($padLen)) . $input;
}
if ($type == STR_PAD_BOTH) {
$padLen/= 2;
$padAmountLeft = floor($padLen);
$padAmountRight = ceil($padLen);
$repeatTimesLeft = ceil($padAmountLeft / $padStrLen);
$repeatTimesRight = ceil($padAmountRight / $padStrLen);
$paddingLeft = utf8_substr(str_repeat($padStr, $repeatTimesLeft), 0, $padAmountLeft);
$paddingRight = utf8_substr(str_repeat($padStr, $repeatTimesRight), 0, $padAmountLeft);
return $paddingLeft . $input . $paddingRight;
}
trigger_error('utf8_str_pad: Unknown padding type (' . $type . ')',E_USER_ERROR);
}

35
libraries/phputf8/str_split.php Normal file
View File

@ -0,0 +1,35 @@
<?php
/**
* @version $Id: str_split.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to str_split
* Convert a string to an array
* Note: requires utf8_strlen to be loaded
* @param string UTF-8 encoded
* @param int number to characters to split string by
* @return string characters in string reverses
* @see http://www.php.net/str_split
* @see utf8_strlen
* @package utf8
* @subpackage strings
*/
function utf8_str_split($str, $split_len = 1) {
if ( !preg_match('/^[0-9]+$/',$split_len) || $split_len < 1 ) {
return FALSE;
}
$len = utf8_strlen($str);
if ( $len <= $split_len ) {
return array($str);
}
preg_match_all('/.{'.$split_len.'}|[^\x00]{1,'.$split_len.'}$/us', $str, $ar);
return $ar[0];
}

26
libraries/phputf8/strcasecmp.php Normal file
View File

@ -0,0 +1,26 @@
<?php
/**
* @version $Id: strcasecmp.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to strcasecmp
* A case insensivite string comparison
* Note: requires utf8_strtolower
* @param string
* @param string
* @return int
* @see http://www.php.net/strcasecmp
* @see utf8_strtolower
* @package utf8
* @subpackage strings
*/
function utf8_strcasecmp($strX, $strY) {
$strX = utf8_strtolower($strX);
$strY = utf8_strtolower($strY);
return strcmp($strX, $strY);
}

41
libraries/phputf8/strcspn.php Normal file
View File

@ -0,0 +1,41 @@
<?php
/**
* @version $Id: strcspn.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to strcspn
* Find length of initial segment not matching mask
* Note: requires utf8_strlen and utf8_substr (if start, length are used)
* @param string
* @return int
* @see http://www.php.net/strcspn
* @see utf8_strlen
* @package utf8
* @subpackage strings
*/
function utf8_strcspn($str, $mask, $start = NULL, $length = NULL) {
if ( empty($mask) || strlen($mask) == 0 ) {
return NULL;
}
$mask = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$mask);
if ( $start !== NULL || $length !== NULL ) {
$str = utf8_substr($str, $start, $length);
}
preg_match('/^[^'.$mask.']+/u',$str, $matches);
if ( isset($matches[0]) ) {
return utf8_strlen($matches[0]);
}
return 0;
}

36
libraries/phputf8/stristr.php Normal file
View File

@ -0,0 +1,36 @@
<?php
/**
* @version $Id: stristr.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to stristr
* Find first occurrence of a string using case insensitive comparison
* Note: requires utf8_strtolower
* @param string
* @param string
* @return int
* @see http://www.php.net/strcasecmp
* @see utf8_strtolower
* @package utf8
* @subpackage strings
*/
function utf8_stristr($str, $search) {
if ( strlen($search) == 0 ) {
return $str;
}
$lstr = utf8_strtolower($str);
$lsearch = utf8_strtolower($search);
preg_match('/^(.*)'.preg_quote($lsearch).'/Us',$lstr, $matches);
if ( count($matches) == 2 ) {
return substr($str, strlen($matches[1]));
}
return FALSE;
}

22
libraries/phputf8/strrev.php Normal file
View File

@ -0,0 +1,22 @@
<?php
/**
* @version $Id: strrev.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to strrev
* Reverse a string
* @param string UTF-8 encoded
* @return string characters in string reverses
* @see http://www.php.net/strrev
* @package utf8
* @subpackage strings
*/
function utf8_strrev($str){
preg_match_all('/./us', $str, $ar);
return join('',array_reverse($ar[0]));
}

36
libraries/phputf8/strspn.php Normal file
View File

@ -0,0 +1,36 @@
<?php
/**
* @version $Id: strspn.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to strspn
* Find length of initial segment matching mask
* Note: requires utf8_strlen and utf8_substr (if start, length are used)
* @param string
* @return int
* @see http://www.php.net/strspn
* @package utf8
* @subpackage strings
*/
function utf8_strspn($str, $mask, $start = NULL, $length = NULL) {
$mask = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$mask);
if ( $start !== NULL || $length !== NULL ) {
$str = utf8_substr($str, $start, $length);
}
preg_match('/^['.$mask.']+/u',$str, $matches);
if ( isset($matches[0]) ) {
return utf8_strlen($matches[0]);
}
return 0;
}

24
libraries/phputf8/substr_replace.php Normal file
View File

@ -0,0 +1,24 @@
<?php
/**
* @version $Id: substr_replace.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware substr_replace.
* Note: requires utf8_substr to be loaded
* @see http://www.php.net/substr_replace
* @see utf8_strlen
* @see utf8_substr
*/
function utf8_substr_replace($str, $repl, $start , $length = NULL ) {
preg_match_all('/./us', $str, $ar);
preg_match_all('/./us', $repl, $rar);
if( $length === NULL ) {
$length = utf8_strlen($str);
}
array_splice( $ar[0], $start, $length, $rar[0] );
return join('',$ar[0]);
}

68
libraries/phputf8/trim.php Normal file
View File

@ -0,0 +1,68 @@
<?php
/**
* @version $Id: trim.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware replacement for ltrim()
* Note: you only need to use this if you are supplying the charlist
* optional arg and it contains UTF-8 characters. Otherwise ltrim will
* work normally on a UTF-8 string
* @author Andreas Gohr <andi@splitbrain.org>
* @see http://www.php.net/ltrim
* @see http://dev.splitbrain.org/view/darcs/dokuwiki/inc/utf8.php
* @return string
* @package utf8
* @subpackage strings
*/
function utf8_ltrim( $str, $charlist = FALSE ) {
if($charlist === FALSE) return ltrim($str);
//quote charlist for use in a characterclass
$charlist = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$charlist);
return preg_replace('/^['.$charlist.']+/u','',$str);
}
//---------------------------------------------------------------
/**
* UTF-8 aware replacement for rtrim()
* Note: you only need to use this if you are supplying the charlist
* optional arg and it contains UTF-8 characters. Otherwise rtrim will
* work normally on a UTF-8 string
* @author Andreas Gohr <andi@splitbrain.org>
* @see http://www.php.net/rtrim
* @see http://dev.splitbrain.org/view/darcs/dokuwiki/inc/utf8.php
* @return string
* @package utf8
* @subpackage strings
*/
function utf8_rtrim( $str, $charlist = FALSE ) {
if($charlist === FALSE) return rtrim($str);
//quote charlist for use in a characterclass
$charlist = preg_replace('!([\\\\\\-\\]\\[/^])!','\\\${1}',$charlist);
return preg_replace('/['.$charlist.']+$/u','',$str);
}
//---------------------------------------------------------------
/**
* UTF-8 aware replacement for trim()
* Note: you only need to use this if you are supplying the charlist
* optional arg and it contains UTF-8 characters. Otherwise trim will
* work normally on a UTF-8 string
* @author Andreas Gohr <andi@splitbrain.org>
* @see http://www.php.net/trim
* @see http://dev.splitbrain.org/view/darcs/dokuwiki/inc/utf8.php
* @return string
* @package utf8
* @subpackage strings
*/
function utf8_trim( $str, $charlist = FALSE ) {
if($charlist === FALSE) return trim($str);
return utf8_ltrim(utf8_rtrim($str, $charlist), $charlist);
}

34
libraries/phputf8/ucfirst.php Normal file
View File

@ -0,0 +1,34 @@
<?php
/**
* @version $Id: ucfirst.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to ucfirst
* Make a string's first character uppercase
* Note: requires utf8_strtoupper
* @param string
* @return string with first character as upper case (if applicable)
* @see http://www.php.net/ucfirst
* @see utf8_strtoupper
* @package utf8
* @subpackage strings
*/
function utf8_ucfirst($str){
switch ( utf8_strlen($str) ) {
case 0:
return '';
break;
case 1:
return utf8_strtoupper($str);
break;
default:
preg_match('/^(.{1})(.*)$/us', $str, $matches);
return utf8_strtoupper($matches[1]).$matches[2];
break;
}
}

44
libraries/phputf8/ucwords.php Normal file
View File

@ -0,0 +1,44 @@
<?php
/**
* @version $Id: ucwords.php,v 1.1 2006/02/25 13:50:17 harryf Exp $
* @package utf8
* @subpackage strings
*/
//---------------------------------------------------------------
/**
* UTF-8 aware alternative to ucwords
* Uppercase the first character of each word in a string
* Note: requires utf8_substr_replace and utf8_strtoupper
* @param string
* @return string with first char of each word uppercase
* @see http://www.php.net/ucwords
* @package utf8
* @subpackage strings
*/
function utf8_ucwords($str) {
// Note: [\x0c\x09\x0b\x0a\x0d\x20] matches;
// form feeds, horizontal tabs, vertical tabs, linefeeds and carriage returns
// This corresponds to the definition of a "word" defined at http://www.php.net/ucwords
$pattern = '/(^|([\x0c\x09\x0b\x0a\x0d\x20]+))([^\x0c\x09\x0b\x0a\x0d\x20]{1})[^\x0c\x09\x0b\x0a\x0d\x20]*/u';
return preg_replace_callback($pattern, 'utf8_ucwords_callback',$str);
}
//---------------------------------------------------------------
/**
* Callback function for preg_replace_callback call in utf8_ucwords
* You don't need to call this yourself
* @param array of matches corresponding to a single word
* @return string with first char of the word in uppercase
* @see utf8_ucwords
* @see utf8_strtoupper
* @package utf8
* @subpackage strings
*/
function utf8_ucwords_callback($matches) {
$leadingws = $matches[2];
$ucfirst = utf8_strtoupper($matches[3]);
$ucword = utf8_substr_replace(ltrim($matches[0]),$ucfirst,0,1);
return $leadingws . $ucword;
}

74
libraries/phputf8/utf8.php Normal file
View File

@ -0,0 +1,74 @@
<?php
/**
* This is the dynamic loader for the library. It checks whether you have
* the mbstring extension available and includes relevant files
* on that basis, falling back to the native (as in written in PHP) version
* if mbstring is unavailabe.
*
* It's probably easiest to use this, if you don't want to understand
* the dependencies involved, in conjunction with PHP versions etc. At
* the same time, you might get better performance by managing loading
* yourself. The smartest way to do this, bearing in mind performance,
* is probably to "load on demand" - i.e. just before you use these
* functions in your code, load the version you need.
*
* It makes sure the the following functions are available;
* utf8_strlen, utf8_strpos, utf8_strrpos, utf8_substr,
* utf8_strtolower, utf8_strtoupper
* Other functions in the ./native directory depend on these
* six functions being available
* @package utf8
*/
/**
* Put the current directory in this constant
*/
if ( !defined('UTF8') ) {
define('UTF8',dirname(__FILE__));
}
/**
* If string overloading is active, it will break many of the
* native implementations. mbstring.func_overload must be set
* to 0, 1 or 4 in php.ini (string overloading disabled).
* Also need to check we have the correct internal mbstring
* encoding
*/
if ( extension_loaded('mbstring')) {
if ( ini_get('mbstring.func_overload') & MB_OVERLOAD_STRING ) {
trigger_error('String functions are overloaded by mbstring',E_USER_ERROR);
}
mb_internal_encoding('UTF-8');
}
/**
* Check whether PCRE has been compiled with UTF-8 support
*/
$UTF8_ar = array();
if ( preg_match('/^.{1}$/u',"ñ",$UTF8_ar) != 1 ) {
trigger_error('PCRE is not compiled with UTF-8 support',E_USER_ERROR);
}
unset($UTF8_ar);
/**
* Load the smartest implementations of utf8_strpos, utf8_strrpos
* and utf8_substr
*/
if ( !defined('UTF8_CORE') ) {
if ( function_exists('mb_substr') ) {
require_once UTF8 . '/mbstring/core.php';
} else {
require_once UTF8 . '/utils/unicode.php';
require_once UTF8 . '/native/core.php';
}
}
/**
* Load the native implementation of utf8_substr_replace
*/
require_once UTF8 . '/substr_replace.php';
/**
* You should now be able to use all the other utf_* string functions
*/

220
libraries/phputf8/utils/ascii.php Normal file
View File

@ -0,0 +1,220 @@
<?php
/**
* Tools to help with ASCII in UTF-8
* @version $Id: ascii.php,v 1.5 2006/10/16 20:38:12 harryf Exp $
* @package utf8
* @subpackage ascii
*/
//--------------------------------------------------------------------
/**
* Tests whether a string contains only 7bit ASCII bytes.
* You might use this to conditionally check whether a string
* needs handling as UTF-8 or not, potentially offering performance
* benefits by using the native PHP equivalent if it's just ASCII e.g.;
*
* <code>
* if ( utf8_is_ascii($someString) ) {
* // It's just ASCII - use the native PHP version
* $someString = strtolower($someString);
* } else {
* $someString = utf8_strtolower($someString);
* }
* </code>
*
* @param string
* @return boolean TRUE if it's all ASCII
* @package utf8
* @subpackage ascii
* @see utf8_is_ascii_ctrl
*/
function utf8_is_ascii($str) {
// Search for any bytes which are outside the ASCII range...
return (preg_match('/(?:[^\x00-\x7F])/',$str) !== 1);
}
//--------------------------------------------------------------------
/**
* Tests whether a string contains only 7bit ASCII bytes with device
* control codes omitted. The device control codes can be found on the
* second table here: http://www.w3schools.com/tags/ref_ascii.asp
*
* @param string
* @return boolean TRUE if it's all ASCII without device control codes
* @package utf8
* @subpackage ascii
* @see utf8_is_ascii
*/
function utf8_is_ascii_ctrl($str) {
if ( strlen($str) > 0 ) {
// Search for any bytes which are outside the ASCII range,
// or are device control codes
return (preg_match('/[^\x09\x0A\x0D\x20-\x7E]/',$str) !== 1);
}
return FALSE;
}
//--------------------------------------------------------------------
/**
* Strip out all non-7bit ASCII bytes
* If you need to transmit a string to system which you know can only
* support 7bit ASCII, you could use this function.
* @param string
* @return string with non ASCII bytes removed
* @package utf8
* @subpackage ascii
* @see utf8_strip_non_ascii_ctrl
*/
function utf8_strip_non_ascii($str) {
ob_start();
while ( preg_match(
'/^([\x00-\x7F]+)|([^\x00-\x7F]+)/S',
$str, $matches) ) {
if ( !isset($matches[2]) ) {
echo $matches[0];
}
$str = substr($str, strlen($matches[0]));
}
$result = ob_get_contents();
ob_end_clean();
return $result;
}
//--------------------------------------------------------------------
/**
* Strip out device control codes in the ASCII range
* which are not permitted in XML. Note that this leaves
* multi-byte characters untouched - it only removes device
* control codes
* @see http://hsivonen.iki.fi/producing-xml/#controlchar
* @param string
* @return string control codes removed
*/
function utf8_strip_ascii_ctrl($str) {
ob_start();
while ( preg_match(
'/^([^\x00-\x08\x0B\x0C\x0E-\x1F\x7F]+)|([\x00-\x08\x0B\x0C\x0E-\x1F\x7F]+)/S',
$str, $matches) ) {
if ( !isset($matches[2]) ) {
echo $matches[0];
}
$str = substr($str, strlen($matches[0]));
}
$result = ob_get_contents();
ob_end_clean();
return $result;
}
//--------------------------------------------------------------------
/**
* Strip out all non 7bit ASCII bytes and ASCII device control codes.
* For a list of ASCII device control codes see the 2nd table here:
* http://www.w3schools.com/tags/ref_ascii.asp
*
* @param string
* @return boolean TRUE if it's all ASCII
* @package utf8
* @subpackage ascii
*/
function utf8_strip_non_ascii_ctrl($str) {
ob_start();
while ( preg_match(
'/^([\x09\x0A\x0D\x20-\x7E]+)|([^\x09\x0A\x0D\x20-\x7E]+)/S',
$str, $matches) ) {
if ( !isset($matches[2]) ) {
echo $matches[0];
}
$str = substr($str, strlen($matches[0]));
}
$result = ob_get_contents();
ob_end_clean();
return $result;
}
//---------------------------------------------------------------
/**
* Replace accented UTF-8 characters by unaccented ASCII-7 "equivalents".
* The purpose of this function is to replace characters commonly found in Latin
* alphabets with something more or less equivalent from the ASCII range. This can
* be useful for converting a UTF-8 to something ready for a filename, for example.
* Following the use of this function, you would probably also pass the string
* through utf8_strip_non_ascii to clean out any other non-ASCII chars
* Use the optional parameter to just deaccent lower ($case = -1) or upper ($case = 1)
* letters. Default is to deaccent both cases ($case = 0)
*
* For a more complete implementation of transliteration, see the utf8_to_ascii package
* available from the phputf8 project downloads:
* http://prdownloads.sourceforge.net/phputf8
*
* @param string UTF-8 string
* @param int (optional) -1 lowercase only, +1 uppercase only, 1 both cases
* @param string UTF-8 with accented characters replaced by ASCII chars
* @return string accented chars replaced with ascii equivalents
* @author Andreas Gohr <andi@splitbrain.org>
* @package utf8
* @subpackage ascii
*/
function utf8_accents_to_ascii( $str, $case=0 ){
static $UTF8_LOWER_ACCENTS = NULL;
static $UTF8_UPPER_ACCENTS = NULL;
if($case <= 0){
if ( is_null($UTF8_LOWER_ACCENTS) ) {
$UTF8_LOWER_ACCENTS = array(
'à' => 'a', 'ô' => 'o', 'ď' => 'd', 'ḟ' => 'f', 'ë' => 'e', 'š' => 's', 'ơ' => 'o',
'ß' => 'ss', 'ă' => 'a', 'ř' => 'r', 'ț' => 't', 'ň' => 'n', 'ā' => 'a', 'ķ' => 'k',
'ŝ' => 's', 'ỳ' => 'y', 'ņ' => 'n', 'ĺ' => 'l', 'ħ' => 'h', 'ṗ' => 'p', 'ó' => 'o',
'ú' => 'u', 'ě' => 'e', 'é' => 'e', 'ç' => 'c', 'ẁ' => 'w', 'ċ' => 'c', 'õ' => 'o',
'ṡ' => 's', 'ø' => 'o', 'ģ' => 'g', 'ŧ' => 't', 'ș' => 's', 'ė' => 'e', 'ĉ' => 'c',
'ś' => 's', 'î' => 'i', 'ű' => 'u', 'ć' => 'c', 'ę' => 'e', 'ŵ' => 'w', 'ṫ' => 't',
'ū' => 'u', 'č' => 'c', 'ö' => 'oe', 'è' => 'e', 'ŷ' => 'y', 'ą' => 'a', 'ł' => 'l',
'ų' => 'u', 'ů' => 'u', 'ş' => 's', 'ğ' => 'g', 'ļ' => 'l', 'ƒ' => 'f', 'ž' => 'z',
'ẃ' => 'w', 'ḃ' => 'b', 'å' => 'a', 'ì' => 'i', 'ï' => 'i', 'ḋ' => 'd', 'ť' => 't',
'ŗ' => 'r', 'ä' => 'ae', 'í' => 'i', 'ŕ' => 'r', 'ê' => 'e', 'ü' => 'ue', 'ò' => 'o',
'ē' => 'e', 'ñ' => 'n', 'ń' => 'n', 'ĥ' => 'h', 'ĝ' => 'g', 'đ' => 'd', 'ĵ' => 'j',
'ÿ' => 'y', 'ũ' => 'u', 'ŭ' => 'u', 'ư' => 'u', 'ţ' => 't', 'ý' => 'y', 'ő' => 'o',
'â' => 'a', 'ľ' => 'l', 'ẅ' => 'w', 'ż' => 'z', 'ī' => 'i', 'ã' => 'a', 'ġ' => 'g',
'ṁ' => 'm', 'ō' => 'o', 'ĩ' => 'i', 'ù' => 'u', 'į' => 'i', 'ź' => 'z', 'á' => 'a',
'û' => 'u', 'þ' => 'th', 'ð' => 'dh', 'æ' => 'ae', 'µ' => 'u', 'ĕ' => 'e',
);
}
$str = str_replace(
array_keys($UTF8_LOWER_ACCENTS),
array_values($UTF8_LOWER_ACCENTS),
$str
);
}
if($case >= 0){
if ( is_null($UTF8_UPPER_ACCENTS) ) {
$UTF8_UPPER_ACCENTS = array(
'À' => 'A', 'Ô' => 'O', 'Ď' => 'D', 'Ḟ' => 'F', 'Ë' => 'E', 'Š' => 'S', 'Ơ' => 'O',
'Ă' => 'A', 'Ř' => 'R', 'Ț' => 'T', 'Ň' => 'N', 'Ā' => 'A', 'Ķ' => 'K',
'Ŝ' => 'S', 'Ỳ' => 'Y', 'Ņ' => 'N', 'Ĺ' => 'L', 'Ħ' => 'H', 'Ṗ' => 'P', 'Ó' => 'O',
'Ú' => 'U', 'Ě' => 'E', 'É' => 'E', 'Ç' => 'C', 'Ẁ' => 'W', 'Ċ' => 'C', 'Õ' => 'O',
'Ṡ' => 'S', 'Ø' => 'O', 'Ģ' => 'G', 'Ŧ' => 'T', 'Ș' => 'S', 'Ė' => 'E', 'Ĉ' => 'C',
'Ś' => 'S', 'Î' => 'I', 'Ű' => 'U', 'Ć' => 'C', 'Ę' => 'E', 'Ŵ' => 'W', 'Ṫ' => 'T',
'Ū' => 'U', 'Č' => 'C', 'Ö' => 'Oe', 'È' => 'E', 'Ŷ' => 'Y', 'Ą' => 'A', 'Ł' => 'L',
'Ų' => 'U', 'Ů' => 'U', 'Ş' => 'S', 'Ğ' => 'G', 'Ļ' => 'L', 'Ƒ' => 'F', 'Ž' => 'Z',
'Ẃ' => 'W', 'Ḃ' => 'B', 'Å' => 'A', 'Ì' => 'I', 'Ï' => 'I', 'Ḋ' => 'D', 'Ť' => 'T',
'Ŗ' => 'R', 'Ä' => 'Ae', 'Í' => 'I', 'Ŕ' => 'R', 'Ê' => 'E', 'Ü' => 'Ue', 'Ò' => 'O',
'Ē' => 'E', 'Ñ' => 'N', 'Ń' => 'N', 'Ĥ' => 'H', 'Ĝ' => 'G', 'Đ' => 'D', 'Ĵ' => 'J',
'Ÿ' => 'Y', 'Ũ' => 'U', 'Ŭ' => 'U', 'Ư' => 'U', 'Ţ' => 'T', 'Ý' => 'Y', 'Ő' => 'O',
'Â' => 'A', 'Ľ' => 'L', 'Ẅ' => 'W', 'Ż' => 'Z', 'Ī' => 'I', 'Ã' => 'A', 'Ġ' => 'G',
'Ṁ' => 'M', 'Ō' => 'O', 'Ĩ' => 'I', 'Ù' => 'U', 'Į' => 'I', 'Ź' => 'Z', 'Á' => 'A',
'Û' => 'U', 'Þ' => 'Th', 'Ð' => 'Dh', 'Æ' => 'Ae', 'Ĕ' => 'E',
);
}
$str = str_replace(
array_keys($UTF8_UPPER_ACCENTS),
array_values($UTF8_UPPER_ACCENTS),
$str
);
}
return $str;
}

421
libraries/phputf8/utils/bad.php Normal file
View File

@ -0,0 +1,421 @@
<?php
/**
* @version $Id: bad.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
* Tools for locating / replacing bad bytes in UTF-8 strings
* The Original Code is Mozilla Communicator client code.
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998
* the Initial Developer. All Rights Reserved.
* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
* @see http://hsivonen.iki.fi/php-utf8/
* @package utf8
* @subpackage bad
* @see utf8_is_valid
*/
//--------------------------------------------------------------------
/**
* Locates the first bad byte in a UTF-8 string returning it's
* byte index in the string
* PCRE Pattern to locate bad bytes in a UTF-8 string
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @param string
* @return mixed integer byte index or FALSE if no bad found
* @package utf8
* @subpackage bad
*/
function utf8_bad_find($str) {
$UTF8_BAD =
'([\x00-\x7F]'. # ASCII (including control chars)
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
'|(.{1}))'; # invalid byte
$pos = 0;
$badList = array();
while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
$bytes = strlen($matches[0]);
if ( isset($matches[2])) {
return $pos;
}
$pos += $bytes;
$str = substr($str,$bytes);
}
return FALSE;
}
//--------------------------------------------------------------------
/**
* Locates all bad bytes in a UTF-8 string and returns a list of their
* byte index in the string
* PCRE Pattern to locate bad bytes in a UTF-8 string
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @param string
* @return mixed array of integers or FALSE if no bad found
* @package utf8
* @subpackage bad
*/
function utf8_bad_findall($str) {
$UTF8_BAD =
'([\x00-\x7F]'. # ASCII (including control chars)
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
'|(.{1}))'; # invalid byte
$pos = 0;
$badList = array();
while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
$bytes = strlen($matches[0]);
if ( isset($matches[2])) {
$badList[] = $pos;
}
$pos += $bytes;
$str = substr($str,$bytes);
}
if ( count($badList) > 0 ) {
return $badList;
}
return FALSE;
}
//--------------------------------------------------------------------
/**
* Strips out any bad bytes from a UTF-8 string and returns the rest
* PCRE Pattern to locate bad bytes in a UTF-8 string
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @param string
* @return string
* @package utf8
* @subpackage bad
*/
function utf8_bad_strip($str) {
$UTF8_BAD =
'([\x00-\x7F]'. # ASCII (including control chars)
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
'|(.{1}))'; # invalid byte
ob_start();
while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
if ( !isset($matches[2])) {
echo $matches[0];
}
$str = substr($str,strlen($matches[0]));
}
$result = ob_get_contents();
ob_end_clean();
return $result;
}
//--------------------------------------------------------------------
/**
* Replace bad bytes with an alternative character - ASCII character
* recommended is replacement char
* PCRE Pattern to locate bad bytes in a UTF-8 string
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @param string to search
* @param string to replace bad bytes with (defaults to '?') - use ASCII
* @return string
* @package utf8
* @subpackage bad
*/
function utf8_bad_replace($str, $replace = '?') {
$UTF8_BAD =
'([\x00-\x7F]'. # ASCII (including control chars)
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
'|(.{1}))'; # invalid byte
ob_start();
while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
if ( !isset($matches[2])) {
echo $matches[0];
} else {
echo $replace;
}
$str = substr($str,strlen($matches[0]));
}
$result = ob_get_contents();
ob_end_clean();
return $result;
}
//--------------------------------------------------------------------
/**
* Return code from utf8_bad_identify() when a five octet sequence is detected.
* Note: 5 octets sequences are valid UTF-8 but are not supported by Unicode so
* do not represent a useful character
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_5OCTET',1);
/**
* Return code from utf8_bad_identify() when a six octet sequence is detected.
* Note: 6 octets sequences are valid UTF-8 but are not supported by Unicode so
* do not represent a useful character
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_6OCTET',2);
/**
* Return code from utf8_bad_identify().
* Invalid octet for use as start of multi-byte UTF-8 sequence
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_SEQID',3);
/**
* Return code from utf8_bad_identify().
* From Unicode 3.1, non-shortest form is illegal
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_NONSHORT',4);
/**
* Return code from utf8_bad_identify().
* From Unicode 3.2, surrogate characters are illegal
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_SURROGATE',5);
/**
* Return code from utf8_bad_identify().
* Codepoints outside the Unicode range are illegal
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_UNIOUTRANGE',6);
/**
* Return code from utf8_bad_identify().
* Incomplete multi-octet sequence
* Note: this is kind of a "catch-all"
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
define('UTF8_BAD_SEQINCOMPLETE',7);
//--------------------------------------------------------------------
/**
* Reports on the type of bad byte found in a UTF-8 string. Returns a
* status code on the first bad byte found
* @author <hsivonen@iki.fi>
* @param string UTF-8 encoded string
* @return mixed integer constant describing problem or FALSE if valid UTF-8
* @see utf8_bad_explain
* @see http://hsivonen.iki.fi/php-utf8/
* @package utf8
* @subpackage bad
*/
function utf8_bad_identify($str, &$i) {
$mState = 0; // cached expected number of octets after the current octet
// until the beginning of the next UTF8 character sequence
$mUcs4 = 0; // cached Unicode character
$mBytes = 1; // cached expected number of octets in the current sequence
$len = strlen($str);
for($i = 0; $i < $len; $i++) {
$in = ord($str{$i});
if ( $mState == 0) {
// When mState is zero we expect either a US-ASCII character or a
// multi-octet sequence.
if (0 == (0x80 & ($in))) {
// US-ASCII, pass straight through.
$mBytes = 1;
} else if (0xC0 == (0xE0 & ($in))) {
// First octet of 2 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x1F) << 6;
$mState = 1;
$mBytes = 2;
} else if (0xE0 == (0xF0 & ($in))) {
// First octet of 3 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x0F) << 12;
$mState = 2;
$mBytes = 3;
} else if (0xF0 == (0xF8 & ($in))) {
// First octet of 4 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x07) << 18;
$mState = 3;
$mBytes = 4;
} else if (0xF8 == (0xFC & ($in))) {
/* First octet of 5 octet sequence.
*
* This is illegal because the encoded codepoint must be either
* (a) not the shortest form or
* (b) outside the Unicode range of 0-0x10FFFF.
*/
return UTF8_BAD_5OCTET;
} else if (0xFC == (0xFE & ($in))) {
// First octet of 6 octet sequence, see comments for 5 octet sequence.
return UTF8_BAD_6OCTET;
} else {
// Current octet is neither in the US-ASCII range nor a legal first
// octet of a multi-octet sequence.
return UTF8_BAD_SEQID;
}
} else {
// When mState is non-zero, we expect a continuation of the multi-octet
// sequence
if (0x80 == (0xC0 & ($in))) {
// Legal continuation.
$shift = ($mState - 1) * 6;
$tmp = $in;
$tmp = ($tmp & 0x0000003F) << $shift;
$mUcs4 |= $tmp;
/**
* End of the multi-octet sequence. mUcs4 now contains the final
* Unicode codepoint to be output
*/
if (0 == --$mState) {
// From Unicode 3.1, non-shortest form is illegal
if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
((4 == $mBytes) && ($mUcs4 < 0x10000)) ) {
return UTF8_BAD_NONSHORT;
// From Unicode 3.2, surrogate characters are illegal
} else if (($mUcs4 & 0xFFFFF800) == 0xD800) {
return UTF8_BAD_SURROGATE;
// Codepoints outside the Unicode range are illegal
} else if ($mUcs4 > 0x10FFFF) {
return UTF8_BAD_UNIOUTRANGE;
}
//initialize UTF8 cache
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
}
} else {
// ((0xC0 & (*in) != 0x80) && (mState != 0))
// Incomplete multi-octet sequence.
$i--;
return UTF8_BAD_SEQINCOMPLETE;
}
}
}
if ( $mState != 0 ) {
// Incomplete multi-octet sequence.
$i--;
return UTF8_BAD_SEQINCOMPLETE;
}
// No bad octets found
$i = NULL;
return FALSE;
}
//--------------------------------------------------------------------
/**
* Takes a return code from utf8_bad_identify() are returns a message
* (in English) explaining what the problem is.
* @param int return code from utf8_bad_identify
* @return mixed string message or FALSE if return code unknown
* @see utf8_bad_identify
* @package utf8
* @subpackage bad
*/
function utf8_bad_explain($code) {
switch ($code) {
case UTF8_BAD_5OCTET:
return 'Five octet sequences are valid UTF-8 but are not supported by Unicode';
break;
case UTF8_BAD_6OCTET:
return 'Six octet sequences are valid UTF-8 but are not supported by Unicode';
break;
case UTF8_BAD_SEQID:
return 'Invalid octet for use as start of multi-byte UTF-8 sequence';
break;
case UTF8_BAD_NONSHORT:
return 'From Unicode 3.1, non-shortest form is illegal';
break;
case UTF8_BAD_SURROGATE:
return 'From Unicode 3.2, surrogate characters are illegal';
break;
case UTF8_BAD_UNIOUTRANGE:
return 'Codepoints outside the Unicode range are illegal';
break;
case UTF8_BAD_SEQINCOMPLETE:
return 'Incomplete multi-octet sequence';
break;
}
trigger_error('Unknown error code: '.$code,E_USER_WARNING);
return FALSE;
}

69
libraries/phputf8/utils/patterns.php Normal file
View File

@ -0,0 +1,69 @@
<?php
/**
* PCRE Regular expressions for UTF-8. Note this file is not actually used by
* the rest of the library but these regular expressions can be useful to have
* available.
* @version $Id: patterns.php,v 1.1 2006/02/25 14:20:02 harryf Exp $
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @package utf8
* @subpackage patterns
*/
//--------------------------------------------------------------------
/**
* PCRE Pattern to check a UTF-8 string is valid
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @package utf8
* @subpackage patterns
*/
$UTF8_VALID = '^('.
'[\x00-\x7F]'. # ASCII (including control chars)
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
')*$';
//--------------------------------------------------------------------
/**
* PCRE Pattern to match single UTF-8 characters
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @package utf8
* @subpackage patterns
*/
$UTF8_MATCH =
'([\x00-\x7F])'. # ASCII (including control chars)
'|([\xC2-\xDF][\x80-\xBF])'. # non-overlong 2-byte
'|(\xE0[\xA0-\xBF][\x80-\xBF])'. # excluding overlongs
'|([\xE1-\xEC\xEE\xEF][\x80-\xBF]{2})'. # straight 3-byte
'|(\xED[\x80-\x9F][\x80-\xBF])'. # excluding surrogates
'|(\xF0[\x90-\xBF][\x80-\xBF]{2})'. # planes 1-3
'|([\xF1-\xF3][\x80-\xBF]{3})'. # planes 4-15
'|(\xF4[\x80-\x8F][\x80-\xBF]{2})'; # plane 16
//--------------------------------------------------------------------
/**
* PCRE Pattern to locate bad bytes in a UTF-8 string
* Comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @package utf8
* @subpackage patterns
*/
$UTF8_BAD =
'([\x00-\x7F]'. # ASCII (including control chars)
'|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
'|(.{1}))'; # invalid byte

173
libraries/phputf8/utils/position.php Normal file
View File

@ -0,0 +1,173 @@
<?php
/**
* Locate a byte index given a UTF-8 character index
* @version $Id: position.php,v 1.1 2006/10/01 00:01:31 harryf Exp $
* @package utf8
* @subpackage position
*/
//--------------------------------------------------------------------
/**
* Given a string and a character index in the string, in
* terms of the UTF-8 character position, returns the byte
* index of that character. Can be useful when you want to
* PHP's native string functions but we warned, locating
* the byte can be expensive
* Takes variable number of parameters - first must be
* the search string then 1 to n UTF-8 character positions
* to obtain byte indexes for - it is more efficient to search
* the string for multiple characters at once, than make
* repeated calls to this function
*
* @author Chris Smith<chris@jalakai.co.uk>
* @param string string to locate index in
* @param int (n times)
* @return mixed - int if only one input int, array if more
* @return boolean TRUE if it's all ASCII
* @package utf8
* @subpackage position
*/
function utf8_byte_position() {
$args = func_get_args();
$str =& array_shift($args);
if (!is_string($str)) return false;
$result = array();
// trivial byte index, character offset pair
$prev = array(0,0);
// use a short piece of str to estimate bytes per character
// $i (& $j) -> byte indexes into $str
$i = utf8_locate_next_chr($str, 300);
// $c -> character offset into $str
$c = strlen(utf8_decode(substr($str,0,$i)));
// deal with arguments from lowest to highest
sort($args);
foreach ($args as $offset) {
// sanity checks FIXME
// 0 is an easy check
if ($offset == 0) { $result[] = 0; continue; }
// ensure no endless looping
$safety_valve = 50;
do {
if ( ($c - $prev[1]) == 0 ) {
// Hack: gone past end of string
$error = 0;
$i = strlen($str);
break;
}
$j = $i + (int)(($offset-$c) * ($i - $prev[0]) / ($c - $prev[1]));
// correct to utf8 character boundary
$j = utf8_locate_next_chr($str, $j);
// save the index, offset for use next iteration
$prev = array($i,$c);
if ($j > $i) {
// determine new character offset
$c += strlen(utf8_decode(substr($str,$i,$j-$i)));
} else {
// ditto
$c -= strlen(utf8_decode(substr($str,$j,$i-$j)));
}
$error = abs($c-$offset);
// ready for next time around
$i = $j;
// from 7 it is faster to iterate over the string
} while ( ($error > 7) && --$safety_valve) ;
if ($error && $error <= 7) {
if ($c < $offset) {
// move up
while ($error--) { $i = utf8_locate_next_chr($str,++$i); }
} else {
// move down
while ($error--) { $i = utf8_locate_current_chr($str,--$i); }
}
// ready for next arg
$c = $offset;
}
$result[] = $i;
}
if ( count($result) == 1 ) {
return $result[0];
}
return $result;
}
//--------------------------------------------------------------------
/**
* Given a string and any byte index, returns the byte index
* of the start of the current UTF-8 character, relative to supplied
* position. If the current character begins at the same place as the
* supplied byte index, that byte index will be returned. Otherwise
* this function will step backwards, looking for the index where
* curent UTF-8 character begins
* @author Chris Smith<chris@jalakai.co.uk>
* @param string
* @param int byte index in the string
* @return int byte index of start of next UTF-8 character
* @package utf8
* @subpackage position
*/
function utf8_locate_current_chr( &$str, $idx ) {
if ($idx <= 0) return 0;
$limit = strlen($str);
if ($idx >= $limit) return $limit;
// Binary value for any byte after the first in a multi-byte UTF-8 character
// will be like 10xxxxxx so & 0xC0 can be used to detect this kind
// of byte - assuming well formed UTF-8
while ($idx && ((ord($str[$idx]) & 0xC0) == 0x80)) $idx--;
return $idx;
}
//--------------------------------------------------------------------
/**
* Given a string and any byte index, returns the byte index
* of the start of the next UTF-8 character, relative to supplied
* position. If the next character begins at the same place as the
* supplied byte index, that byte index will be returned.
* @author Chris Smith<chris@jalakai.co.uk>
* @param string
* @param int byte index in the string
* @return int byte index of start of next UTF-8 character
* @package utf8
* @subpackage position
*/
function utf8_locate_next_chr( &$str, $idx ) {
if ($idx <= 0) return 0;
$limit = strlen($str);
if ($idx >= $limit) return $limit;
// Binary value for any byte after the first in a multi-byte UTF-8 character
// will be like 10xxxxxx so & 0xC0 can be used to detect this kind
// of byte - assuming well formed UTF-8
while (($idx < $limit) && ((ord($str[$idx]) & 0xC0) == 0x80)) $idx++;
return $idx;
}

131
libraries/phputf8/utils/specials.php Normal file
View File

@ -0,0 +1,131 @@
<?php
/**
* Utilities for processing "special" characters in UTF-8. "Special" largely means anything which would
* be regarded as a non-word character, like ASCII control characters and punctuation. This has a "Roman"
* bias - it would be unaware of modern Chinese "punctuation" characters for example.
* Note: requires utils/unicode.php to be loaded
* @version $Id: specials.php,v 1.2 2006/10/16 21:13:59 harryf Exp $
* @package utf8
* @subpackage utils
* @see utf8_is_valid
*/
//--------------------------------------------------------------------
/**
* Used internally. Builds a PCRE pattern from the $UTF8_SPECIAL_CHARS
* array defined in this file
* The $UTF8_SPECIAL_CHARS should contain all special characters (non-letter/non-digit)
* defined in the various local charsets - it's not a complete list of
* non-alphanum characters in UTF-8. It's not perfect but should match most
* cases of special chars.
* This function adds the control chars 0x00 to 0x19 to the array of
* special chars (they are not included in $UTF8_SPECIAL_CHARS)
* @package utf8
* @subpackage utils
* @return string
* @see utf8_from_unicode
* @see utf8_is_word_chars
* @see utf8_strip_specials
*/
function utf8_specials_pattern() {
static $pattern = NULL;
if ( !$pattern ) {
$UTF8_SPECIAL_CHARS = array(
0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023,
0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c,
0x002f, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x005b,
0x005c, 0x005d, 0x005e, 0x0060, 0x007b, 0x007c, 0x007d, 0x007e,
0x007f, 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088,
0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092,
0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c,
0x009d, 0x009e, 0x009f, 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6,
0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, 0x00b0,
0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba,
0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, 0x00d7, 0x00f7, 0x02c7, 0x02d8, 0x02d9,
0x02da, 0x02db, 0x02dc, 0x02dd, 0x0300, 0x0301, 0x0303, 0x0309, 0x0323, 0x0384,
0x0385, 0x0387, 0x03b2, 0x03c6, 0x03d1, 0x03d2, 0x03d5, 0x03d6, 0x05b0, 0x05b1,
0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, 0x05b8, 0x05b9, 0x05bb, 0x05bc,
0x05bd, 0x05be, 0x05bf, 0x05c0, 0x05c1, 0x05c2, 0x05c3, 0x05f3, 0x05f4, 0x060c,
0x061b, 0x061f, 0x0640, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, 0x0650, 0x0651,
0x0652, 0x066a, 0x0e3f, 0x200c, 0x200d, 0x200e, 0x200f, 0x2013, 0x2014, 0x2015,
0x2017, 0x2018, 0x2019, 0x201a, 0x201c, 0x201d, 0x201e, 0x2020, 0x2021, 0x2022,
0x2026, 0x2030, 0x2032, 0x2033, 0x2039, 0x203a, 0x2044, 0x20a7, 0x20aa, 0x20ab,
0x20ac, 0x2116, 0x2118, 0x2122, 0x2126, 0x2135, 0x2190, 0x2191, 0x2192, 0x2193,
0x2194, 0x2195, 0x21b5, 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x2200, 0x2202,
0x2203, 0x2205, 0x2206, 0x2207, 0x2208, 0x2209, 0x220b, 0x220f, 0x2211, 0x2212,
0x2215, 0x2217, 0x2219, 0x221a, 0x221d, 0x221e, 0x2220, 0x2227, 0x2228, 0x2229,
0x222a, 0x222b, 0x2234, 0x223c, 0x2245, 0x2248, 0x2260, 0x2261, 0x2264, 0x2265,
0x2282, 0x2283, 0x2284, 0x2286, 0x2287, 0x2295, 0x2297, 0x22a5, 0x22c5, 0x2310,
0x2320, 0x2321, 0x2329, 0x232a, 0x2469, 0x2500, 0x2502, 0x250c, 0x2510, 0x2514,
0x2518, 0x251c, 0x2524, 0x252c, 0x2534, 0x253c, 0x2550, 0x2551, 0x2552, 0x2553,
0x2554, 0x2555, 0x2556, 0x2557, 0x2558, 0x2559, 0x255a, 0x255b, 0x255c, 0x255d,
0x255e, 0x255f, 0x2560, 0x2561, 0x2562, 0x2563, 0x2564, 0x2565, 0x2566, 0x2567,
0x2568, 0x2569, 0x256a, 0x256b, 0x256c, 0x2580, 0x2584, 0x2588, 0x258c, 0x2590,
0x2591, 0x2592, 0x2593, 0x25a0, 0x25b2, 0x25bc, 0x25c6, 0x25ca, 0x25cf, 0x25d7,
0x2605, 0x260e, 0x261b, 0x261e, 0x2660, 0x2663, 0x2665, 0x2666, 0x2701, 0x2702,
0x2703, 0x2704, 0x2706, 0x2707, 0x2708, 0x2709, 0x270c, 0x270d, 0x270e, 0x270f,
0x2710, 0x2711, 0x2712, 0x2713, 0x2714, 0x2715, 0x2716, 0x2717, 0x2718, 0x2719,
0x271a, 0x271b, 0x271c, 0x271d, 0x271e, 0x271f, 0x2720, 0x2721, 0x2722, 0x2723,
0x2724, 0x2725, 0x2726, 0x2727, 0x2729, 0x272a, 0x272b, 0x272c, 0x272d, 0x272e,
0x272f, 0x2730, 0x2731, 0x2732, 0x2733, 0x2734, 0x2735, 0x2736, 0x2737, 0x2738,
0x2739, 0x273a, 0x273b, 0x273c, 0x273d, 0x273e, 0x273f, 0x2740, 0x2741, 0x2742,
0x2743, 0x2744, 0x2745, 0x2746, 0x2747, 0x2748, 0x2749, 0x274a, 0x274b, 0x274d,
0x274f, 0x2750, 0x2751, 0x2752, 0x2756, 0x2758, 0x2759, 0x275a, 0x275b, 0x275c,
0x275d, 0x275e, 0x2761, 0x2762, 0x2763, 0x2764, 0x2765, 0x2766, 0x2767, 0x277f,
0x2789, 0x2793, 0x2794, 0x2798, 0x2799, 0x279a, 0x279b, 0x279c, 0x279d, 0x279e,
0x279f, 0x27a0, 0x27a1, 0x27a2, 0x27a3, 0x27a4, 0x27a5, 0x27a6, 0x27a7, 0x27a8,
0x27a9, 0x27aa, 0x27ab, 0x27ac, 0x27ad, 0x27ae, 0x27af, 0x27b1, 0x27b2, 0x27b3,
0x27b4, 0x27b5, 0x27b6, 0x27b7, 0x27b8, 0x27b9, 0x27ba, 0x27bb, 0x27bc, 0x27bd,
0x27be, 0xf6d9, 0xf6da, 0xf6db, 0xf8d7, 0xf8d8, 0xf8d9, 0xf8da, 0xf8db, 0xf8dc,
0xf8dd, 0xf8de, 0xf8df, 0xf8e0, 0xf8e1, 0xf8e2, 0xf8e3, 0xf8e4, 0xf8e5, 0xf8e6,
0xf8e7, 0xf8e8, 0xf8e9, 0xf8ea, 0xf8eb, 0xf8ec, 0xf8ed, 0xf8ee, 0xf8ef, 0xf8f0,
0xf8f1, 0xf8f2, 0xf8f3, 0xf8f4, 0xf8f5, 0xf8f6, 0xf8f7, 0xf8f8, 0xf8f9, 0xf8fa,
0xf8fb, 0xf8fc, 0xf8fd, 0xf8fe, 0xfe7c, 0xfe7d,
);
$pattern = preg_quote(utf8_from_unicode($UTF8_SPECIAL_CHARS), '/');
$pattern = '/[\x00-\x19'.$pattern.']/u';
}
return $pattern;
}
//--------------------------------------------------------------------
/**
* Checks a string for whether it contains only word characters. This
* is logically equivalent to the \w PCRE meta character. Note that
* this is not a 100% guarantee that the string only contains alpha /
* numeric characters but just that common non-alphanumeric are not
* in the string, including ASCII device control characters.
* @package utf8
* @subpackage utils
* @param string to check
* @return boolean TRUE if the string only contains word characters
* @see utf8_specials_pattern
*/
function utf8_is_word_chars($str) {
return !(bool)preg_match(utf8_specials_pattern(),$str);
}
//--------------------------------------------------------------------
/**
* Removes special characters (nonalphanumeric) from a UTF-8 string
*
* This can be useful as a helper for sanitizing a string for use as
* something like a file name or a unique identifier. Be warned though
* it does not handle all possible non-alphanumeric characters and is
* not intended is some kind of security / injection filter.
*
* @package utf8
* @subpackage utils
* @author Andreas Gohr <andi@splitbrain.org>
* @param string $string The UTF8 string to strip of special chars
* @param string (optional) $repl Replace special with this string
* @return string with common non-alphanumeric characters removed
* @see utf8_specials_pattern
*/
function utf8_strip_specials($string, $repl=''){
return preg_replace(utf8_specials_pattern(), $repl, $string);
}

269
libraries/phputf8/utils/unicode.php Normal file
View File

@ -0,0 +1,269 @@
<?php
/**
* @version $Id: unicode.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
* Tools for conversion between UTF-8 and unicode
* The Original Code is Mozilla Communicator client code.
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998
* the Initial Developer. All Rights Reserved.
* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
* @see http://hsivonen.iki.fi/php-utf8/
* @package utf8
* @subpackage unicode
*/
//--------------------------------------------------------------------
/**
* Takes an UTF-8 string and returns an array of ints representing the
* Unicode characters. Astral planes are supported ie. the ints in the
* output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
* are not allowed.
* Returns false if the input string isn't a valid UTF-8 octet sequence
* and raises a PHP error at level E_USER_WARNING
* Note: this function has been modified slightly in this library to
* trigger errors on encountering bad bytes
* @author <hsivonen@iki.fi>
* @param string UTF-8 encoded string
* @return mixed array of unicode code points or FALSE if UTF-8 invalid
* @see utf8_from_unicode
* @see http://hsivonen.iki.fi/php-utf8/
* @package utf8
* @subpackage unicode
*/
function utf8_to_unicode($str) {
$mState = 0; // cached expected number of octets after the current octet
// until the beginning of the next UTF8 character sequence
$mUcs4 = 0; // cached Unicode character
$mBytes = 1; // cached expected number of octets in the current sequence
$out = array();
$len = strlen($str);
for($i = 0; $i < $len; $i++) {
$in = ord($str{$i});
if ( $mState == 0) {
// When mState is zero we expect either a US-ASCII character or a
// multi-octet sequence.
if (0 == (0x80 & ($in))) {
// US-ASCII, pass straight through.
$out[] = $in;
$mBytes = 1;
} else if (0xC0 == (0xE0 & ($in))) {
// First octet of 2 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x1F) << 6;
$mState = 1;
$mBytes = 2;
} else if (0xE0 == (0xF0 & ($in))) {
// First octet of 3 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x0F) << 12;
$mState = 2;
$mBytes = 3;
} else if (0xF0 == (0xF8 & ($in))) {
// First octet of 4 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x07) << 18;
$mState = 3;
$mBytes = 4;
} else if (0xF8 == (0xFC & ($in))) {
/* First octet of 5 octet sequence.
*
* This is illegal because the encoded codepoint must be either
* (a) not the shortest form or
* (b) outside the Unicode range of 0-0x10FFFF.
* Rather than trying to resynchronize, we will carry on until the end
* of the sequence and let the later error handling code catch it.
*/
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x03) << 24;
$mState = 4;
$mBytes = 5;
} else if (0xFC == (0xFE & ($in))) {
// First octet of 6 octet sequence, see comments for 5 octet sequence.
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 1) << 30;
$mState = 5;
$mBytes = 6;
} else {
/* Current octet is neither in the US-ASCII range nor a legal first
* octet of a multi-octet sequence.
*/
trigger_error(
'utf8_to_unicode: Illegal sequence identifier '.
'in UTF-8 at byte '.$i,
E_USER_WARNING
);
return FALSE;
}
} else {
// When mState is non-zero, we expect a continuation of the multi-octet
// sequence
if (0x80 == (0xC0 & ($in))) {
// Legal continuation.
$shift = ($mState - 1) * 6;
$tmp = $in;
$tmp = ($tmp & 0x0000003F) << $shift;
$mUcs4 |= $tmp;
/**
* End of the multi-octet sequence. mUcs4 now contains the final
* Unicode codepoint to be output
*/
if (0 == --$mState) {
/*
* Check for illegal sequences and codepoints.
*/
// From Unicode 3.1, non-shortest form is illegal
if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
((4 == $mBytes) && ($mUcs4 < 0x10000)) ||
(4 < $mBytes) ||
// From Unicode 3.2, surrogate characters are illegal
(($mUcs4 & 0xFFFFF800) == 0xD800) ||
// Codepoints outside the Unicode range are illegal
($mUcs4 > 0x10FFFF)) {
trigger_error(
'utf8_to_unicode: Illegal sequence or codepoint '.
'in UTF-8 at byte '.$i,
E_USER_WARNING
);
return FALSE;
}
if (0xFEFF != $mUcs4) {
// BOM is legal but we don't want to output it
$out[] = $mUcs4;
}
//initialize UTF8 cache
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
}
} else {
/**
*((0xC0 & (*in) != 0x80) && (mState != 0))
* Incomplete multi-octet sequence.
*/
trigger_error(
'utf8_to_unicode: Incomplete multi-octet '.
' sequence in UTF-8 at byte '.$i,
E_USER_WARNING
);
return FALSE;
}
}
}
return $out;
}
//--------------------------------------------------------------------
/**
* Takes an array of ints representing the Unicode characters and returns
* a UTF-8 string. Astral planes are supported ie. the ints in the
* input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
* are not allowed.
* Returns false if the input array contains ints that represent
* surrogates or are outside the Unicode range
* and raises a PHP error at level E_USER_WARNING
* Note: this function has been modified slightly in this library to use
* output buffering to concatenate the UTF-8 string (faster) as well as
* reference the array by it's keys
* @param array of unicode code points representing a string
* @return mixed UTF-8 string or FALSE if array contains invalid code points
* @author <hsivonen@iki.fi>
* @see utf8_to_unicode
* @see http://hsivonen.iki.fi/php-utf8/
* @package utf8
* @subpackage unicode
*/
function utf8_from_unicode($arr) {
ob_start();
foreach (array_keys($arr) as $k) {
# ASCII range (including control chars)
if ( ($arr[$k] >= 0) && ($arr[$k] <= 0x007f) ) {
echo chr($arr[$k]);
# 2 byte sequence
} else if ($arr[$k] <= 0x07ff) {
echo chr(0xc0 | ($arr[$k] >> 6));
echo chr(0x80 | ($arr[$k] & 0x003f));
# Byte order mark (skip)
} else if($arr[$k] == 0xFEFF) {
// nop -- zap the BOM
# Test for illegal surrogates
} else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) {
// found a surrogate
trigger_error(
'utf8_from_unicode: Illegal surrogate '.
'at index: '.$k.', value: '.$arr[$k],
E_USER_WARNING
);
return FALSE;
# 3 byte sequence
} else if ($arr[$k] <= 0xffff) {
echo chr(0xe0 | ($arr[$k] >> 12));
echo chr(0x80 | (($arr[$k] >> 6) & 0x003f));
echo chr(0x80 | ($arr[$k] & 0x003f));
# 4 byte sequence
} else if ($arr[$k] <= 0x10ffff) {
echo chr(0xf0 | ($arr[$k] >> 18));
echo chr(0x80 | (($arr[$k] >> 12) & 0x3f));
echo chr(0x80 | (($arr[$k] >> 6) & 0x3f));
echo chr(0x80 | ($arr[$k] & 0x3f));
} else {
trigger_error(
'utf8_from_unicode: Codepoint out of Unicode range '.
'at index: '.$k.', value: '.$arr[$k],
E_USER_WARNING
);
// out of range
return FALSE;
}
}
$result = ob_get_contents();
ob_end_clean();
return $result;
}

185
libraries/phputf8/utils/validation.php Normal file
View File

@ -0,0 +1,185 @@
<?php
/**
* @version $Id: validation.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
* Tools for validing a UTF-8 string is well formed.
* The Original Code is Mozilla Communicator client code.
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998
* the Initial Developer. All Rights Reserved.
* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
* @see http://hsivonen.iki.fi/php-utf8/
* @package utf8
* @subpackage validation
*/
//--------------------------------------------------------------------
/**
* Tests a string as to whether it's valid UTF-8 and supported by the
* Unicode standard
* Note: this function has been modified to simple return true or false
* @author <hsivonen@iki.fi>
* @param string UTF-8 encoded string
* @return boolean true if valid
* @see http://hsivonen.iki.fi/php-utf8/
* @see utf8_compliant
* @package utf8
* @subpackage validation
*/
function utf8_is_valid($str) {
$mState = 0; // cached expected number of octets after the current octet
// until the beginning of the next UTF8 character sequence
$mUcs4 = 0; // cached Unicode character
$mBytes = 1; // cached expected number of octets in the current sequence
$len = strlen($str);
for($i = 0; $i < $len; $i++) {
$in = ord($str{$i});
if ( $mState == 0) {
// When mState is zero we expect either a US-ASCII character or a
// multi-octet sequence.
if (0 == (0x80 & ($in))) {
// US-ASCII, pass straight through.
$mBytes = 1;
} else if (0xC0 == (0xE0 & ($in))) {
// First octet of 2 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x1F) << 6;
$mState = 1;
$mBytes = 2;
} else if (0xE0 == (0xF0 & ($in))) {
// First octet of 3 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x0F) << 12;
$mState = 2;
$mBytes = 3;
} else if (0xF0 == (0xF8 & ($in))) {
// First octet of 4 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x07) << 18;
$mState = 3;
$mBytes = 4;
} else if (0xF8 == (0xFC & ($in))) {
/* First octet of 5 octet sequence.
*
* This is illegal because the encoded codepoint must be either
* (a) not the shortest form or
* (b) outside the Unicode range of 0-0x10FFFF.
* Rather than trying to resynchronize, we will carry on until the end
* of the sequence and let the later error handling code catch it.
*/
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x03) << 24;
$mState = 4;
$mBytes = 5;
} else if (0xFC == (0xFE & ($in))) {
// First octet of 6 octet sequence, see comments for 5 octet sequence.
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 1) << 30;
$mState = 5;
$mBytes = 6;
} else {
/* Current octet is neither in the US-ASCII range nor a legal first
* octet of a multi-octet sequence.
*/
return FALSE;
}
} else {
// When mState is non-zero, we expect a continuation of the multi-octet
// sequence
if (0x80 == (0xC0 & ($in))) {
// Legal continuation.
$shift = ($mState - 1) * 6;
$tmp = $in;
$tmp = ($tmp & 0x0000003F) << $shift;
$mUcs4 |= $tmp;
/**
* End of the multi-octet sequence. mUcs4 now contains the final
* Unicode codepoint to be output
*/
if (0 == --$mState) {
/*
* Check for illegal sequences and codepoints.
*/
// From Unicode 3.1, non-shortest form is illegal
if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
((4 == $mBytes) && ($mUcs4 < 0x10000)) ||
(4 < $mBytes) ||
// From Unicode 3.2, surrogate characters are illegal
(($mUcs4 & 0xFFFFF800) == 0xD800) ||
// Codepoints outside the Unicode range are illegal
($mUcs4 > 0x10FFFF)) {
return FALSE;
}
//initialize UTF8 cache
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
}
} else {
/**
*((0xC0 & (*in) != 0x80) && (mState != 0))
* Incomplete multi-octet sequence.
*/
return FALSE;
}
}
}
return TRUE;
}
//--------------------------------------------------------------------
/**
* Tests whether a string complies as UTF-8. This will be much
* faster than utf8_is_valid but will pass five and six octet
* UTF-8 sequences, which are not supported by Unicode and
* so cannot be displayed correctly in a browser. In other words
* it is not as strict as utf8_is_valid but it's faster. If you use
* is to validate user input, you place yourself at the risk that
* attackers will be able to inject 5 and 6 byte sequences (which
* may or may not be a significant risk, depending on what you are
* are doing)
* @see utf8_is_valid
* @see http://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
* @param string UTF-8 string to check
* @return boolean TRUE if string is valid UTF-8
* @package utf8
* @subpackage validation
*/
function utf8_compliant($str) {
if ( strlen($str) == 0 ) {
return TRUE;
}
// If even just the first character can be matched, when the /u
// modifier is used, then it's valid UTF-8. If the UTF-8 is somehow
// invalid, nothing at all will match, even if the string contains
// some valid sequences
return (preg_match('/^.{1}/us',$str,$ar) == 1);
}