mirror of
https://github.com/Llewellynvdm/php-ml.git
synced 2024-06-26 19:53:33 +00:00
21 lines
462 B
PHP
21 lines
462 B
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Phpml\Tokenization;
|
|
|
|
use Phpml\Exception\InvalidArgumentException;
|
|
|
|
class WhitespaceTokenizer implements Tokenizer
|
|
{
|
|
public function tokenize(string $text): array
|
|
{
|
|
$substrings = preg_split('/[\pZ\pC]+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
|
|
if ($substrings === false) {
|
|
throw new InvalidArgumentException('preg_split failed on: '.$text);
|
|
}
|
|
|
|
return $substrings;
|
|
}
|
|
}
|