php-ml/src/Phpml/Tokenization/WordTokenizer.php

22 lines
344 B
PHP
Raw Normal View History

2016-05-07 21:17:52 +00:00
<?php
2016-11-20 21:53:17 +00:00
declare(strict_types=1);
2016-05-07 21:17:52 +00:00
namespace Phpml\Tokenization;
class WordTokenizer implements Tokenizer
{
/**
* @param string $text
*
* @return array
*/
public function tokenize(string $text): array
{
$tokens = [];
preg_match_all('/\w\w+/u', $text, $tokens);
return $tokens[0];
}
}