Fixes #3939: `\yii\Inflector::slug()` improvements:

framework/CHANGELOG.md

@@ -90,6 +90,12 @@ Yii Framework 2 Change Log
 - Enh #3773: Added `FileValidator::mimeTypes` to support validating MIME types of files (Ragazzo)
 - Enh #3774: Added `FileValidator::checkExtensionByMimeType` to support validating file types against file mime-types (Ragazzo)
 - Enh #3801: Base migration controller `yii\console\controllers\BaseMigrateController` extracted (klimov-paul)
+- Enh #3939: `\yii\Inflector::slug()` improvements (samdark)
+    - Added protected `\yii\Inflector::transliterate()` that could be replaced with custom translit implementation.
+    - Added proper tests for both intl-based slug and PHP fallback.
+    - Removed character maps for non-latin languages.
+    - Improved overall slug results.
+    - Added note about the fact that intl is required for non-latin languages to requirements checker.
 - Enh: Added support for using sub-queries when building a DB query with `IN` condition (qiangxue)
 - Enh: Supported adding a new response formatter without the need to reconfigure existing formatters (qiangxue)
 - Enh: Added `yii\web\UrlManager::addRules()` to simplify adding new URL rules (qiangxue)

framework/helpers/BaseInflector.php

@@ -15,6 +15,7 @@ use Yii;
  * Do not use BaseInflector. Use [[Inflector]] instead.
  *
  * @author Antonio Ramirez <amigo.cobos@gmail.com>
+ * @author Alexander Makarov <sam@rmcreative.ru>
  * @since 2.0
  */
 class BaseInflector
@@ -217,10 +218,9 @@ class BaseInflector
     ];
 
     /**
-     * @var array map of special chars and its translation. This is used by [[slug()]].
+     * @var array fallback map for transliteration used by [[slug()]] when intl isn't available.
      */
     public static $transliteration = [
-        // Latin
         'À' => 'A', 'Á' => 'A', 'Â' => 'A', 'Ã' => 'A', 'Ä' => 'A', 'Å' => 'A', 'Æ' => 'AE', 'Ç' => 'C',
         'È' => 'E', 'É' => 'E', 'Ê' => 'E', 'Ë' => 'E', 'Ì' => 'I', 'Í' => 'I', 'Î' => 'I', 'Ï' => 'I',
         'Ð' => 'D', 'Ñ' => 'N', 'Ò' => 'O', 'Ó' => 'O', 'Ô' => 'O', 'Õ' => 'O', 'Ö' => 'O', 'Ő' => 'O',
@@ -231,62 +231,6 @@ class BaseInflector
         'ð' => 'd', 'ñ' => 'n', 'ò' => 'o', 'ó' => 'o', 'ô' => 'o', 'õ' => 'o', 'ö' => 'o', 'ő' => 'o',
         'ø' => 'o', 'ù' => 'u', 'ú' => 'u', 'û' => 'u', 'ü' => 'u', 'ű' => 'u', 'ý' => 'y', 'þ' => 'th',
         'ÿ' => 'y',
-        // Latin symbols
-        '©' => '(c)',
-        // Greek
-        'Α' => 'A', 'Β' => 'B', 'Γ' => 'G', 'Δ' => 'D', 'Ε' => 'E', 'Ζ' => 'Z', 'Η' => 'H', 'Θ' => '8',
-        'Ι' => 'I', 'Κ' => 'K', 'Λ' => 'L', 'Μ' => 'M', 'Ν' => 'N', 'Ξ' => '3', 'Ο' => 'O', 'Π' => 'P',
-        'Ρ' => 'R', 'Σ' => 'S', 'Τ' => 'T', 'Υ' => 'Y', 'Φ' => 'F', 'Χ' => 'X', 'Ψ' => 'PS', 'Ω' => 'W',
-        'Ά' => 'A', 'Έ' => 'E', 'Ί' => 'I', 'Ό' => 'O', 'Ύ' => 'Y', 'Ή' => 'H', 'Ώ' => 'W', 'Ϊ' => 'I',
-        'Ϋ' => 'Y',
-        'α' => 'a', 'β' => 'b', 'γ' => 'g', 'δ' => 'd', 'ε' => 'e', 'ζ' => 'z', 'η' => 'h', 'θ' => '8',
-        'ι' => 'i', 'κ' => 'k', 'λ' => 'l', 'μ' => 'm', 'ν' => 'n', 'ξ' => '3', 'ο' => 'o', 'π' => 'p',
-        'ρ' => 'r', 'σ' => 's', 'τ' => 't', 'υ' => 'y', 'φ' => 'f', 'χ' => 'x', 'ψ' => 'ps', 'ω' => 'w',
-        'ά' => 'a', 'έ' => 'e', 'ί' => 'i', 'ό' => 'o', 'ύ' => 'y', 'ή' => 'h', 'ώ' => 'w', 'ς' => 's',
-        'ϊ' => 'i', 'ΰ' => 'y', 'ϋ' => 'y', 'ΐ' => 'i',
-        // Turkish
-        'Ş' => 'S', 'İ' => 'I', 'Ç' => 'C', 'Ü' => 'U', 'Ö' => 'O', 'Ğ' => 'G',
-        'ş' => 's', 'ı' => 'i', 'ç' => 'c', 'ü' => 'u', 'ö' => 'o', 'ğ' => 'g',
-        // Russian
-        'А' => 'A', 'Б' => 'B', 'В' => 'V', 'Г' => 'G', 'Д' => 'D', 'Е' => 'E', 'Ё' => 'Yo', 'Ж' => 'Zh',
-        'З' => 'Z', 'И' => 'I', 'Й' => 'J', 'К' => 'K', 'Л' => 'L', 'М' => 'M', 'Н' => 'N', 'О' => 'O',
-        'П' => 'P', 'Р' => 'R', 'С' => 'S', 'Т' => 'T', 'У' => 'U', 'Ф' => 'F', 'Х' => 'H', 'Ц' => 'C',
-        'Ч' => 'Ch', 'Ш' => 'Sh', 'Щ' => 'Sh', 'Ъ' => '', 'Ы' => 'Y', 'Ь' => '', 'Э' => 'E', 'Ю' => 'Yu',
-        'Я' => 'Ya',
-        'а' => 'a', 'б' => 'b', 'в' => 'v', 'г' => 'g', 'д' => 'd', 'е' => 'e', 'ё' => 'yo', 'ж' => 'zh',
-        'з' => 'z', 'и' => 'i', 'й' => 'j', 'к' => 'k', 'л' => 'l', 'м' => 'm', 'н' => 'n', 'о' => 'o',
-        'п' => 'p', 'р' => 'r', 'с' => 's', 'т' => 't', 'у' => 'u', 'ф' => 'f', 'х' => 'h', 'ц' => 'c',
-        'ч' => 'ch', 'ш' => 'sh', 'щ' => 'sh', 'ъ' => '', 'ы' => 'y', 'ь' => '', 'э' => 'e', 'ю' => 'yu',
-        'я' => 'ya',
-        // Ukrainian
-        'Є' => 'Ye', 'І' => 'I', 'Ї' => 'Yi', 'Ґ' => 'G',
-        'є' => 'ye', 'і' => 'i', 'ї' => 'yi', 'ґ' => 'g',
-        // Czech
-        'Č' => 'C', 'Ď' => 'D', 'Ě' => 'E', 'Ň' => 'N', 'Ř' => 'R', 'Š' => 'S', 'Ť' => 'T', 'Ů' => 'U',
-        'Ž' => 'Z',
-        'č' => 'c', 'ď' => 'd', 'ě' => 'e', 'ň' => 'n', 'ř' => 'r', 'š' => 's', 'ť' => 't', 'ů' => 'u',
-        'ž' => 'z',
-        // Polish
-        'Ą' => 'A', 'Ć' => 'C', 'Ę' => 'e', 'Ł' => 'L', 'Ń' => 'N', 'Ó' => 'o', 'Ś' => 'S', 'Ź' => 'Z',
-        'Ż' => 'Z',
-        'ą' => 'a', 'ć' => 'c', 'ę' => 'e', 'ł' => 'l', 'ń' => 'n', 'ó' => 'o', 'ś' => 's', 'ź' => 'z',
-        'ż' => 'z',
-        // Latvian
-        'Ā' => 'A', 'Č' => 'C', 'Ē' => 'E', 'Ģ' => 'G', 'Ī' => 'i', 'Ķ' => 'k', 'Ļ' => 'L', 'Ņ' => 'N',
-        'Š' => 'S', 'Ū' => 'u', 'Ž' => 'Z',
-        'ā' => 'a', 'č' => 'c', 'ē' => 'e', 'ģ' => 'g', 'ī' => 'i', 'ķ' => 'k', 'ļ' => 'l', 'ņ' => 'n',
-        'š' => 's', 'ū' => 'u', 'ž' => 'z',
-        //Vietnamese
-        'Ấ' => 'A', 'Ầ' => 'A', 'Ẩ' => 'A', 'Ẫ' => 'A', 'Ậ' => 'A',
-        'Ắ' => 'A', 'Ằ' => 'A', 'Ẳ' => 'A', 'Ẵ' => 'A', 'Ặ' => 'A',
-        'Ố' => 'O', 'Ồ' => 'O', 'Ổ' => 'O', 'Ỗ' => 'O', 'Ộ' => 'O',
-        'Ớ' => 'O', 'Ờ' => 'O', 'Ở' => 'O', 'Ỡ' => 'O', 'Ợ' => 'O',
-        'Ế' => 'E', 'Ề' => 'E', 'Ể' => 'E', 'Ễ' => 'E', 'Ệ' => 'E',
-        'ấ' => 'a', 'ầ' => 'a', 'ẩ' => 'a', 'ẫ' => 'a', 'ậ' => 'a',
-        'ắ' => 'a', 'ằ' => 'a', 'ẳ' => 'a', 'ẵ' => 'a', 'ặ' => 'a',
-        'ố' => 'o', 'ồ' => 'o', 'ổ' => 'o', 'ỗ' => 'o', 'ộ' => 'o',
-        'ớ' => 'o', 'ờ' => 'o', 'ở' => 'o', 'ỡ' => 'o', 'ợ' => 'o',
-        'ế' => 'e', 'ề' => 'e', 'ể' => 'e', 'ễ' => 'e', 'ệ' => 'e'
     ];
 
     /**
@@ -456,9 +400,13 @@ class BaseInflector
     }
 
     /**
-     * Returns a string with all spaces converted to given replacement and
-     * non word characters removed.  Maps special characters to ASCII using
-     * [[$transliteration]] array.
+     * Returns a string with all spaces converted to given replacement,
+     * non word characters removed and the rest of characters transliterated.
+     *
+     * If intl extension isn't available uses fallback that converts latin characters only
+     * and removes the rest. You may customize characters map via $transliteration property
+     * of the helper.
+     *
      * @param string $string An arbitrary string to convert
      * @param string $replacement The replacement to use for spaces
      * @param boolean $lowercase whether to return the string in lowercase or not. Defaults to `true`.
@@ -466,20 +414,42 @@ class BaseInflector
      */
     public static function slug($string, $replacement = '-', $lowercase = true)
     {
-        if (extension_loaded('intl') === true) {
-            $options = 'Any-Latin; NFKD; [:Punctuation:] Remove; [^\u0000-\u007E] Remove';
-            $string = transliterator_transliterate($options, $string);
-            $string = preg_replace('/[-=\s]+/', $replacement, $string);
-        } else {
-            $string = str_replace(array_keys(static::$transliteration), static::$transliteration, $string);
-            $string = preg_replace('/[^\p{L}\p{Nd}]+/u', $replacement, $string);
-        }
+        $string = static::transliterate($string);
+        $string = preg_replace('/[^a-zA-Z=\s—–-]+/u', '', $string);
+        $string = preg_replace('/[=\s—–-]+/u', $replacement, $string);
         $string = trim($string, $replacement);
 
         return $lowercase ? strtolower($string) : $string;
     }
 
     /**
+     * Returns transliterated version of a string.
+     *
+     * If intl extension isn't available uses fallback that converts latin characters only
+     * and removes the rest. You may customize characters map via $transliteration property
+     * of the helper.
+     *
+     * @param string $string input string
+     * @return string
+     */
+    protected static function transliterate($string)
+    {
+        if (static::hasIntl()) {
+            return transliterator_transliterate('Any-Latin; NFKD', $string);
+        } else {
+            return str_replace(array_keys(static::$transliteration), static::$transliteration, $string);
+        }
+    }
+
+    /**
+     * @return boolean if intl extension is loaded
+     */
+    protected static function hasIntl()
+    {
+        return extension_loaded('intl');
+    }
+
+    /**
      * Converts a table name to its class name. For example, converts "people" to "Person"
      * @param string $tableName
      * @return string

framework/requirements/requirements.php

@@ -44,7 +44,8 @@ return array(
         'condition' => $this->checkPhpExtensionVersion('intl', '1.0.2', '>='),
         'by' => '<a href="http://www.php.net/manual/en/book.intl.php">Internationalization</a> support',
         'memo' => 'PHP Intl extension 1.0.2 or higher is required when you want to use advanced parameters formatting
-        in <code>Yii::t()</code>, <abbr title="Internationalized domain names">IDN</abbr>-feature of
+        in <code>Yii::t()</code>, non-latin languages with <code>Inflector::slug()</code>,
+        <abbr title="Internationalized domain names">IDN</abbr>-feature of
         <code>EmailValidator</code> or <code>UrlValidator</code> or the <code>yii\i18n\Formatter</code> class.'
     ),
     array(

tests/unit/framework/helpers/FallbackInflector.php

+<?php
+
+
+namespace yiiunit\framework\helpers;
+
+
+use yii\helpers\BaseInflector;
+
+/**
+ * Forces Inflector::slug to use PHP even if intl is available
+ */
+class FallbackInflector extends BaseInflector
+{
+    /**
+     * @inheritdoc
+     */
+    protected static function hasIntl()
+    {
+        return false;
+    }
+}
\ No newline at end of file

tests/unit/framework/helpers/InflectorTest.php

@@ -122,23 +122,75 @@ class InflectorTest extends TestCase
         $this->assertEquals("customer_tables", Inflector::tableize('customerTable'));
     }
 
-    public function testSlug()
+    public function testSlugCommons()
     {
         $data = [
-            'Привет. Hello, Йии-- Framework !--- Как дела ? How it goes ?' => 'privet-hello-jii-framework-kak-dela-how-it-goes',
-            'this is a title' => 'this-is-a-title',
-            'недвижимость' => 'nedvizimost',
+            '' => '',
+            'hello world' => 'hello-world',
+            'remove.!?[]{}…symbols' => 'removesymbols',
+            'minus-sign' => 'minus-sign',
+            'mdash—sign' => 'mdash-sign',
+            'ndash–sign' => 'ndash-sign',
             'áàâéèêíìîóòôúùûã' => 'aaaeeeiiiooouuua',
-            'Ναδάλης ṃỹṛèşưḿĕ' => 'nadales-myresume',
-            'E=mc²' => 'e-mc2',
-            '載å¥' => 'e14a',
+            'älä lyö ääliö ööliä läikkyy' => 'ala-lyo-aalio-oolia-laikkyy',
         ];
 
         foreach ($data as $source => $expected) {
+            if (extension_loaded('intl')) {
+                $this->assertEquals($expected, FallbackInflector::slug($source));
+            }
             $this->assertEquals($expected, Inflector::slug($source));
         }
     }
 
+    public function testSlugIntl()
+    {
+        if (!extension_loaded('intl')) {
+            $this->markTestSkipped('intl extension is required.');
+        }
+
+        // Some test strings are from https://github.com/bergie/midgardmvc_helper_urlize. Thank you, Henri Bergius!
+        $data = [
+            // Korean
+            '해동검도' => 'haedong-geomdo',
+
+            // Hiragana
+            'ひらがな' => 'hiragana',
+
+            // Georgian
+            'საქართველო' => 'sakartvelo',
+
+            // Arabic
+            'العربي' => 'alrby',
+            'عرب' => 'rb',
+
+            // Hebrew
+            'עִבְרִית' => 'iberiyt',
+
+            // Turkish
+            'Sanırım hepimiz aynı şeyi düşünüyoruz.' => 'sanrm-hepimiz-ayn-seyi-dusunuyoruz',
+
+            // Russian
+            'недвижимость' => 'nedvizimost',
+            'Контакты' => 'kontakty',
+        ];
+
+        foreach ($data as $source => $expected) {
+            $this->assertEquals($expected, Inflector::slug($source));
+        }
+    }
+
+    public function testSlugPhp()
+    {
+        $data = [
+            'we have недвижимость' => 'we-have',
+        ];
+
+        foreach ($data as $source => $expected) {
+            $this->assertEquals($expected, FallbackInflector::slug($source));
+        }
+    }
+
     public function testClassify()
     {
         $this->assertEquals("CustomerTable", Inflector::classify('customer_tables'));