[Utils/Trivia] Handle smart quotes (#2162)

Adds a new filter function for substituting out smart-quotes.

Makes trivia use it.

redbot/cogs/trivia/session.py

@@ -6,6 +6,7 @@ from collections import Counter
 import discord
 from redbot.core.bank import deposit_credits
 from redbot.core.utils.chat_formatting import box
+from redbot.core.utils.common_filters import normalize_smartquotes
 from .log import LOG
 
 __all__ = ["TriviaSession"]
@@ -222,6 +223,7 @@ class TriviaSession:
 
             self._last_response = time.time()
             guess = message.content.lower()
+            guess = normalize_smartquotes(guess)
             for answer in answers:
                 if " " in answer and answer in guess:
                     # Exact matching, issue #331

redbot/core/utils/common_filters.py

@@ -8,6 +8,7 @@ __all__ = [
     "filter_invites",
     "filter_mass_mentions",
     "filter_various_mentions",
+    "normalize_smartquotes",
 ]
 
 # regexes
@@ -19,6 +20,16 @@ MASS_MENTION_RE = re.compile(r"(@)(?=everyone|here)")  # This only matches the @
 
 OTHER_MENTION_RE = re.compile(r"(<)(@[!&]?|#)(\d+>)")
 
+SMART_QUOTE_REPLACEMENT_DICT = {
+    "\u2018": "'",  # Left single quote
+    "\u2019": "'",  # Right single quote
+    "\u201C": '"',  # Left double quote
+    "\u201D": '"',  # Right double quote
+}
+
+SMART_QUOTE_REPLACE_RE = re.compile("|".join(SMART_QUOTE_REPLACEMENT_DICT.keys()))
+
+
 # convenience wrappers
 def filter_urls(to_filter: str) -> str:
     """Get a string with URLs sanitized.
@@ -101,3 +112,24 @@ def filter_various_mentions(to_filter: str) -> str:
         The sanitized string.
     """
     return OTHER_MENTION_RE.sub(r"\1\\\2\3", to_filter)
+
+
+def normalize_smartquotes(to_normalize: str) -> str:
+    """
+    Get a string with smart quotes replaced with normal ones
+
+    Parameters
+    ----------
+    to_normalize : str
+        The string to normalize.
+
+    Returns
+    -------
+    str
+        The normalized string.
+    """
+
+    def replacement_for(obj):
+        return SMART_QUOTE_REPLACEMENT_DICT.get(obj.group(0), "")
+
+    return SMART_QUOTE_REPLACE_RE.sub(replacement_for, to_normalize)

tests/core/test_utils.py

@@ -2,12 +2,12 @@ import asyncio
 import pytest
 import random
 import textwrap
-import warnings
 from redbot.core.utils import (
     chat_formatting,
     bounded_gather,
     bounded_gather_iter,
     deduplicate_iterables,
+    common_filters,
 )
 
 
@@ -191,3 +191,8 @@ async def test_bounded_gather_iter_cancel():
     assert 0 < status[1] <= num_concurrent
     assert quit_on <= status[2] <= quit_on + num_concurrent
     assert num_failed <= num_fail
+
+
+def test_normalize_smartquotes():
+    assert common_filters.normalize_smartquotes("Should\u2018 normalize") == "Should' normalize"
+    assert common_filters.normalize_smartquotes("Same String") == "Same String"