Support "real" UTF-8 by changing to utf8mb4 (winterrodeln tables) and binary (Mediawi...

git-svn-id: http://www.winterrodeln.org/svn/wrpylib/trunk@2916 7aebc617-e5e2-0310-91dc-80fb5f6d2477

diff --git a/scripts/createthumbnails.py b/scripts/createthumbnails.py
index a6a95d3edad4a24ab793bd1c4cad704551590256..6b909323b85b86d542899abfd0b17812dbfdf09c 100755 (executable)
--- a/scripts/createthumbnails.py
+++ b/scripts/createthumbnails.py
@@ -118,7 +118,7 @@ def update_image_dir_by_inifile(inifile, mode):
     dbname = config.get('mysql', 'dbname')
     user = config.get('mysql', 'user_name')
     passwd = config.get('mysql', 'user_pass')
-    mcon = MySQLdb.connect(host=host, db=dbname, user=user, passwd=passwd, charset='utf8')
+    mcon = MySQLdb.connect(host=host, db=dbname, user=user, passwd=passwd, charset='utf8mb4')
 
     mediawiki_image_dir = config.get('mediawiki', 'imagedir')
     if mode == 'thumbnail':

diff --git a/scripts/updatewrinncache.py b/scripts/updatewrinncache.py
index d0ffa58186bef0992b2887adde4e432bd635e2b5..46c28b9c424a78f072e6ade8bca8ada8f14a279c 100755 (executable)
--- a/scripts/updatewrinncache.py
+++ b/scripts/updatewrinncache.py
@@ -32,8 +32,7 @@ def update_wrinncache(inifile):
     user = config.get('mysql', 'user_name')
     passwd = config.get('mysql', 'user_pass')
 
-    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?charset=utf8&passwd={passwd}&use_unicode=0')
-    # "use_unicode=0": see: https://sourceforge.net/tracker/?func=detail&aid=2837134&group_id=22307&atid=374932
+    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?passwd={passwd}&charset=utf8mb4')
     wrpylib.wrmwcache.update_wrinncache(engine.connect())
 
 

diff --git a/scripts/updatewrmapcache.py b/scripts/updatewrmapcache.py
index 652a940a4f4dca4c66390dac3d6e4b34ec7226ee..66ebbbbb7d1aa1b59da8d26a853e1337114d4fb8 100755 (executable)
--- a/scripts/updatewrmapcache.py
+++ b/scripts/updatewrmapcache.py
@@ -32,8 +32,7 @@ def update_wrmapcache(inifile):
     user = config.get('mysql', 'user_name')
     passwd = config.get('mysql', 'user_pass')
 
-    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?charset=utf8&passwd={passwd}&use_unicode=0')
-    # "use_unicode=0": see: https://sourceforge.net/tracker/?func=detail&aid=2837134&group_id=22307&atid=374932
+    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?passwd={passwd}&charset=utf8mb4')
     wrpylib.wrmwcache.update_wrmapcache(engine.connect())
 
 

diff --git a/scripts/updatewrregioncache.py b/scripts/updatewrregioncache.py
index 5ec983e8e2357347d7660d032e56943d870bda46..0a258ac1154fc828ea01adc647ef157a876fc8ef 100755 (executable)
--- a/scripts/updatewrregioncache.py
+++ b/scripts/updatewrregioncache.py
@@ -30,7 +30,7 @@ def update_wrregioncache(inifiles):
     user = config.get('mysql', 'user_name')
     passwd = config.get('mysql', 'user_pass')
 
-    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?charset=utf8&passwd={passwd}&use_unicode=1')
+    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?passwd={passwd}&charset=utf8mb4')
     wrpylib.wrmwcache.update_wrregioncache(engine.connect())
 
 

diff --git a/scripts/updatewrreportcache.py b/scripts/updatewrreportcache.py
index fb005e776ef77f5bc246559252585b5033175944..29061d61da7dd54ab6ce4fc1ef346eab35b9b916 100755 (executable)
--- a/scripts/updatewrreportcache.py
+++ b/scripts/updatewrreportcache.py
@@ -32,8 +32,7 @@ def update_wrreportcache(inifile):
     user = config.get('mysql', 'user_name')
     passwd = config.get('mysql', 'user_pass')
 
-    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?charset=utf8&passwd={passwd}&use_unicode=1')
-    # "use_unicode=1": see: https://sourceforge.net/tracker/?func=detail&aid=2837134&group_id=22307&atid=374932
+    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?passwd={passwd}&charset=utf8mb4')
     wrpylib.wrmwcache.update_wrreportcache(engine.connect())
 
 

diff --git a/scripts/updatewrsledruncache.py b/scripts/updatewrsledruncache.py
index 1a9fa65b49cddd984c0e83f77728b7bca13722ba..ab9928f77aaa9f581a63c0288c1fa638eedbda42 100755 (executable)
--- a/scripts/updatewrsledruncache.py
+++ b/scripts/updatewrsledruncache.py
@@ -32,8 +32,7 @@ def update_wrsledruncache(inifile):
     user = config.get('mysql', 'user_name')
     passwd = config.get('mysql', 'user_pass')
 
-    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?charset=utf8&passwd={passwd}&use_unicode=0')
-    # "use_unicode=0": see: https://sourceforge.net/tracker/?func=detail&aid=2837134&group_id=22307&atid=374932
+    engine = create_engine(f'mysql://{user}@{host}:3306/{dbname}?passwd={passwd}&charset=utf8mb4')
     wrpylib.wrmwcache.update_wrsledruncache(engine.connect())
 
 

diff --git a/tests/test_mwdb.py b/tests/test_mwdb.py
index 4e6dc8f859794bc66bb1e2b26a7235a2f9f59c64..3e94135ee48ed5713ad42d0d87de12b0fdb25415 100644 (file)
--- a/tests/test_mwdb.py
+++ b/tests/test_mwdb.py
@@ -10,7 +10,7 @@ import wrpylib.mwdb
 class TestMwDb(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+        cls.engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
         cls.Session = sessionmaker(bind=cls.engine)
 
     def setUp(self):
@@ -71,7 +71,7 @@ class TestMySqlPython(unittest.TestCase):
     This has been fixed in MySQL_python version 1.2.4."""
     @classmethod
     def setUpClass(cls):
-        cls.db = MySQLdb.connect(db='philipp_winterrodeln_wiki', charset='utf8', use_unicode=True)
+        cls.db = MySQLdb.connect(db='philipp_winterrodeln_wiki', charset='utf8mb4')
 
     def setUp(self):
         self.cursor = self.db.cursor()

diff --git a/tests/test_wrmwcache.py b/tests/test_wrmwcache.py
index fa15c8c92b60e46a6a75ca093b08677af905bc44..e4ea4b3e0c716fc3a8931ce76f44eb65cc789ef0 100644 (file)
--- a/tests/test_wrmwcache.py
+++ b/tests/test_wrmwcache.py
@@ -6,7 +6,7 @@ import unittest
 
 class TestWrMwCache(unittest.TestCase):
     def setUp(self):
-        self.engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+        self.engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
         self.connection = self.engine.connect()
 
     def tearDown(self):

diff --git a/tests/test_wrmwdb.py b/tests/test_wrmwdb.py
index 0c6949d5a125e3f50a21e350f07af425660abda3..30638b17ea5b2e48802b2b448d5b0d480edd1235 100644 (file)
--- a/tests/test_wrmwdb.py
+++ b/tests/test_wrmwdb.py
@@ -7,7 +7,7 @@ from sqlalchemy.orm import sessionmaker
 import wrpylib.wrmwdb
 
 
-TEST_DB = 'mysql://philipp@localhost:3306/philipp_winterrodeln_wrpylib_wrpylib_test?charset=utf8&use_unicode=1'
+TEST_DB = 'mysql://philipp@localhost:3306/philipp_winterrodeln_wrpylib_wrpylib_test?charset=utf8mb4'
 
 
 def setup_test_database():

diff --git a/wrpylib/mwdb.py b/wrpylib/mwdb.py
index 8da6ed6907f5dac3195293046f4d2e1fb9994876..79d36015d13e59c9088a36baaae8ef12b5ee1e5a 100644 (file)
--- a/wrpylib/mwdb.py
+++ b/wrpylib/mwdb.py
@@ -11,12 +11,12 @@ def page_table(metadata):
     return Table("page", metadata,
     Column("page_id", types.Integer, primary_key=True),
     Column("page_namespace", types.Integer, nullable=False),
-    Column("page_title", types.Unicode(255), nullable=False), # varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
+    Column("page_title", types.String(255, convert_unicode='force'), nullable=False),
     Column("page_restrictions", types.String, nullable=False), # tinyblob NOT NULL
     Column("page_is_redirect", types.Integer, nullable=False),
     Column("page_is_new", types.Integer, nullable=False),
     Column("page_random", types.Float, nullable=False),
-    Column("page_touched", types.String(14, convert_unicode='force'), nullable=False), # binary(14) NOT NULL
+    Column("page_touched", types.String(14, convert_unicode='force'), nullable=False),
     Column("page_latest", types.Integer, nullable=False),
     Column("page_len", types.Integer, nullable=False),
     Column("page_content_model", types.String(32, convert_unicode='force')),
@@ -35,8 +35,8 @@ def revision_table(metadata):
     Column("rev_text_id", types.Integer, nullable=False),
     Column("rev_comment", types.String(convert_unicode='force'), nullable=False), # tinyblob NOT NULL
     Column("rev_user", types.Integer, nullable=False),
-    Column("rev_user_text", types.Unicode(255), nullable=False), # varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
-    Column("rev_timestamp", types.String(14, convert_unicode='force'), nullable=False), # binary(14) NOT NULL
+    Column("rev_user_text", types.Unicode(255), nullable=False),
+    Column("rev_timestamp", types.String(14, convert_unicode='force'), nullable=False),
     Column("rev_minor_edit", types.Integer, nullable=False),
     Column("rev_deleted", types.Integer, nullable=False),
     Column("rev_len", types.Integer, nullable=False),
@@ -50,8 +50,8 @@ def text_table(metadata):
     """
     return Table("text", metadata,
     Column("old_id", types.Integer, primary_key=True),
-    Column("old_text", types.String(convert_unicode='force')), # mediumblob NOT NULL
-    Column("old_flags", types.String(convert_unicode='force')), # tinyblob NOT NULL
+    Column("old_text", types.String(convert_unicode='force')),
+    Column("old_flags", types.String(convert_unicode='force')),
     )
 
 
@@ -61,12 +61,12 @@ def user_table(metadata):
     """
     return Table('user', metadata,
     Column("user_id", types.Integer, primary_key=True),
-    Column("user_name", types.Unicode(255), nullable=False), # varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
-    Column("user_real_name", types.Unicode(255), nullable=False), # varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
-    Column("user_password", types.UnicodeText, nullable=False),  # tinyblob
+    Column("user_name", types.Unicode(255), nullable=False),
+    Column("user_real_name", types.Unicode(255), nullable=False),
+    Column("user_password", types.UnicodeText, nullable=False),
     # "user_newpassword"
     # "user_newpass_time"
-    Column("user_email", types.Unicode, nullable=False), # tinytext NOT NULL
+    Column("user_email", types.Unicode, nullable=False),
     # "user_touched"
     # "user_token"
     # "user_email_authenticated"
@@ -83,7 +83,7 @@ def categorylinks_table(metadata):
     """
     return Table("categorylinks", metadata,
     Column("cl_from", types.Integer, nullable=False, primary_key=True),
-    Column("cl_to", types.Unicode(255), nullable=False, primary_key=True), # varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
-    Column("cl_sortkey", types.String(convert_unicode='force'), nullable=False), # varbinary(230) NOT NULL
+    Column("cl_to", types.Unicode(255), nullable=False, primary_key=True),
+    Column("cl_sortkey", types.String(convert_unicode='force'), nullable=False),
     Column("cl_timestamp", types.DateTime, nullable=False),
     )

diff --git a/wrpylib/wrintermaps.py b/wrpylib/wrintermaps.py
index b7d54999151207216bb682bb7bda3849a9411ba7..f8d9291a2e4e28078488307880e8a5ee43052eb7 100644 (file)
--- a/wrpylib/wrintermaps.py
+++ b/wrpylib/wrintermaps.py
@@ -37,7 +37,7 @@ def update_wrintermapssledrun(session: Session, json_content: dict):
 
     >>> import json
     >>> from sqlalchemy.engine import create_engine
-    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
     >>> Session = orm.sessionmaker(bind=engine)
     >>> with open('2019-12-06_090500.json') as fp:
     >>>     content = json.load(fp)

diff --git a/wrpylib/wrmwcache.py b/wrpylib/wrmwcache.py
index b6283494441ead822984c5273b06cea602402308..81f6b4eae5389e8cdb14db12d6458bcdde1fe40e 100644 (file)
--- a/wrpylib/wrmwcache.py
+++ b/wrpylib/wrmwcache.py
@@ -15,7 +15,7 @@ def update_wrsledruncache(connection):
     is raised. No other exception type should be raised under normal circumstances.
     
     >>> from sqlalchemy.engine import create_engine
-    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
     >>> update_wrsledruncache(engine.connect())
     """
     metadata = schema.MetaData()
@@ -62,7 +62,7 @@ def update_wrinncache(connection):
     is raised. No other exception type should be raised under normal circumstances.
     
     >>> from sqlalchemy.engine import create_engine
-    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
     >>> update_wrinncache(engine.connect())
     """
     metadata = schema.MetaData()
@@ -108,7 +108,7 @@ def update_wrreportcache(connection, page_id=None):
         with the specified page_id. Use None for this parameter to update the whole table.
 
     >>> from sqlalchemy.engine import create_engine
-    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
     >>> update_wrreportcache(engine.connect())
     """
     metadata = schema.MetaData()
@@ -146,8 +146,8 @@ def update_wrmapcache(connection):
     is raised. No other exception type should be raised under normal circumstances.
     
     >>> from sqlalchemy.engine import create_engine
-    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
-    >>> # or: engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1&passwd=XXXXX')
+    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
+    >>> # or: engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4&passwd=XXXXX')
     >>> update_wrmapcache(engine.connect())
     """
     metadata = schema.MetaData()
@@ -220,8 +220,8 @@ def update_wrregioncache(connection):
     No exceptions should be raised under normal circumstances.
     
     >>> from sqlalchemy.engine import create_engine
-    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
-    >>> # or: engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1&passwd=XXXXX')
+    >>> engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
+    >>> # or: engine = create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4&passwd=XXXXX')
     >>> update_wrregioncache(engine.connect())
     """
     metadata = schema.MetaData()

diff --git a/wrpylib/wrorm.py b/wrpylib/wrorm.py
index 7e9a0bef21eea09a742e121d603c6e41bdd682ba..2cc2664e9ff33e34ae42edf3d47c4ce7dba8a85e 100644 (file)
--- a/wrpylib/wrorm.py
+++ b/wrpylib/wrorm.py
@@ -2,7 +2,7 @@
 Usage:
 >>> import sqlalchemy.orm
 >>> import sqlalchemy.engine
->>> engine = sqlalchemy.engine.create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8&use_unicode=1')
+>>> engine = sqlalchemy.engine.create_engine('mysql://philipp@localhost:3306/philipp_winterrodeln_wiki?charset=utf8mb4')
 >>> Session = sqlalchemy.orm.sessionmaker(bind=engine)
 >>> session = Session()
 """