Fix: Request.GetPostData() throws UnicodeEncodeError (cztomczak#517).

cztomczak · cztomczak · commit 8f1484c1e774 · 2020-01-11T12:31:31.000+01:00
diff --git a/src/cefpython.pyx b/src/cefpython.pyx
@@ -147,9 +147,11 @@ IF PY_MAJOR_VERSION == 2:
     from urllib import pathname2url as urllib_pathname2url
     # noinspection PyUnresolvedReferences
     from urllib import urlencode as urllib_urlencode
+    from urllib import quote as urlparse_quote
 ELSE:
     # noinspection PyUnresolvedReferences
     from urllib import parse as urlparse
+    from urllib.parse import quote as urlparse_quote
     # noinspection PyUnresolvedReferences
     from urllib.request import pathname2url as urllib_pathname2url
     # noinspection PyUnresolvedReferences
@@ -1027,7 +1029,8 @@ cpdef LoadCrlSetsFile(py_string path):
     CefLoadCRLSetsFile(PyToCefStringValue(path))
 
 cpdef GetDataUrl(data, mediatype="html"):
-    html = data.encode("utf-8", "replace")
-    b64 = base64.b64encode(html).decode("utf-8", "replace")
+    if PY_MAJOR_VERSION >= 3:
+        data = data.encode("utf-8", "replace")
+    b64 = base64.b64encode(data).decode("utf-8", "replace")
     ret = "data:text/html;base64,{data}".format(data=b64)
     return ret
diff --git a/src/request.pyx b/src/request.pyx
@@ -119,8 +119,16 @@ cdef class PyRequest:
                     retMultipart.append(pyData)
                 else:
                     # Content-Type: application/x-www-form-urlencoded
-                    retUrlEncoded.update(urlparse.parse_qsl(qs=pyData, 
+                    quoted = urlparse_quote(pyData, safe="=")
+                    retUrlEncoded.update(urlparse.parse_qsl(qs=quoted,
                             keep_blank_values=True))
+                    if PY_MAJOR_VERSION >= 3:
+                        retUrlEncoded_copy = copy.deepcopy(retUrlEncoded)
+                        retUrlEncoded = dict()
+                        for key in retUrlEncoded_copy:
+                            retUrlEncoded[key.encode("utf-8", "replace")] =\
+                                    retUrlEncoded_copy[key].encode(
+                                                    "utf-8", "replace")
             elif postDataElement.get().GetType() == cef_types.PDE_TYPE_FILE:
                 pyFile = CefToPyBytes(postDataElement.get().GetFile())
                 retMultipart.append(b"@"+pyFile)
@@ -156,7 +164,7 @@ cdef class PyRequest:
                 postData.get().AddElement(postDataElement)
             self.GetCefRequest().get().SetPostData(postData)
         elif type(pyPostData) == dict:
-            pyElement = urllib_urlencode(pyPostData).encode()
+            pyElement = urllib_urlencode(pyPostData).encode("utf-8", "replace")
             postDataElement = CefPostDataElement_Create()
             postDataElement.get().SetToBytes(len(pyElement), <char*>pyElement)
             postData.get().AddElement(postDataElement)
diff --git a/unittests/issue517.py b/unittests/issue517.py
@@ -0,0 +1,43 @@
+# coding=utf8
+from cefpython3 import cefpython as cef
+import sys
+
+html = """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<script>
+  window.onload = function() {
+   fetch('http://127.0.0.1:8000', {
+       method: 'POST',
+       headers: {'Content-Type': 'application/x-www-form-urlencoded'},
+       body: 'key=' + encodeURI('&#127843; asd'),
+   }).then().catch();
+  }
+</script>
+</body>
+</html>
+"""
+
+
+class RequestHandler:
+    def GetResourceHandler(self, browser, frame, request):
+        print(request.GetPostData())
+        return None
+
+def main():
+    sys.excepthook = cef.ExceptHook
+    cef.Initialize()
+    browser = cef.CreateBrowserSync(url=cef.GetDataUrl(html))
+    browser.SetClientHandler(RequestHandler())
+    cef.MessageLoop()
+    del browser
+    cef.Shutdown()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/unittests/main_test.py b/unittests/main_test.py
@@ -235,7 +235,7 @@ def test_main(self):
         req_file = os.path.dirname(os.path.abspath(__file__))
         req_file = os.path.join(req_file, "main_test.py")
         if sys.version_info.major > 2:
-            req_file = req_file.encode()
+            req_file = req_file.encode("utf-8")
         req_data = [b"--key=value", b"@"+req_file]
         req.SetMethod("POST")
         req.SetPostData(req_data)
