起源:
所下载视频,有音视频分离者,需要合并起来,采用python之subprocess.Popen()调用ffmpeg实现。python版本为2.7.13,而音视频文件路径,有unicode字符者,合并失败。
此问题由来已久,终于不忍受,用尽工夫寻其机现,终于寻得蛛丝蚂迹,完成其修复。
其原因为:python 2.7.x中subprocess.Popen()函数,最终调用了kernel32.dll中的CreateProcess函数Ansi版本CreateProcessA,传非Ansi参数给它会被它拒绝,而触发异常。
测试代码如下:
# encoding: utf-8 from __future__ import unicode_literals import subprocess file_path = r'D:\Percy Faith [CA US] by chkjns ♫ 175 songs\v.txt' args = u'notepad "%s"' % file_path try: p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() if p.returncode != 0: stderr = stderr.decode('utf-8', 'replace') print stderr except Exception as e: print e.message
其异常为UnicodeEncodeError,表现字串为:'ascii' codec can't encode character u'\u266b' in position 42: ordinal not in range(128)
1、Python 2.x先天缺陷
此问题百度不到有效答案,于是上stackoverlow,一路摸去,找到了python官方对此bug描述,其链接如下:
Issue 19264: subprocess.Popen doesn't support unicode on Windows - Python tracker
七嘴八舌众说纷纭,但总算大致捋清原委,即上述其调用了Ansi版本的CreateProcess所致。循此原因,是否替换为Unicode函数CreateProcessW即可?
最后一条回复:
msg289664 - (view) Author: Valentin LAB (Valentin LAB) Date: 2017-03-15 10:39
给出了折中方案,其方案即如所思,做函数层替换。
2、win_subprocess.py
Valentin LAB这哥们,就在github上开放了源码,解决此问题。其核心为win_subprocess.py,内容如下:
(其代码中有用os,他却忘记import,贴代码时已做修正)
## issue: https://bugs.python.org/issue19264 import ctypes import subprocess import _subprocess import os from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \ Structure, sizeof, c_wchar, WinError from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \ HANDLE ## ## Types ## CREATE_UNICODE_ENVIRONMENT = 0x00000400 LPCTSTR = c_char_p LPTSTR = c_wchar_p LPSECURITY_ATTRIBUTES = c_void_p LPBYTE = ctypes.POINTER(BYTE) class STARTUPINFOW(Structure): _fields_ = [ ("cb", DWORD), ("lpReserved", LPWSTR), ("lpDesktop", LPWSTR), ("lpTitle", LPWSTR), ("dwX", DWORD), ("dwY", DWORD), ("dwXSize", DWORD), ("dwYSize", DWORD), ("dwXCountChars", DWORD), ("dwYCountChars", DWORD), ("dwFillAtrribute", DWORD), ("dwFlags", DWORD), ("wShowWindow", WORD), ("cbReserved2", WORD), ("lpReserved2", LPBYTE), ("hStdInput", HANDLE), ("hStdOutput", HANDLE), ("hStdError", HANDLE), ] LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW) class PROCESS_INFORMATION(Structure): _fields_ = [ ("hProcess", HANDLE), ("hThread", HANDLE), ("dwProcessId", DWORD), ("dwThreadId", DWORD), ] LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION) class DUMMY_HANDLE(ctypes.c_void_p): def __init__(self, *a, **kw): super(DUMMY_HANDLE, self).__init__(*a, **kw) self.closed = False def Close(self): if not self.closed: windll.kernel32.CloseHandle(self) self.closed = True def __int__(self): return self.value CreateProcessW = windll.kernel32.CreateProcessW CreateProcessW.argtypes = [ LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES, LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR, LPSTARTUPINFOW, LPPROCESS_INFORMATION, ] CreateProcessW.restype = BOOL ## ## Patched functions/classes ## def CreateProcess(executable, args, _p_attr, _t_attr, inherit_handles, creation_flags, env, cwd, startup_info): """Create a process supporting unicode executable and args for win32 Python implementation of CreateProcess using CreateProcessW for Win32 """ si = STARTUPINFOW( dwFlags=startup_info.dwFlags, wShowWindow=startup_info.wShowWindow, cb=sizeof(STARTUPINFOW), ## XXXvlab: not sure of the casting here to ints. hStdInput=int(startup_info.hStdInput), hStdOutput=int(startup_info.hStdOutput), hStdError=int(startup_info.hStdError), ) wenv = None if env is not None: ## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar env = (unicode("").join([ unicode("%s=%s\0") % (k, v) for k, v in env.items()])) + unicode("\0") wenv = (c_wchar * len(env))() wenv.value = env pi = PROCESS_INFORMATION() creation_flags |= CREATE_UNICODE_ENVIRONMENT if CreateProcessW(executable, args, None, None, inherit_handles, creation_flags, wenv, cwd, byref(si), byref(pi)): return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread), pi.dwProcessId, pi.dwThreadId) raise WinError() class Popen(subprocess.Popen): """This superseeds Popen and corrects a bug in cPython 2.7 implem""" def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Code from part of _execute_child from Python 2.7 (9fbb65e) There are only 2 little changes concerning the construction of the the final string in shell mode: we preempt the creation of the command string when shell is True, because original function will try to encode unicode args which we want to avoid to be able to sending it as-is to ``CreateProcess``. """ if not isinstance(args, subprocess.types.StringTypes): args = subprocess.list2cmdline(args) if startupinfo is None: startupinfo = subprocess.STARTUPINFO() if shell: startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = _subprocess.SW_HIDE comspec = os.environ.get("COMSPEC", unicode("cmd.exe")) args = unicode('{} /c "{}"').format(comspec, args) if (_subprocess.GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com"): w9xpopen = self._find_w9xpopen() args = unicode('"%s" %s') % (w9xpopen, args) creationflags |= _subprocess.CREATE_NEW_CONSOLE super(Popen, self)._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, False, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) _subprocess.CreateProcess = CreateProcess
3、使用方法
若已在.py文件中引入unicode标记(建议自有项目,皆加以unicode支持):
from __future__ import unicode_literals
那么,直接import win_subprocess就行,其代码中,已以自定义CreateProcess替换了_subprocess.CreateProcess同名函数。
当然,也可直接以win_subprocess.Popen()调用。
但是,经过验证,直接引用win_subprocess就能很好工作了,因此推荐直接引过去就行。
参考资料:
Fixing python 2.7 windows unicode issue with 'subprocess.Popen'