Python 源码剖析（三）【字符串对象】
三、字符串对象
1、PyStringObject与PyString_Type
2、创建PyStringObject对象
3、Intern 机制
4、字符缓冲池
5、PyStringObject 效率相关问题
6、Hack PyStringObject
1、PyStringObject与PyString_Type
PyStringObject 对象的声明：
[stringobject.h] typedef struct { PyObject_VAR_HEAD long ob_shash; int ob_sstate; char ob_sval[1]; } PyStringObject;
PyObject_VAR_HEAD 中含有字符串长度ob_size，ob_sval指向字符串存储内存，大小为ob_size+1，且ob_sval[ob_size]='\0'；
ob_shash保存字符串哈希值，没计算过的默认为0，计算方法为：
[stringobject.c] static long string_hash(PyStringObject *a) { register int len; register unsigned char *p; register long x; if (a->ob_shash != -1) return a->ob_shash; len = a->ob_size; p = (unsigned char *) a->ob_sval; x = *p << 7; while (--len >= 0) x = (1000003*x) ^ *p++; x ^= a->ob_size; if (x == -1) x = -2; a->ob_shash = x; return x; }
ob_sstate表示该对象是否被Intern。
PyStringObject对应的类型对象：
[stringobject.c] PyTypeObject PyString_Type = { PyObject_HEAD_INIT(&PyType_Type) 0, "str", sizeof(PyStringObject), sizeof(char), …… (reprfunc)string_repr, /* tp_repr */ &string_as_number, /* tp_as_number */ &string_as_sequence, /* tp_as_sequence */ &string_as_mapping, /* tp_as_mapping */ (hashfunc)string_hash, /* tp_hash */ 0, /* tp_call */ …… string_new, /* tp_new */ PyObject_Del, /* tp_free */ };
2、创建PyStringObject对象
最一般的方法PyString_FromString：
[stringobject.c] PyObject * PyString_FromString(const char *str) { register size_t size; register PyStringObject *op; assert(str != NULL); /*判断字符串长度*/ size = strlen(str); if (size > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "string is too long for a Python string"); return NULL; } /*处理null string*/ if (size == 0 && (op = nullstring) != NULL) { #ifdef COUNT_ALLOCS null_strings++; #endif Py_INCREF(op); return (PyObject *)op; } if (size == 1 && (op = characters[*str & UCHAR_MAX]) != NULL) { #ifdef COUNT_ALLOCS one_strings++; #endif Py_INCREF(op); return (PyObject *)op; } /* 创建新的PyStringObject对象，并初始化 */ /* Inline PyObject_NewVar */ op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size); if (op == NULL) return PyErr_NoMemory(); PyObject_INIT_VAR(op, &PyString_Type, size); op->ob_shash = -1; op->ob_sstate = SSTATE_NOT_INTERNED; memcpy(op->ob_sval, str, size+1); /* Itern（共享）长度较短的PyStringObject对象 */ if (size == 0) { PyObject *t = (PyObject *)op; PyString_InternInPlace(&t); op = (PyStringObject *)t; nullstring = op; Py_INCREF(op); } else if (size == 1) { PyObject *t = (PyObject *)op; PyString_InternInPlace(&t); op = (PyStringObject *)t; characters[*str & UCHAR_MAX] = op; Py_INCREF(op); } return (PyObject *) op; }
首先判断有没有超过长度限制（INT_MAX 2147483647 ），然后判断是不是为空串，如果是并且之前创建过空串对象就直接返回，接着就判断其是否为一个字符，是而且在Intern机制中创建过就直接返回，否则，就开始申请内存（PyStringObject + size），创建对象并返回。
另一个创建PyStringObject对象的途径：
[stringobject.c] PyObject* PyString_FromStringAndSize(const char *str, int size) { register PyStringObject *op; /*处理null string*/ if (size == 0 && (op = nullstring) != NULL) { #ifdef COUNT_ALLOCS null_strings++; #endif Py_INCREF(op); return (PyObject *)op; } if (size == 1 && str != NULL && (op = characters[*str & UCHAR_MAX]) != NULL) { #ifdef COUNT_ALLOCS one_strings++; #endif Py_INCREF(op); return (PyObject *)op; } /* Inline PyObject_NewVar */ op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size); if (op == NULL) return PyErr_NoMemory(); PyObject_INIT_VAR(op, &PyString_Type, size); op->ob_shash = -1; op->ob_sstate = SSTATE_NOT_INTERNED; if (str != NULL) memcpy(op->ob_sval, str, size); op->ob_sval[size] = '\0'; /* share short strings */ if (size == 0) { PyObject *t = (PyObject *)op; PyString_InternInPlace(&t); op = (PyStringObject *)t; nullstring = op; Py_INCREF(op); } else if (size == 1 && str != NULL) { PyObject *t = (PyObject *)op; PyString_InternInPlace(&t); op = (PyStringObject *)t; characters[*str & UCHAR_MAX] = op; Py_INCREF(op); } return (PyObject *) op; }
和前一个差不多。
3、Intern 机制
现在看下前面提到的Intern机制，即创建PyStringObject对象函数里的PyString_InternInPlace：
[stringobjec.c] void PyString_InternInPlace(PyObject **p) { register PyStringObject *s = (PyStringObject *)(*p); PyObject *t; if (s == NULL || !PyString_Check(s)) Py_FatalError("PyString_InternInPlace: strings only please!"); /* If it's a string subclass, we don't really know what putting it in the interned dict might do. */ if (!PyString_CheckExact(s)) return; if (PyString_CHECK_INTERNED(s)) return; if (interned == NULL) { interned = PyDict_New(); if (interned == NULL) { PyErr_Clear(); /* Don't leave an exception */ return; } } t = PyDict_GetItem(interned, (PyObject *)s); if (t) { Py_INCREF(t); Py_DECREF(*p); *p = t; return; } if (PyDict_SetItem(interned, (PyObject *)s, (PyObject *)s) < 0) { PyErr_Clear(); return; } /* The two references in interned are not counted by refcnt. The string deallocator will take care of this */ s->ob_refcnt -= 2; PyString_CHECK_INTERNED(s) = SSTATE_INTERNED_MORTAL; }
首先进行一系列检测，而核心在于 interned（PyDictObject对象），如果创建对象在interned中，增加interned中对象的引用计数，减少传入对象的引用计数（其为临时变量）；如果创建对象不在interned中，将创建对象加入interned，并将其引用计数减2（interned中对象的引用应当无效，否则无法删除）。对象引用计数为0时会被销毁：
[stringobject.c] static void string_dealloc(PyObject *op) { switch (PyString_CHECK_INTERNED(op)) { case SSTATE_NOT_INTERNED: break; case SSTATE_INTERNED_MORTAL: /* revive dead object temporarily for DelItem */ op->ob_refcnt = 3; if (PyDict_DelItem(interned, op) != 0) Py_FatalError( "deletion of interned string failed"); break; case SSTATE_INTERNED_IMMORTAL: Py_FatalError("Immortal interned string died."); default: Py_FatalError("Inconsistent interned string state."); } op->ob_type->tp_free(op); }
4、字符缓冲池
PyStringObject 为一个字节的字符对象设计了一个类似PyIntObject小整数对象池一样的对象池characters：
static PyStringObject *characters[UCHAR_MAX + 1];
#define UCHAR_MAX 0xff /* maximum unsigned char value */
由字符串创建的代码可看出characters缓冲池在字符串生成时才开始生成。
虽然在创建PyStringObject时Intern机制只作用于 空串和字符（对应nullstring和characters），但Intern机制会作用到其他地方。
5、PyStringObject 效率相关问题
字符串连接问题，用 '+' 效率低下，连接N个PyStringObject对象将进行N-1次内存申请及搬运，推荐使用join（只分配一次内存）。
'+' 调用string_concat：
[stringobject.c] static PyObject* string_concat(register PyStringObject *a, register PyObject *bb) { register unsigned int size; register PyStringObject *op; #define b ((PyStringObject *)bb) …… size = a->ob_size + b->ob_size; /* Inline PyObject_NewVar */ op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size); if (op == NULL) return PyErr_NoMemory(); PyObject_INIT_VAR(op, &PyString_Type, size); op->ob_shash = -1; op->ob_sstate = SSTATE_NOT_INTERNED; memcpy(op->ob_sval, a->ob_sval, (int) a->ob_size); memcpy(op->ob_sval + a->ob_size, b->ob_sval, (int) b->ob_size); op->ob_sval[size] = '\0'; return (PyObject *) op; #undef b }
而join则这样（先计算处list或tuple中PyStringObject的总大小再申请一次内存）：
[stringobject.c] static PyObject* string_join(PyStringObject *self, PyObject *orig) { char *sep = PyString_AS_STRING(self); const int seplen = PyString_GET_SIZE(self); PyObject *res = NULL; char *p; int seqlen = 0; size_t sz = 0; int i; PyObject *seq, *item; 。。。。。。//获得list中PyStringObject对象的个数，保存在seqlen中 for (i = 0; i < seqlen; i++) { const size_t old_sz = sz; item = PySequence_Fast_GET_ITEM(seq, i); sz += PyString_GET_SIZE(item); if (i != 0) sz += seplen; } /* 申请内存空间 */ res = PyString_FromStringAndSize((char*)NULL, (int)sz); /* 连接list中的每一个PyStringObject对象*/ p = PyString_AS_STRING(res); for (i = 0; i < seqlen; ++i) { size_t n; /* 获得list中的一个PyStringObject对象*/ item = PySequence_Fast_GET_ITEM(seq, i); n = PyString_GET_SIZE(item); memcpy(p, PyString_AS_STRING(item), n); p += n; if (i < seqlen - 1) { memcpy(p, sep, seplen); p += seplen; } } Py_DECREF(seq); return res; }
6、Hack PyStringObject
可在string_length中添加打印地址代码，运行len()时可观察到相同的字符串地址一样，这就是Intern机制的作用；可在len()中添加代码：
static void ShowCharater() { char a = 'a'; PyStringObject** posA = characters+(unsigned short)a; int i; for(i = 0; i < 5; ++i) { PyStringObject* strObj = posA[i]; printf("%s, %d\n", strObj->ob_sval, strObj->ob_refcnt); } }
观察是否使用缓冲池对象。
