移除重复字符的几个算法简单比较

 

    class Program
    {
        static void Main(string[] args)
        {
            string s = File.ReadAllText(@"e:\test.txt");
            Program p = new Program();
            string r = p.RemoveDuplicatedChar_1(s);
            r = p.RemoveDuplicatedChar_2(s);
            r = p.RemoveDuplicatedChar_6(s);
        }

        /// <summary>
        /// 既然是C#,首先应该想到的是利用类库现有的实现
        /// HashSet是理智的选择，要注意这是3.5才有的类
        /// </summary>
        public string RemoveDuplicatedChar_1(string s)
        {
            if (string.IsNullOrEmpty(s) || s.Length < 2)
            {
                return s;
            }

            HashSet<char> hs = new HashSet<char>(s.ToCharArray());
            return new string(hs.ToArray());
        }

        /// <summary>
        /// Distinct函数也可以做到
        /// </summary>
        public string RemoveDuplicatedChar_2(string s)
        {
            if (string.IsNullOrEmpty(s) || s.Length < 2)
            {
                return s;
            }

            return new string(s.Distinct().ToArray());
        }

        //Distinct的实现使用到了链表和哈希，感兴趣的可以参考一下实现
        /****
                [__DynamicallyInvokable]
                public static IEnumerable<TSource> Distinct<TSource>(this IEnumerable<TSource> source)
                {
                    if (source == null)
                    {
                        throw Error.ArgumentNull("source");
                    }
                    return DistinctIterator<TSource>(source, null);
                }

                private static IEnumerable<TSource> DistinctIterator<TSource>(IEnumerable<TSource> source, IEqualityComparer<TSource> comparer)
                {
                    Set<TSource> iteratorVariable0 = new Set<TSource>(comparer);
                    foreach (TSource iteratorVariable1 in source)
                    {
                        if (iteratorVariable0.Add(iteratorVariable1))
                        {
                            yield return iteratorVariable1;
                        }
                    }
                }

                public bool Add(TElement value)
                {
                    return !this.Find(value, true);
                }

                private bool Find(TElement value, bool add)
                {
                    int hashCode = this.InternalGetHashCode(value);
                    for (int i = this.buckets[hashCode % this.buckets.Length] - 1; i >= 0; i = this.slots[i].next)
                    {
                        if ((this.slots[i].hashCode == hashCode) && this.comparer.Equals(this.slots[i].value, value))
                        {
                            return true;
                        }
                    }
                    if (add)
                    {
                        int freeList;
                        if (this.freeList >= 0)
                        {
                            freeList = this.freeList;
                            this.freeList = this.slots[freeList].next;
                        }
                        else
                        {
                            if (this.count == this.slots.Length)
                            {
                                this.Resize();
                            }
                            freeList = this.count;
                            this.count++;
                        }
                        int index = hashCode % this.buckets.Length;
                        this.slots[freeList].hashCode = hashCode;
                        this.slots[freeList].value = value;
                        this.slots[freeList].next = this.buckets[index] - 1;
                        this.buckets[index] = freeList + 1;
                    }
                    return false;
                }
         ****/

        /// <summary>
        /// 别忘了我们是在做题！所以假设来了
        /// 第一个假设：字符存在范围'a'-'z'
        /// 于是26个字母可以用32位的整型值来影射
        /// </summary>
        public string RemoveDuplicatedChar_3(string s)
        {
            if (string.IsNullOrEmpty(s) || s.Length < 2)
            {
                return s;
            }

            char[] charArray = s.ToCharArray();
            Int32 flags = 0;
            int newIndex = 0;
            for (int i = 0; i < charArray.Length; i++)
            {
                if (charArray[i] < 'a' || charArray[i] > 'z')
                {
                    throw new ArgumentException("char should be in range(a-z)");
                }

                int relative_position = (charArray[i]-'a') % 32;
                if ((flags & (1 << relative_position))==0)
                {
                    charArray[newIndex] = charArray[i];
                    newIndex++;
                    flags |= (1 << relative_position);
                }
            }

            return new string(charArray, 0, newIndex);
        }

        /// <summary>
        /// 256个ASCII码可以用8个32位整型值映射
        /// </summary>
        public string RemoveDuplicatedChar_4(string s)
        {
            if (string.IsNullOrEmpty(s) || s.Length < 2)
            {
                return s;
            }

            char[] charArray = s.ToCharArray();
            Int32[] flags = new Int32[8];
            int newIndex = 0;

            for (int i = 0; i < charArray.Length; i++)
            {
                if (charArray[i]>255)
                {
                    throw new ArgumentException("char should be in ASCII");
                }

                int index = charArray[i] / 32;
                int relative_position = charArray[i] % 32;
                if ((flags[index] & (1 << relative_position)) == 0)
                {
                    charArray[newIndex]=charArray[i];
                    newIndex++;
                    flags[index] |= (1 << relative_position);
                }
            }

            return new string(charArray, 0, newIndex);
        }

        /// <summary>
        /// 似曾相识
        /// </summary>
        public string RemoveDuplicatedChar_5(string s)
        {
            if (string.IsNullOrEmpty(s) || s.Length < 2)
            {
                return s;
            }

            char[] charArray = s.ToCharArray();
            bool[] flags = new bool[256];
            int newIndex = 0;

            for (int i = 0; i < charArray.Length; i++)
            {
                if (charArray[i] > 255)
                {
                    throw new ArgumentException("char should be in ASCII");
                }

                char c = charArray[i];
                if (!flags[c])
                {
                    charArray[newIndex] = charArray[i];
                    newIndex++;
                    flags[c] = true;
                }
            }

            return new string(charArray, 0, newIndex);
        }

        /// <summary>
        /// O(n平方)的实现，没有借助额外的buffer数组或高级数据结构
        /// 不用考虑是否为ASCII，在实际使用中是唯一有通用意义的算法，
        /// 但是至于在处理大字符串时其时间效率应该要远低于使用第一个和第二个方法
        /// 稍后给出运行时间比较
        /// </summary>
        public string RemoveDuplicatedChar_6(string s)
        {
            if (string.IsNullOrEmpty(s) || s.Length < 2)
            {
                return s;
            }

            char[] charArray = s.ToCharArray();
            int newIndex = 1;

            for (int i = 1; i < charArray.Length; i++)
            {
                int j = 0;
                for (j = 0; j < newIndex; j++)
                {
                    if (charArray[i] == charArray[j])
                    {
                        break;
                    }
                }

                //no duplcaited char found in existed read sub-string
                //then set the new char as the new index
                if (j == newIndex)
                {
                    charArray[newIndex] = charArray[i];
                    newIndex++;
                }
            }

            return new string(charArray, 0, newIndex);
        }
    }

在处理百万级别的文本时，1,2,6的运行时间比较：

/****
对于纯ASCII的大字符串，给出了5个方法的时间比较如下图
可以看出4,5在时间效率上超过了1和2
所以在处理混合型字符串时，是否应该考虑综合应用这些算法呢？
答案是明显的
****/