C#根据url生成唯一的key

根据url生成唯一的idkey,记录并分享:

    public class UrlToUniqueKey
    {
        private static Hashtable generators = new Hashtable(10);
        public static long[,] polynomials = new long[65, 2]
    {
      {
        0L,
        0L
      },
      {
        -4611686018427387904L,
        -4611686018427387904L
      },
      {
        -2305843009213693952L,
        -2305843009213693952L
      },
      {
        -3458764513820540928L,
        -5764607523034234880L
      },
      {
        -576460752303423488L,
        -576460752303423488L
      },
      {
        -1441151880758558720L,
        -4899916394579099648L
      },
      {
        -2738188573441261568L,
        -5332261958806667264L
      },
      {
        -1945555039024054272L,
        -1945555039024054272L
      },
      {
        -7602076171001397248L,
        -3134505340649865216L
      },
      {
        -9169328841326329856L,
        -8628896886041870336L
      },
      {
        -5719571526760529920L,
        -1612288666598637568L
      },
      {
        -2742692173068632064L,
        -3913628076184961024L
      },
      {
        -1776670052997660672L,
        -2407174000829530112L
      },
      {
        -2005227734086713344L,
        -7718043861406187520L
      },
      {
        -3112550292466434048L,
        -5147051424131055616L
      },
      {
        -2113595600120315904L,
        -7045600142044430336L
      },
      {
        -3682677857793867776L,
        -7529033414544982016L
      },
      {
        -2812286861060341760L,
        -6211097204841512960L
      },
      {
        -7338580408428134400L,
        -181868019327172608L
      },
      {
        -4070603152259284992L,
        -7580349821236543488L
      },
      {
        -4690912428278415360L,
        -1656541810593366016L
      },
      {
        -8730258663983284224L,
        -1124153882377715712L
      },
      {
        -7455702586042089472L,
        -7995369281233616896L
      },
      {
        -58297206016311296L,
        -5584750510474264576L
      },
      {
        -8198983991290757120L,
        -6900195776585007104L
      },
      {
        -3368309616348758016L,
        -777578196575977472L
      },
      {
        -7950798515817414656L,
        -5274165001345564672L
      },
      {
        -1814110279349305344L,
        -3073563053254508544L
      },
      {
        -7333090856108294144L,
        -299958969764413440L
      },
      {
        -5862047463258456064L,
        -1088406921112715264L
      },
      {
        -2699705949936943104L,
        -6180435270106611712L
      },
      {
        -5010893113942605824L,
        -745326200523587584L
      },
      {
        -7247248171544346624L,
        -7637035222663430144L
      },
      {
        -379617898928275456L,
        -2128952956840574976L
      },
      {
        -8982278211127738368L,
        -8328970742175629312L
      },
      {
        -2699136824088985600L,
        -657743201922187264L
      },
      {
        -5723859659287166976L,
        -3191693334452109312L
      },
      {
        -5937852116755283968L,
        -8585462099679903744L
      },
      {
        -4340513297460625408L,
        -2535761738024878080L
      },
      {
        -7223736098545991680L,
        -5810321976182439936L
      },
      {
        -8771886216598519808L,
        -8198507976365965312L
      },
      {
        -1991715210309664768L,
        -3768584310714531840L
      },
      {
        -9162551332419141632L,
        -8616797614972600320L
      },
      {
        -719027294788648960L,
        -6649233720883544064L
      },
      {
        -3559483092657242112L,
        -7475962079365038080L
      },
      {
        -713657193466691584L,
        -3981624362359062528L
      },
      {
        -2043424698022887424L,
        -3853773324916752384L
      },
      {
        -4041900042031005696L,
        -2750302262364798976L
      },
      {
        -4436928417950760960L,
        -8365357674703716352L
      },
      {
        -5903553087924551680L,
        -232537924545363968L
      },
      {
        -8728448178608234496L,
        -4183444001187078144L
      },
      {
        -2341229233224585216L,
        -6731777037870903296L
      },
      {
        -1649801635265304576L,
        -2988673641163274240L
      },
      {
        -7620924682850272256L,
        -4999586984279010304L
      },
      {
        -6766504171168936448L,
        -3233736554084598272L
      },
      {
        -4307058212505990400L,
        -7352830162319492864L
      },
      {
        -6559722688844616064L,
        -3544821563555732608L
      },
      {
        -5503660899170514624L,
        -9177641346571454016L
      },
      {
        -187437317177603296L,
        -1362648669985435232L
      },
      {
        -24562796859796592L,
        -1298422235250900464L
      },
      {
        -3825548837209328312L,
        -4007863133653759080L
      },
      {
        -5023278451590057620L,
        -8135395730479815484L
      },
      {
        -1370071926941500434L,
        -7957612307809224958L
      },
      {
        -2163743556238479977L,
        -2346333074831914631L
      },
      {
        -2935493925047926053L,
        -7378668602973054136L
      }
    };
        public static UrlToUniqueKey std64 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[64, 0], 64);
        public static UrlToUniqueKey std32 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[32, 0], 32);
        public static UrlToUniqueKey std40 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[40, 0], 40);
        public static UrlToUniqueKey std24 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[24, 0], 24);
        private long zero = 0L;
        private long one = long.MinValue;
        public long empty;
        public int degree;
        public long polynomial;
        private long[,] ByteModTable;

        private UrlToUniqueKey(long polynomial, int degree)
        {
            this.degree = degree;
            this.polynomial = polynomial;
            this.ByteModTable = new long[16, 256];
            long[] numArray = new long[128];
            long num1 = long.MinValue;
            long num2 = this.rightMove(long.MinValue, degree - 1);
            for (int index = 0; index < 128; ++index)
            {
                numArray[index] = num1;
                bool flag = (num1 & num2) != 0L;
                num1 = this.rightMove(num1, 1);
                if (flag)
                    num1 ^= polynomial;
            }
            this.empty = numArray[64];
            for (int index1 = 0; index1 < 16; ++index1)
            {
                for (int index2 = 0; index2 < 256; ++index2)
                {
                    long num3 = 0L;
                    for (int index3 = 0; index3 < 8; ++index3)
                    {
                        if ((index2 & 1 << index3) != 0)
                            num3 ^= numArray[(int)sbyte.MaxValue - index1 * 8 - index3];
                    }
                    this.ByteModTable[index1, index2] = num3;
                }
            }
        }

        public static UrlToUniqueKey make(long polynomial, int degree)
        {
            long num = polynomial;
            UrlToUniqueKey heritrix = (UrlToUniqueKey)UrlToUniqueKey.generators[(object)1];
            if (heritrix == null)
            {
                heritrix = new UrlToUniqueKey(polynomial, degree);
                UrlToUniqueKey.generators.Add((object)num, (object)heritrix);
            }
            return heritrix;
        }

        private long rightMove(long value, int pos)
        {
            if (pos != 0)
            {
                long num = long.MaxValue;
                value >>= 1;
                value &= num;
                value >>= pos - 1;
            }
            return value;
        }

        public static long long_rightMove(long value, int pos)
        {
            if (pos != 0)
            {
                long num = long.MaxValue;
                value >>= 1;
                value &= num;
                value >>= pos - 1;
            }
            return value;
        }

        private int int_rightMove(int value, int pos)
        {
            if (pos != 0)
            {
                int num = int.MaxValue;
                value >>= 1;
                value &= num;
                value >>= pos - 1;
            }
            return value;
        }

        public long reduce(long fp)
        {
            int num1 = 8 - this.degree / 8;
            long num2 = num1 == 8 ? 0L : fp & -1L << 8 * num1;
            long num3 = 0L;
            for (int index = 0; index < num1; ++index)
            {
                num3 ^= this.ByteModTable[8 + index, (int)fp & (int)byte.MaxValue];
                fp = this.rightMove(fp, 8);
            }
            return num2 ^ num3;
        }

        public long extend_byte(long f, int v)
        {
            f ^= (long)((int)byte.MaxValue & v);
            int num = (int)f;
            return this.rightMove(f, 8) ^ this.ByteModTable[7, num & (int)byte.MaxValue];
        }

        public long extend_char(long f, int v)
        {
            f ^= (long)((int)ushort.MaxValue & v);
            int num = (int)f;
            return this.rightMove(f, 16) ^ this.ByteModTable[6, num & (int)byte.MaxValue] ^ this.ByteModTable[7, this.int_rightMove(num, 8) & (int)byte.MaxValue];
        }

        public long extend_int(long f, int v)
        {
            f ^= (long)uint.MaxValue & (long)v;
            int num1 = (int)f;
            long num2 = this.rightMove(f, 32) ^ this.ByteModTable[4, num1 & (int)byte.MaxValue];
            int num3 = this.int_rightMove(num1, 8);
            long num4 = num2 ^ this.ByteModTable[5, num3 & (int)byte.MaxValue];
            int num5 = this.int_rightMove(num3, 8);
            return num4 ^ this.ByteModTable[6, num5 & (int)byte.MaxValue] ^ this.ByteModTable[7, this.int_rightMove(num5, 8) & (int)byte.MaxValue];
        }

        public long extend_long(long f, long v)
        {
            f ^= v;
            long num1 = this.ByteModTable[0, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            long num2 = num1 ^ this.ByteModTable[1, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            long num3 = num2 ^ this.ByteModTable[2, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            long num4 = num3 ^ this.ByteModTable[3, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            long num5 = num4 ^ this.ByteModTable[4, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            long num6 = num5 ^ this.ByteModTable[5, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            long num7 = num6 ^ this.ByteModTable[6, (int)(f & (long)byte.MaxValue)];
            f = this.rightMove(f, 8);
            return num7 ^ this.ByteModTable[7, (int)(f & (long)byte.MaxValue)];
        }

        public long fp(byte[] buf, int start, int n)
        {
            return this.extend(this.empty, buf, start, n);
        }

        public long fp(char[] buf, int start, int n)
        {
            return this.extend(this.empty, buf, start, n);
        }

        public long fp(string s)
        {
            return this.extend(this.empty, s);
        }

        public long fp(int[] buf, int start, int n)
        {
            return this.extend(this.empty, buf, start, n);
        }

        public long fp(long[] buf, int start, int n)
        {
            return this.extend(this.empty, buf, start, n);
        }

        public long fp8(string s)
        {
            return this.extend8(this.empty, s);
        }

        public long fp8(char[] buf, int start, int n)
        {
            return this.extend8(this.empty, buf, start, n);
        }

        public long extend(long f, byte v)
        {
            return this.reduce(this.extend_byte(f, (int)v));
        }

        public long extend(long f, char v)
        {
            return this.reduce(this.extend_char(f, (int)v));
        }

        public long extend(long f, int v)
        {
            return this.reduce(this.extend_int(f, v));
        }

        public long extend(long f, long v)
        {
            return this.reduce(this.extend_long(f, v));
        }

        public long extend(long f, byte[] buf, int start, int n)
        {
            for (int index = 0; index < n; ++index)
                f = this.extend_byte(f, (int)buf[start + index]);
            return this.reduce(f);
        }

        public long extend(long f, char[] buf, int start, int n)
        {
            for (int index = 0; index < n; ++index)
                f = this.extend_char(f, (int)buf[start + index]);
            return this.reduce(f);
        }

        public long extend(long f, string s)
        {
            int length = s.Length;
            for (int index = 0; index < length; ++index)
            {
                int v = (int)s[index];
                f = this.extend_char(f, v);
            }
            return this.reduce(f);
        }

        public long extend(long f, int[] buf, int start, int n)
        {
            for (int index = 0; index < n; ++index)
                f = this.extend_int(f, buf[start + index]);
            return this.reduce(f);
        }

        public long extend(long f, long[] buf, int start, int n)
        {
            for (int index = 0; index < n; ++index)
                f = this.extend_long(f, buf[start + index]);
            return this.reduce(f);
        }

        public long extend8(long f, string s)
        {
            int length = s.Length;
            for (int index = 0; index < length; ++index)
            {
                int v = (int)s[index];
                f = this.extend_byte(f, v);
            }
            return this.reduce(f);
        }

        public long extend8(long f, char[] buf, int start, int n)
        {
            for (int index = 0; index < n; ++index)
                f = this.extend_byte(f, (int)buf[start + index]);
            return this.reduce(f);
        }

        public static ulong createUrlkey(string url)
        {
            int length = url.IndexOf("://");
            if (length > 0)
                length = url.IndexOf('/', length + "://".Length);
            string s = length == -1 ? url : url.Substring(0, length);
            return Convert.ToUInt64(Convert.ToString(UrlToUniqueKey.std32.fp(s) | UrlToUniqueKey.long_rightMove(UrlToUniqueKey.std40.fp(url), 24), 2).PadLeft(64, '0'), 2);
        }
        /// <summary>
        /// 根据url生成唯一id
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static long createUrlToLong(string url)
        {
            int length = url.IndexOf("://");
            if (length > 0)
                length = url.IndexOf('/', length + "://".Length);
            string s = length == -1 ? url : url.Substring(0, length);
            return UrlToUniqueKey.std32.fp(s) | UrlToUniqueKey.long_rightMove(UrlToUniqueKey.std40.fp(url), 24);
        }
    }