关于万门大学api接口反爬措施的解决
爬取站点: https://www.wanmen.org/uni/catalog
1.打开该站点,使用F12调试.可以找到课程的节点
2.利用eolinker查看headers里必须的参数有哪些，发现x-token和x-time为必填参数。
3.查看了cookie没有发现x-token等参数，怀疑是js生成，全局查找，发现目标js文件
1 function E() { 2 var e = localStorage.getItem("authorization") 3 , t = h.a.tokenKey 4 , r = Math.round((new Date).getTime() / 1e3).toString(16) 5 , n = u()("".concat(t).concat(r)).toLowerCase() 6 , i = (Object(m.q)(), 7 Object(m.m)()) 8 , a = f.a.store.getDistinctId() 9 , o = f.a.quick("getAnonymousID") 10 , c = { 11 "X-Time": r, 12 "X-Token": n, 13 "x-sa": Object(m.u)(a, o), 14 "x-platform": Object(m.m)() ? "wap" : "web" 15 } 16 , s = d.a.parse(location.search.split("?")[1]) 17 , l = s.Authorization 18 , _ = s.appVersion 19 , p = s.app 20 , v = s.appCode; 21 return l || !_ && !v || localStorage.removeItem("authorization"), 22 e && (c = g({}, c, { 23 Authorization: e 24 })), 25 i && (l && (c = g({}, c, { 26 Authorization: l 27 })), 28 p && (c = g({}, c, { 29 "X-App": p 30 }))), 31 c 32 }
5.分析上文代码构成,发现r,n这两个参数就是需要的值,并且r是直接通过时间取十六进制构成.关键在于n参数需要在 n = u()("".concat(t).concat(r)).toLowerCase()处打断点进行调试.得到加密函数:
（F12调试以前没做过，b站上一个up主给了很大的启发 https://www.bilibili.com/video/BV1GJ41177yC?from=search&seid=9833286364967765612）
1 function u(e, t, r) { 2 var n, i, a, o, c; 3 n = r(1636), i = r(778).utf8, a = r(1637), o = r(778).bin, (c = function (e, t) { 4 e.constructor == String ? e = t && "binary" === t.encoding ? o.stringToBytes(e) : i.stringToBytes(e) : a(e) ? e = Array.prototype.slice.call(e, 0) : Array.isArray(e) || (e = e.toString()); 5 for (var r = n.bytesToWords(e), s = 8 * e.length, l = 1732584193, u = -271733879, _ = -1732584194, d = 271733878, p = 0; p < r.length; p++) r[p] = 16711935 & (r[p] << 8 | r[p] >>> 24) | 4278255360 & (r[p] << 24 | r[p] >>> 8); 6 r[s >>> 5] |= 128 << s % 32, r[14 + (s + 64 >>> 9 << 4)] = s; 7 var f = c._ff, h = c._gg, m = c._hh, v = c._ii; 8 for (p = 0; p < r.length; p += 16) { 9 var g = l, b = u, y = _, E = d; 10 u = v(u = v(u = v(u = v(u = m(u = m(u = m(u = m(u = h(u = h(u = h(u = h(u = f(u = f(u = f(u = f(u, _ = f(_, d = f(d, l = f(l, u, _, d, r[p + 0], 7, -680876936), u, _, r[p + 1], 12, -389564586), l, u, r[p + 2], 17, 606105819), d, l, r[p + 3], 22, -1044525330), _ = f(_, d = f(d, l = f(l, u, _, d, r[p + 4], 7, -176418897), u, _, r[p + 5], 12, 1200080426), l, u, r[p + 6], 17, -1473231341), d, l, r[p + 7], 22, -45705983), _ = f(_, d = f(d, l = f(l, u, _, d, r[p + 8], 7, 1770035416), u, _, r[p + 9], 12, -1958414417), l, u, r[p + 10], 17, -42063), d, l, r[p + 11], 22, -1990404162), _ = f(_, d = f(d, l = f(l, u, _, d, r[p + 12], 7, 1804603682), u, _, r[p + 13], 12, -40341101), l, u, r[p + 14], 17, -1502002290), d, l, r[p + 15], 22, 1236535329), _ = h(_, d = h(d, l = h(l, u, _, d, r[p + 1], 5, -165796510), u, _, r[p + 6], 9, -1069501632), l, u, r[p + 11], 14, 643717713), d, l, r[p + 0], 20, -373897302), _ = h(_, d = h(d, l = h(l, u, _, d, r[p + 5], 5, -701558691), u, _, r[p + 10], 9, 38016083), l, u, r[p + 15], 14, -660478335), d, l, r[p + 4], 20, -405537848), _ = h(_, d = h(d, l = h(l, u, _, d, r[p + 9], 5, 568446438), u, _, r[p + 14], 9, -1019803690), l, u, r[p + 3], 14, -187363961), d, l, r[p + 8], 20, 1163531501), _ = h(_, d = h(d, l = h(l, u, _, d, r[p + 13], 5, -1444681467), u, _, r[p + 2], 9, -51403784), l, u, r[p + 7], 14, 1735328473), d, l, r[p + 12], 20, -1926607734), _ = m(_, d = m(d, l = m(l, u, _, d, r[p + 5], 4, -378558), u, _, r[p + 8], 11, -2022574463), l, u, r[p + 11], 16, 1839030562), d, l, r[p + 14], 23, -35309556), _ = m(_, d = m(d, l = m(l, u, _, d, r[p + 1], 4, -1530992060), u, _, r[p + 4], 11, 1272893353), l, u, r[p + 7], 16, -155497632), d, l, r[p + 10], 23, -1094730640), _ = m(_, d = m(d, l = m(l, u, _, d, r[p + 13], 4, 681279174), u, _, r[p + 0], 11, -358537222), l, u, r[p + 3], 16, -722521979), d, l, r[p + 6], 23, 76029189), _ = m(_, d = m(d, l = m(l, u, _, d, r[p + 9], 4, -640364487), u, _, r[p + 12], 11, -421815835), l, u, r[p + 15], 16, 530742520), d, l, r[p + 2], 23, -995338651), _ = v(_, d = v(d, l = v(l, u, _, d, r[p + 0], 6, -198630844), u, _, r[p + 7], 10, 1126891415), l, u, r[p + 14], 15, -1416354905), d, l, r[p + 5], 21, -57434055), _ = v(_, d = v(d, l = v(l, u, _, d, r[p + 12], 6, 1700485571), u, _, r[p + 3], 10, -1894986606), l, u, r[p + 10], 15, -1051523), d, l, r[p + 1], 21, -2054922799), _ = v(_, d = v(d, l = v(l, u, _, d, r[p + 8], 6, 1873313359), u, _, r[p + 15], 10, -30611744), l, u, r[p + 6], 15, -1560198380), d, l, r[p + 13], 21, 1309151649), _ = v(_, d = v(d, l = v(l, u, _, d, r[p + 4], 6, -145523070), u, _, r[p + 11], 10, -1120210379), l, u, r[p + 2], 15, 718787259), d, l, r[p + 9], 21, -343485551), l = l + g >>> 0, u = u + b >>> 0, _ = _ + y >>> 0, d = d + E >>> 0 11 } 12 return n.endian([l, u, _, d]) 13 })._ff = function (e, t, r, n, i, a, o) { 14 var c = e + (t & r | ~t & n) + (i >>> 0) + o; 15 return (c << a | c >>> 32 - a) + t 16 }, c._gg = function (e, t, r, n, i, a, o) { 17 var c = e + (t & n | r & ~n) + (i >>> 0) + o; 18 return (c << a | c >>> 32 - a) + t 19 }, c._hh = function (e, t, r, n, i, a, o) { 20 var c = e + (t ^ r ^ n) + (i >>> 0) + o; 21 return (c << a | c >>> 32 - a) + t 22 }, c._ii = function (e, t, r, n, i, a, o) { 23 var c = e + (r ^ (t | ~n)) + (i >>> 0) + o; 24 return (c << a | c >>> 32 - a) + t 25 }, c._blocksize = 16, c._digestsize = 16, e.exports = function (e, t) { 26 if (null == e) throw new Error("Illegal argument " + e); 27 var r = n.wordsToBytes(c(e, t)); 28 return t && t.asBytes ? r : t && t.asString ? o.bytesToString(r) : n.bytesToHex(r) 29 } 30 }
6.对加密函数进行整理,得到传输一个参数就可返回token
1 //定义x-token 2 function antitoken(e) { 3 var a12 = { 4 utf8: { 5 stringToBytes: function (e) { 6 return a12.bin.stringToBytes(unescape(encodeURIComponent(e))) 7 }, bytesToString: function (e) { 8 return decodeURIComponent(escape(a.bin.bytesToString(e))) 9 } 10 }, bin: { 11 stringToBytes: function (e) { 12 for (var t = [], a = 0; a < e.length; a++) t.push(255 & e.charCodeAt(a)); 13 return t 14 }, bytesToString: function (e) { 15 for (var t = [], a = 0; a < e.length; a++) t.push(String.fromCharCode(e[a])); 16 return t.join("") 17 } 18 } 19 }; 20 var t = null; 21 var n, i, o, s, r; 22 n = { 23 rotl: function (e, t) { 24 return e << t | e >>> 32 - t 25 }, rotr: function (e, t) { 26 return e << 32 - t | e >>> t 27 }, endian: function (e) { 28 if (e.constructor == Number) return 16711935 & n.rotl(e, 8) | 4278255360 & n.rotl(e, 24); 29 for (var t = 0; t < e.length; t++) e[t] = n.endian(e[t]); 30 return e 31 }, randomBytes: function (e) { 32 for (var t = []; e > 0; e--) t.push(Math.floor(256 * Math.random())); 33 return t 34 }, bytesToWords: function (e) { 35 for (var t = [], a = 0, n = 0; a < e.length; a++, n += 8) t[n >>> 5] |= e[a] << 24 - n % 32; 36 return t 37 }, wordsToBytes: function (e) { 38 for (var t = [], a = 0; a < 32 * e.length; a += 8) t.push(e[a >>> 5] >>> 24 - a % 32 & 255); 39 return t 40 }, bytesToHex: function (e) { 41 for (var t = [], a = 0; a < e.length; a++) t.push((e[a] >>> 4).toString(16)), t.push((15 & e[a]).toString(16)); 42 return t.join("") 43 }, hexToBytes: function (e) { 44 for (var t = [], a = 0; a < e.length; a += 2) t.push(parseInt(e.substr(a, 2), 16)); 45 return t 46 }, bytesToBase64: function (e) { 47 for (var t = [], n = 0; n < e.length; n += 3) for (var i = e[n] << 16 | e[n + 1] << 8 | e[n + 2], o = 0; o < 4; o++) 8 * n + 6 * o <= 8 * e.length ? t.push(a.charAt(i >>> 6 * (3 - o) & 63)) : t.push("="); 48 return t.join("") 49 }, base64ToBytes: function (e) { 50 e = e.replace(/[^A-Z0-9+\/]/gi, ""); 51 for (var t = [], n = 0, i = 0; n < e.length; i = ++n % 4) 0 != i && t.push((a.indexOf(e.charAt(n - 1)) & Math.pow(2, -2 * i + 8) - 1) << 2 * i | a.indexOf(e.charAt(n)) >>> 6 - 2 * i); 52 return t 53 } 54 }, i = a12.utf8, o = null, s = a12.bin, (r = function (e, t) { 55 e.constructor == String ? e = t && "binary" === t.encoding ? s.stringToBytes(e) : i.stringToBytes(e) : o(e) ? e = Array.prototype.slice.call(e, 0) : Array.isArray(e) || (e = e.toString()); 56 for (var a = n.bytesToWords(e), l = 8 * e.length, c = 1732584193, d = -271733879, p = -1732584194, u = 271733878, m = 0; m < a.length; m++) a[m] = 16711935 & (a[m] << 8 | a[m] >>> 24) | 4278255360 & (a[m] << 24 | a[m] >>> 8); 57 a[l >>> 5] |= 128 << l % 32; 58 a[14 + (l + 64 >>> 9 << 4)] = l; 59 var f = r._ff, h = r._gg, v = r._hh, g = r._ii; 60 for (m = 0; m < a.length; m += 16) { 61 var y = c, _ = d, b = p, $ = u; 62 d = g(d = g(d = g(d = g(d = v(d = v(d = v(d = v(d = h(d = h(d = h(d = h(d = f(d = f(d = f(d = f(d, p = f(p, u = f(u, c = f(c, d, p, u, a[m + 0], 7, -680876936), d, p, a[m + 1], 12, -389564586), c, d, a[m + 2], 17, 606105819), u, c, a[m + 3], 22, -1044525330), p = f(p, u = f(u, c = f(c, d, p, u, a[m + 4], 7, -176418897), d, p, a[m + 5], 12, 1200080426), c, d, a[m + 6], 17, -1473231341), u, c, a[m + 7], 22, -45705983), p = f(p, u = f(u, c = f(c, d, p, u, a[m + 8], 7, 1770035416), d, p, a[m + 9], 12, -1958414417), c, d, a[m + 10], 17, -42063), u, c, a[m + 11], 22, -1990404162), p = f(p, u = f(u, c = f(c, d, p, u, a[m + 12], 7, 1804603682), d, p, a[m + 13], 12, -40341101), c, d, a[m + 14], 17, -1502002290), u, c, a[m + 15], 22, 1236535329), p = h(p, u = h(u, c = h(c, d, p, u, a[m + 1], 5, -165796510), d, p, a[m + 6], 9, -1069501632), c, d, a[m + 11], 14, 643717713), u, c, a[m + 0], 20, -373897302), p = h(p, u = h(u, c = h(c, d, p, u, a[m + 5], 5, -701558691), d, p, a[m + 10], 9, 38016083), c, d, a[m + 15], 14, -660478335), u, c, a[m + 4], 20, -405537848), p = h(p, u = h(u, c = h(c, d, p, u, a[m + 9], 5, 568446438), d, p, a[m + 14], 9, -1019803690), c, d, a[m + 3], 14, -187363961), u, c, a[m + 8], 20, 1163531501), p = h(p, u = h(u, c = h(c, d, p, u, a[m + 13], 5, -1444681467), d, p, a[m + 2], 9, -51403784), c, d, a[m + 7], 14, 1735328473), u, c, a[m + 12], 20, -1926607734), p = v(p, u = v(u, c = v(c, d, p, u, a[m + 5], 4, -378558), d, p, a[m + 8], 11, -2022574463), c, d, a[m + 11], 16, 1839030562), u, c, a[m + 14], 23, -35309556), p = v(p, u = v(u, c = v(c, d, p, u, a[m + 1], 4, -1530992060), d, p, a[m + 4], 11, 1272893353), c, d, a[m + 7], 16, -155497632), u, c, a[m + 10], 23, -1094730640), p = v(p, u = v(u, c = v(c, d, p, u, a[m + 13], 4, 681279174), d, p, a[m + 0], 11, -358537222), c, d, a[m + 3], 16, -722521979), u, c, a[m + 6], 23, 76029189), p = v(p, u = v(u, c = v(c, d, p, u, a[m + 9], 4, -640364487), d, p, a[m + 12], 11, -421815835), c, d, a[m + 15], 16, 530742520), u, c, a[m + 2], 23, -995338651), p = g(p, u = g(u, c = g(c, d, p, u, a[m + 0], 6, -198630844), d, p, a[m + 7], 10, 1126891415), c, d, a[m + 14], 15, -1416354905), u, c, a[m + 5], 21, -57434055), p = g(p, u = g(u, c = g(c, d, p, u, a[m + 12], 6, 1700485571), d, p, a[m + 3], 10, -1894986606), c, d, a[m + 10], 15, -1051523), u, c, a[m + 1], 21, -2054922799), p = g(p, u = g(u, c = g(c, d, p, u, a[m + 8], 6, 1873313359), d, p, a[m + 15], 10, -30611744), c, d, a[m + 6], 15, -1560198380), u, c, a[m + 13], 21, 1309151649), p = g(p, u = g(u, c = g(c, d, p, u, a[m + 4], 6, -145523070), d, p, a[m + 11], 10, -1120210379), c, d, a[m + 2], 15, 718787259), u, c, a[m + 9], 21, -343485551), c = c + y >>> 0; 63 d = d + _ >>> 0; 64 p = p + b >>> 0; 65 u = u + $ >>> 0; 66 } 67 return n.endian([c, d, p, u]) 68 })._ff = function (e, t, a, n, i, o, s) { 69 var r = e + (t & a | ~t & n) + (i >>> 0) + s; 70 return (r << o | r >>> 32 - o) + t 71 }; 72 r._gg = function (e, t, a, n, i, o, s) { 73 var r = e + (t & n | a & ~n) + (i >>> 0) + s; 74 return (r << o | r >>> 32 - o) + t 75 }; 76 r._hh = function (e, t, a, n, i, o, s) { 77 var r = e + (t ^ a ^ n) + (i >>> 0) + s; 78 return (r << o | r >>> 32 - o) + t 79 }; 80 r._ii = function (e, t, a, n, i, o, s) { 81 var r = e + (a ^ (t | ~n)) + (i >>> 0) + s; 82 return (r << o | r >>> 32 - o) + t 83 }; 84 r._blocksize = 16; 85 r._digestsize = 16; 86 var a = n.wordsToBytes(r(e, t)); 87 return t && t.asBytes ? a : t && t.asString ? s.bytesToString(a) : n.bytesToHex(a); 88
7.通过n = u()("".concat(t).concat(r)).toLowerCase()可知道x-token需要先生成t与r合并成的字符串作为参数其中r在已经给出,t通过查找tokenKey发现固定值为xxxxxxxxxxx,最后合并导入到antitoken函数中就可生成x-token,检测结果符合条件:
得到了token生成的方式，最后使用js2py在爬虫中执行，成功获取api返回值
总结：token一般都是包含在网页返回的cookie或者前端js自动生成，js生成也更加的复杂隐蔽，需要对于浏览器控制台的熟悉使用。
