shell抓取
#!/bin/sh dir=`dirname $0` configDir="$dir/config" ipport="$configDir/ip_port" url="http://www.youdaili.cn/Daili/http/" indexs=$(curl -s --max-time 200 "$url" |piconv -f utf8 -t gbk|awk '$0~/http:\/\/www.youdaili.cn\/static\/images\/hot.gif/{print substr($2,41,length($2)-46)}') pages="$(curl -s --max-time 200 "${url}${indexs}.html"|piconv -f utf8 -t gbk|awk '$0~/共.*页/{page=gensub(/.*共([^页]+).*/,"\\1","1",$0);print page}')" for((page=1;page<=$pages;page++)) do if [[ $page -eq 1 ]] then curl -s --max-time 200 "${url}${indexs}.html"|piconv -f utf8 -t gbk|awk '$0~/.*@HTTP#.*<br \/>/{gsub(".*<p>","",$0);gsub(".*<span>","",$0);gsub("@HTTP#.*","",$0);print}' else link="${url}${indexs}_$page.html" curl -s --max-time 200 "$link"|piconv -f utf8 -t gbk|awk '$0~/.*@HTTP#.*<br \/>/{gsub(".*<p>","",$0);gsub(".*<span>","",$0);gsub("@HTTP#.*","",$0);print}' fi done | sort -u >$ipport