mac系统phantomjs+java爬取动态加载数据的网站

 

第一步打开:

 https://www.baiydu.com

1.配置phantomjs环境变量

  打开终端输入：vim ~/.bash_profile

  编辑增加  export PATH=$PATH:/Users/liaohang/Desktop/phantomjs-macosx/bin

 

 

 

环境变量增加后在终端输入,phantomjs

 

 

 

 

2.  phantomjs.js封装方法

 将下面的代码复制到一个文本里，后缀修改为.js

var page = require('webpage').create(),
  system = require('system'),
  t, address;

page.settings.loadImages = false;  //为了提升加载速度，不加载图片
page.settings.resourceTimeout = 10000;//超过10秒放弃加载
//此处是用来设置截图的参数。不截图没啥用
page.viewportSize = {
  width: 1280,
  height: 800
};
block_urls = ['baidu.com'];//为了提升速度，屏蔽一些需要时间长的。比如百度广告
page.onResourceRequested = function(requestData, request){
    for(url in block_urls) {
        if(requestData.url.indexOf(block_urls[url]) !== -1) {
            request.abort();
           return;
        }
    }            
}
 
address = system.args[1];
page.open(address, function(status) {
  if (status !== 'success') {
    console.log('FAIL to load the address');
  } else {
 
    console.log(page.content);
    setTimeout(function(){ phantom.exit(); }, 6000);
  }
  phantom.exit();
});

 

   3.java代码

String HTML="";
                            String  exePath= "/Users/liaohang/Desktop/phantomjs-macosx/bin/phantomjs";
	    		    String jsPath = "/Users/liaohang/Desktop/phantomjs.js ";
	    		    System.out.println(jsPath);
	    		    System.out.println(exePath);
	    		    Runtime rt = Runtime.getRuntime();
	    		    Process p;
	    		    try {
	    		        p = rt.exec(exePath + " " + jsPath + " " + "https://www.so.com/s?q=app%E6%8E%A8%E5%B9%BF%E5%B9%B3%E5%8F%B0&pn=3");
	    		 
	    		    InputStream is = p.getInputStream();
	    		    BufferedReader br = new BufferedReader(new InputStreamReader(is));
	    		    StringBuffer sbf = new StringBuffer();
	    		    String tmp = "";
	    		    while ((tmp = br.readLine()) != null)
	    		    {
	    		        sbf.append(tmp);
	    		    }
	    		    HTML=sbf.toString();
	    		  
	    		    System.out.print(HTML);
	    		  is.close();
	    		  br.close();
	    		  sbf=null;
	    		  is=null;
	    		  br=null;
	    		    } catch (IOException e) {
	    		      
	    		        e.printStackTrace();
	    		    }

　　

原本用httpclient和jsonp不能获取的动态数据加载页面，比如ajax异步加载的，这个都能获取，直接用。