ArrayBlockingQueue 阻塞队列 生产者 与消费者案例

 

 

package com.originalityTest;

import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;

import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.yqm.nlp.cn.seg.ISegTagger;
import org.yqm.nlp.cn.seg.impl.CharNgramSegTagger;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
import com.originalityTest.Test.Consumer;
import com.originalityTest.Test.Producer;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import us.codecraft.background.entity.KeywordDetailed;
import us.codecraft.background.solr.SolrService;
import us.codecraft.background.solr.VSMTextSimilarity;
import us.codecraft.webmagic.main.CollectInterface;
import us.codecraft.webmagic.main.testMain;
import us.codecraft.webmagic.model.samples.iask.IaskQuestionModel;
import us.codecraft.webmagic.utils.HttpUtils;
import us.codecraft.webmagic.utils.MongoUtils;

/**
 * 
 *-----------------------------------------------------------------------------
 * <br>Copyright (c)  2018   深圳问我时代科技有限公司
 *   
 * <p>跑360采集数据跑SEO质量得分 </p>
 * 
 * @project name    : webmagic-samples
 * @package name    : com.originalityTest
 * @file name        : RunJob.java  
 * @author           : flm  
 * @date              : 2018年10月29日  <br>
 *
 *-----------------------------------------------------------------------------
 */
public class RunJob {

    protected static String host = "192.168.1.90"; //192.168.9.40:30000
    
    protected static int port = 30000;
    protected static String dbname = "5118baiduzhidao";
    protected static DB mongoDB = null;
    protected static DB mongoDBURL = null;
    protected static boolean isTestFlag = false;
    protected static int initDate = 1000*60*10;
    

    private int queueSize = 10000000;
    private ArrayBlockingQueue<BasicDBObject> queue = new ArrayBlockingQueue<BasicDBObject>(queueSize);
    
    static {
        MongoClientOptions.Builder buide = new MongoClientOptions.Builder();
        buide.connectionsPerHost(100);// 与目标数据库可以建立的最大链接数
        buide.connectTimeout(1000 * 60 * 20);// 与数据库建立链接的超时时间
        buide.maxWaitTime(100 * 60 * 5);// 一个线程成功获取到一个可用数据库之前的最大等待时间
        buide.threadsAllowedToBlockForConnectionMultiplier(100);
        buide.maxConnectionIdleTime(0);
        buide.maxConnectionLifeTime(0);
        buide.socketTimeout(0);
        buide.socketKeepAlive(true);
        MongoClientOptions myOptions = buide.build();
        try {
            MongoClient  mongoClient = new MongoClient(new ServerAddress(host, port), myOptions);
            mongoDB = mongoClient.getDB(dbname);
            mongoDBURL =  mongoClient.getDB("seo_keyword");
        } catch (UnknownHostException e) {
            e.printStackTrace();
            System.exit(0);
        }
        
    }
    /**
     * 手动执行方法
     * @param args
     * @throws Exception 
     */
    public static void main(String[] args) throws Exception {
        RunJob runJob = new RunJob();
        Producer producer = runJob.new Producer();
        Consumer consumer1 = runJob.new Consumer(1);
        Consumer consumer2 = runJob.new Consumer(2);
        Consumer consumer3 = runJob.new Consumer(3);
        Consumer consumer4 = runJob.new Consumer(4);
        Consumer consumer5 = runJob.new Consumer(5);
        Consumer consumer6 = runJob.new Consumer(6);
        Consumer consumer7 = runJob.new Consumer(7);
        Consumer consumer8 = runJob.new Consumer(8);
        Consumer consumer9 = runJob.new Consumer(9);
        Consumer consumer10 = runJob.new Consumer(10);
        // 生产数据
        producer.start();
        
        // 跑数据原创度
        consumer1.start();
        consumer2.start();
        consumer3.start();
        consumer4.start();
        consumer5.start();
        consumer6.start();
        consumer7.start();
        consumer8.start();
        consumer9.start();
        consumer10.start();
    }
    
    
    
    
    
    
    class Consumer extends Thread{
        int i;
        public Consumer(int i){
            this.i = i;
        }
        @Override
        public void run() {
            consume();
        }
         
        private void consume() {
            while(true){
                try {
                    System.out.println("队列获取   队列i:"+i);
                    DBCollection collQuestion = mongoDB.getCollection("soQA");
                    DBCollection collection = mongoDBURL.getCollection("domain");
                    
                    BasicDBObject d = queue.take();
                    
                    List<DBObject> answers=   (List<DBObject>) d.get("answers");
                    String questionTxt = d.getString("title") + d.getString("quest");
                    
                    String questionTitle = "";
                    if(d.getString("title")!=null&&d.getString("title")!=""){
                        questionTitle = d.getString("title");
                    }else{
                        questionTitle = d.getString("quest");
                    }
                    
                    float score = 0F;
                    
                    float answerLenOriginality = OriginalityUtitls.getAnswerLenOriginality(answers);
                    float answerSizeOriginality = OriginalityUtitls.getAnswerrSizeOriginality(answers);
                    float goodOriginality = OriginalityUtitls.getGoodOriginality(answers);
                    float questionOriginality = OriginalityUtitls.getQuestionOriginality(questionTxt, answers);
                    float titleOriginality = OriginalityUtitls.getTitleOriginality(questionTitle);
                    float wenwoOriginality = OriginalityUtitls.getWenwoOriginality(questionTitle, collection);
                    
                    score += Float.valueOf(questionOriginality*0.3+"");
                    score += Float.valueOf(answerLenOriginality*0.2+"");
                    score += Float.valueOf(answerSizeOriginality*0.1+"");
                    score += Float.valueOf(goodOriginality*0.05+"");
                    score += Float.valueOf(titleOriginality*0.2+"");
                    score += Float.valueOf(titleOriginality*0.2+"");
                    score += Float.valueOf(wenwoOriginality*0.1+"");
                    System.out.println("score :"+score);
                    
                    d.put("originality", score);
                    d.put("run", 0);
                    collQuestion.save(d);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
    }
     
    
    
    class Producer extends Thread{
        @Override
        public void run() {
            produce();
        }
         
        private void produce() {
            try {
                DBCollection collQuestion = mongoDB.getCollection("soQA");
                
                DBObject idQuery = new BasicDBObject();
                idQuery.put("state",1);
                DBCursor lists = collQuestion.find(idQuery);
                lists.addOption(com.mongodb.Bytes.QUERYOPTION_NOTIMEOUT); 
                int i = 0;
                while(lists.hasNext()){
                    BasicDBObject d = (BasicDBObject)lists.next(); 
                    queue.put(d);
                    System.out.println((++i)+"条记录       ,向队列取中插入一个元素，队列剩余空间："+(queueSize-queue.size()));
                    
                    try {
                        Thread.sleep(500);        // 控制生产速度，防止队列满
                    } catch (Exception e) {
                        System.err.println("Thread.sleep.....");
                    }
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    
    
}