‍

　　单位的项目是做的，每个查询的里面都有很多判断

　　 上次优化之后，其中的一个分支报错，但是作为，不可能排查每一个分支.　　 所以，干脆用爬过所有的网页，主动的异常.　　 这样有两个好处 　　 1.可以主动查看网页是否异常 (500错误，404错误) 　　 2.可以筛查速度较慢的网页，从这个方向也可以定位慢吧.（也有服务器资源不足，造成网络超时的情况） 　　 前提， 　　 必须是互联网公司，大多数网页不用登录也可以浏览 　　 首先，建表 　　 CREATE SEQUENCE seq_probe_id INCREMENT BY 1 START WITH 1 NOMAXvalue NOCYCLE CACHE 2000; 　　 create table probe( 　　 id int primary key， 　　 host varchar(40) not null， 　　 path varchar(500) not null， 　　 state int not null， 　　 taskTime int not null， 　　 type varchar(10) not null， 　　 createtime date default sysdate not null 　　 ) ; 　　 其中是域名，是网页的相对路径，是HTTP状态码，是网页获取时间，单位是毫秒，是类型(html，htm，jpg等) 　　 

　　分三个主要步骤，再分别用三个队列实现生产者消费者模式.

　　 1..根据队列的目标，使用获取网页，然后放入队列　　 2..根据队列的内容，使用获取该网页的合法，将其再放入队列.然后将的网页放入队列 　　 3..将队列的内容存入，以便查询。 　　 程序使用三个步骤并行，每个步骤可以并发的方式.

$new_page$

　　但是通常来说，和可以分别用的方式执行.

import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.InputStreamReader;import java.io.OutputStreamWriter;import java.net.InetAddress;import java.net.Socket;import java.sql.Connection;import java.sql.DriverManager;import java.sql.PreparedStatement;import java.sql.SQLException;import java.util.ArrayList;import java.util.Iterator;import java.util.List;import java.util.Set;import java.util.concurrent.BlockingQueue;import java.util.concurrent.ConcurrentSkipListSet;import java.util.concurrent.CopyOnWriteArrayList;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.atomic.AtomicInteger;import java.util.regex.Matcher;import java.util.regex.Pattern;public class Probe { private static final BlockingQueue<Task> CONNECTLIST = new LinkedBlockingQueue<Task>();private static final BlockingQueue<Task> PARSELIST = new LinkedBlockingQueue<Task>();private static final BlockingQueue<Task> PERSISTENCELIST = new LinkedBlockingQueue<Task>();private static ExecutorService CONNECTTHREADPOOL;private static ExecutorService PARSETHREADPOOL;private static ExecutorService PERSISTENCETHREADPOOL;private static final List<String> DOMAINLIST = new CopyOnWriteArrayList<>();static { CONNECTTHREADPOOL = Executors.newFixedThreadPool(200);PARSETHREADPOOL = Executors.newSingleThreadExecutor();PERSISTENCETHREADPOOL = Executors.newFixedThreadPool(1);DOMAINLIST.add("域名");}public static void main(String args[]) throws Exception { long start = System.currentTimeMillis();CONNECTLIST.put(new Task("域名"， 80， "/static/index.html"));for (int i = 0; i < 600; i++) { CONNECTTHREADPOOL.submit(new ConnectHandler(CONNECTLIST， PARSELIST));}PARSETHREADPOOL.submit(new ParseHandler(CONNECTLIST， PARSELIST， PERSISTENCELIST， DOMAINLIST));PERSISTENCETHREADPOOL.submit(new PersistenceHandler(PERSISTENCELIST));while (true) { Thread.sleep(1000);long end = System.currentTimeMillis();float interval = ((end - start) / 1000);int connectTotal = ConnectHandler.GETCOUNT();int parseTotal = ParseHandler.GETCOUNT();int persistenceTotal = PersistenceHandler.GETCOUNT();int connectps = Math.round(connectTotal / interval);int parseps = Math.round(parseTotal / interval);int persistenceps = Math.round(persistenceTotal / interval);System.out.print(" 连接总数:" + connectTotal + " 每秒连接:" + connectps + " 连接队列剩余:" + CONNECTLIST.size()+ " 解析总数:" + parseTotal + " 每秒解析:" + parseps + " 解析队列剩余:" + PARSELIST.size() + " 持久化总数:"+ persistenceTotal + " 每秒持久化:" + persistenceps + " 持久化队列剩余:" + PERSISTENCELIST.size());}}}class Task { public Task() { }public void init(String host， int port， String path) { this.setCurrentPath(path);this.host = host;this.port = port;}public Task(String host， int port， String path) { init(host， port， path);}private String host;private int port;private String currentPath;private long taskTime;private String type;private String content;private int state;public int getState() { return state;}public void setState(int state) { this.state = state;}public String getCurrentPath() { return currentPath;}public void setCurrentPath(String currentPath) { this.currentPath = currentPath;this.type = currentPath.substring(currentPath.indexOf(".") + 1，currentPath.indexOf("?") != -1 ? currentPath.indexOf("?") : currentPath.length());}public long getTaskTime() { return taskTime;}public void setTaskTime(long taskTime) { this.taskTime = taskTime;}public String getType() { return type;}public void setType(String type) { this.type = type;}public String getHost() { return host;}public int getPort() { return port;}public String getContent() { return content;}public void setContent(String content) { this.content = content;}}class ParseHandler implements Runnable { private static Set<String> SET = new ConcurrentSkipListSet<String>();public static int GETCOUNT() { return COUNT.get();}private static final AtomicInteger COUNT = new AtomicInteger();private BlockingQueue<Task> connectlist;private BlockingQueue<Task> parselist;private BlockingQueue<Task> persistencelist;List<String> domainlist;

本文选自：