/**
 	Created by IntelliJ IDEA.
 	User: 凌秋枫
 	Date: 2018/11/15
    Time: 10:59
    To change this template use File | Settings | File Templates.
**/
package com.ling.javareptiledemo;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import org.apache.http.impl.client.HttpClients;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

@RunWith(SpringRunner.class)
@SpringBootTest
public class JavareptiledemoApplicationTests {

	@Test
	public void contextLoads() {
		// 创建默认的客户端实例
		CloseableHttpClient httpClient = HttpClients.createDefault();

		// 创建list数组,方便后续插入数据库
		List<String> Books = new ArrayList<String>();

		// 循环爬出数据 (这里设置爬出多少页，ps:想要爬出多少页都行)
		for (int i=0;i<=300;i+=20) {
			//创建get请求实例
			String url =
					"https://read.douban.com/kind/505?start="+ i +"&sort=hot&promotion_only=False&min_price=None&max_price=None&works_type=None";
			HttpGet httpGet = new HttpGet(url);
			System.out.println("执行请求: " + httpGet.getURI());
			try {
				// 客户端执行httpGet方法 ,返回响应
				CloseableHttpResponse closeableHttpResponse = httpClient.execute(httpGet);

				// 服务器状态响应行 判断网页是否正确运行
				System.out.println("服务器状态响应行: " + closeableHttpResponse.getStatusLine().toString());

				if (closeableHttpResponse.getStatusLine().getStatusCode() == 200) {
				//  得到响应实体(就是网页的源代码)
					String entity = EntityUtils.toString(closeableHttpResponse.getEntity(), "utf-8");
                //   System.out.println("得到的网页的内容是: "+entity);

					// 用到Jsoup的部分
					Document doc = Jsoup.parse(entity);
					Elements elements = doc.select("ul[class=list-lined ebook-list column-list]").select("li[class=item store-item]");
					for (Element element : elements) {
						String bookName = element.select("div[class=title]").select("a").text();
						String bookDescription = element.select("div[class=article-desc-brief]").text();
						if (!bookName.equals("")&&!bookDescription.equals("")){
							System.out.println("书名是: " + bookName);
							System.out.println("书简介是: " + bookDescription);
							System.out.println();
							Books.add(bookName);
						}
					}
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		// 用完后要记得关闭链接,这里会出现一个异常处理
		try {
			httpClient.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		System.out.println("得到数据"+ Books.size() +"条");
	}
}