GetData.java
2.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
package com.cnlive.shenhe.jsoup;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
/**
* @author Administrator
* @Auther: 陈浩
* @Date: 2019/6/21 14:31
* @Description:
*/
public class GetData {
public static void main(String[] args) {
String url = "http://www.chinanews.com/society.shtml";
Document document = getDocument(url);
while (true) {
List<String> jobList = getJobList(document);
for (String joburl : jobList) {
try {
new newCrawl().abc(joburl);
} catch (Exception e) {
e.printStackTrace();
}
}
try {
Thread.sleep(1000);
} catch (Exception e) {
e.printStackTrace();
}
}
}
/**
* 通过document获取具体的岗位数据
*/
private static List<String> getJobList(Document dom) {
String URL = "http://www.chinanews.com";
ArrayList<String> list = new ArrayList<>();
//获取到整个列表数据
Elements select = dom.select(".content_list ul li");
for (Element element : select) {
/* 处理个别li标签是空格问题，直接跳过不处理*/
if (element.attr("id") != "" && !element.attr("id").equals("")) {
continue;
}
//得到带连接的a标签
Elements elementa = element.select(".dd_bt a");
//从a标签里面获取到他的相对路径
String href = elementa.attr("href");
href = URL + href;
// String jobName = select1.text();
// Elements select2 = element.select(".t2 a");
// String addr = element.select(".t3").text();
// String salary = element.select(".t4").text();
// String date = element.select(".t5").text();
// JobBean jobBean = new JobBean();
// jobBean.set(jobName, companyName, addr, salary, date);
list.add(href);
}
return list;
}
/**
* 获取document
*
* @param url
* @return
*/
private static Document getDocument(String url) {
try {
Document document = Jsoup.parse(new URL(url), 4000);
return document;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}