java 写了个爬虫的竞彩 APP 出票分析助手,想找个 UI 前端的朋友一起完善优化下
public class LotteryProcessor implements PageProcessor {
private Site site =
Site.me().setRetryTimes(3).setSleepTime(300);
@
Resource private LotteryPipeline lotteryPipeline;
@
Value("${webdriver.chrome.driver.path}")
private String chromeDriverPath;
@
Override public void process(Page page) {
Html html = page.getHtml();
if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL1)) {
List<FootballMatchDO> footballMatchList = new ArrayList<>();
List<Selectable> nodes = html.css(".bet-date-wrap").nodes();
for (int i = 1; i <= nodes.size(); i++) {
Selectable selectableDate = html.xpath("//*[@id='relativeContainer']//*[@class='bet-date-wrap'][" + (i + 1) + "]");
Selectable selectableTable = html.xpath("//*[@id='relativeContainer']//*[@class='bet-tb-dg'][" + i + "]");
List<Selectable> tr = selectableTable.css("tr").nodes();
int index = 0;
for (int j = 0; j < tr.size(); j++) {
String match = tr.get(j).css(".bet-tb-tr .td-evt a", "text").toString();
String number = tr.get(j).css(".bet-tb-tr .td-no a", "text").toString();
if (StrUtil.isBlank(match) && StrUtil.isBlank(number)) {
continue;
}
index += 2;
FootballMatchDO footballMatch = new FootballMatchDO();
footballMatch.setStartTime(selectableDate.css(".bet-date", "text").get());
footballMatch.setNumber(number);
footballMatch.setMatch(match);
String color = tr.get(j).xpath("//*[@class='td-evt']/a/@style").toString();
footballMatch.setColor(color.substring(color.indexOf("#"), color.length() - 1));
footballMatch.setOpenTime(tr.get(j).css(".td-endtime", "text").toString());
footballMatch.setHomeTeam(tr.get(j).css(".td-team .team-l i", "text").toString() + tr.get(j).css(".td-team .team-l a", "text").toString());
footballMatch.setVisitingTeam(tr.get(j).css(".td-team .team-r i", "text").toString() + tr.get(j).css(".td-team .team-r a", "text").toString());
footballMatch.setLetBall(tr.get(j).css(".td-rang .itm-rangA2", "text").toString());
footballMatch.setNotLetOdds(StrUtil.join(",", tr.get(j).css(".itm-rangB1 .betbtn span:first-child", "text").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
footballMatch.setLetOdds(StrUtil.join(",", tr.get(j).css(".itm-rangB2 .betbtn span:first-child", "text").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
footballMatch.setIsSingle((tr.get(j).css(".td-rang .itm-rangA1 .ico-dg", "text").toString()) != null || (tr.get(j).css(".td-rang .itm-rangA2 .ico-dg", "text").toString()) != null ? "1" : "0");
footballMatch.setHalfWholeOdds(StrUtil.join(",", html.xpath("/html/body/div[6]/div/div[3]/table[" + i + "]/tbody/tr[" + index + "]/td/div/table[1]/tbody/tr/td/p/i/text()").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
footballMatch.setScoreOdds(StrUtil.join(",", html.xpath("/html/body/div[6]/div/div[3]/table[" + i + "]/tbody/tr[" + index + "]/td/div/table[2]/tbody/tr/td/p/i/text()").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
footballMatch.setGoalOdds(StrUtil.join(",", html.xpath("/html/body/div[6]/div/div[3]/table[" + i + "]/tbody/tr[" + index + "]/td/div/table[3]/tbody/tr/td/p/i/text()").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
footballMatch.setDeadline(LotteryAlgorithmUtil.calculationDeadline(footballMatch.getOpenTime(), footballMatch.getStartTime()));
footballMatch.setCreateTime(new Date());
footballMatch.setUpdateTime(new Date());
footballMatchList.add(footballMatch);
}
}
page.putField("footballGoalList", footballMatchList);
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL2) || ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL16) || ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL17) || ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL20)) {
//排列 开奖结果爬取
PermutationAwardDO permutationAward = new PermutationAwardDO();
permutationAward.setStageNumber(Integer.valueOf(page.getHtml().css(".td_title01 .cfont2 strong", "text").get()));
List<String> rewardList = page.getHtml().css(".ball_box01 .ball_orange", "text").all();
if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL2)) {
permutationAward.setType(LotteryOrderTypeEnum.ARRAY.getKey());
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL16)) {
permutationAward.setType(LotteryOrderTypeEnum.ARRANGE.getKey());
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL17)) {
permutationAward.setType(LotteryOrderTypeEnum.SEVEN_STAR.getKey());
permutationAward.setMoneyAward(html.xpath("/html/body/div[6]/div[3]/div[2]/div[1]/div[2]/table[2]/tbody/tr[4]/td[3]/text()").toString().replaceAll(",", ""));
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL20)) {
permutationAward.setType(LotteryOrderTypeEnum.GRAND_LOTTO.getKey());
rewardList = page.getHtml().css(".ball_box01 li", "text").all();
permutationAward.setMoneyAward(html.xpath("/html/body/div[6]/div[3]/div[2]/div[1]/div[2]/table[2]/tbody/tr[3]/td[4]/text()").toString().replaceAll(",", "") + "," + html.xpath("/html/body/div[6]/div[3]/div[2]/div[1]/div[2]/table[2]/tbody/tr[5]/td[4]/text()").toString().replaceAll(",", ""));
}
permutationAward.setReward(StrUtil.join(",", rewardList));
page.putField("permutation", permutationAward);
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL3)) {
//抓取竞彩网的队伍分析数据
List<FootballMatchDO> footballMatchList = new ArrayList<>();
List<Selectable> nodes = html.css(".mainArea .match_item").nodes();
for (int i = 0; i < nodes.size(); i++) {
FootballMatchDO footballMatch = new FootballMatchDO();
footballMatch.setNumber(nodes.get(i).css(".list_main .matchNo", "text").toString());
// footballMatch.setHomeTeam(nodes.get(i).css(".list_main .analysis home", "text").toString());
// footballMatch.setVisitingTeam(nodes.get(i).css(".list_main .analysis guest", "text").toString());
String clk = html.xpath("//*[@id=\"divPage_Main\"]/div[2]/div[" + (i + 1) + "]/div[1]/div[2]/div[1]/@onclick").toString();
footballMatch.setAnalysis("
http://wap.310win.com" + clk.substring(clk.indexOf("('") + 2, clk.indexOf("')")));
footballMatchList.add(footballMatch);
}
page.putField("footballGoalList", footballMatchList);
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL4)) {
List<BasketballMatchDO> basketballMatchList = new ArrayList<>();
List<Selectable> nodes = html.css(".bet-date-wrap").nodes();
for (int i = 1; i <= nodes.size(); i++) {
Selectable selectableDate = html.xpath("//*[@id='relativeContainer']//*[@class='bet-date-wrap'][" + (i + 1) + "]");
Selectable selectableTable = html.xpath("//*[@id='relativeContainer']//*[@class='bet-tb-dg'][" + i + "]");
List<Selectable> tr = selectableTable.css("tr").nodes();
int index = 0;
for (int j = 0; j < tr.size(); j++) {
String match = tr.get(j).css(".bet-tb-tr .td-evt a", "text").toString();
String number = tr.get(j).css(".bet-tb-tr .td-no a", "text").toString();
if (StrUtil.isBlank(match) && StrUtil.isBlank(number)) {
continue;
}
index += 2;
BasketballMatchDO basketballMatch = new BasketballMatchDO();
basketballMatch.setStartTime(selectableDate.css(".bet-date", "text").get());
basketballMatch.setNumber(number);
basketballMatch.setMatch(match);
String color = tr.get(j).xpath("//*[@class='td-evt']/a/@style").toString();
basketballMatch.setColor(color.substring(color.indexOf("#"), color.length() - 1));
basketballMatch.setOpenTime(tr.get(j).css(".td-endtime span", "text").toString());
basketballMatch.setVisitingTeam(tr.get(j).css(".td-team .team-l i", "text").toString() + tr.get(j).css(".td-team .team-l a", "text").toString());
basketballMatch.setHomeTeam(tr.get(j).css(".td-team .team-r i", "text").toString() + tr.get(j).css(".td-team .team-r a", "text").toString());
basketballMatch.setWinNegativeOdds(StrUtil.join(",", tr.get(j).css(".betbtn-row-sf .betbtn span", "text").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
basketballMatch.setCedePointsOdds(StrUtil.join(",", tr.get(j).css(".betbtn-row-rfsf .betbtn span", "text").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
basketballMatch.setCedePoints(StrUtil.join(",", tr.get(j).css(".betbtn-row-rfsf .betmsg span", "text").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
basketballMatch.setSizeOdds(StrUtil.join(",", tr.get(j).css(".betbtn-row-dxf p span", "text").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
basketballMatch.setDifferenceOdds(StrUtil.join(",", html.xpath("/html/body/div[6]/div/div[2]/table[" + i + "]/tbody/tr[" + index + "]/td/div/table/tbody/tr/td/p/i/text()").all()).replaceAll(",↑", "").replaceAll(",↓", ""));
basketballMatch.setDeadline(LotteryAlgorithmUtil.calculationDeadline(basketballMatch.getOpenTime(), basketballMatch.getStartTime()));
basketballMatch.setCreateTime(new Date());
basketballMatch.setUpdateTime(new Date());
basketballMatchList.add(basketballMatch);
}
}
page.putField("basketballMatchList", basketballMatchList);
} else if (ObjectUtil.equal(page.getUrl().toString(), CrawlingAddressConstant.URL5)) {
//抓取竞彩网的队伍分析数据
List<BasketballMatchDO> basketballMatchList = new ArrayList<>();
List<Selectable> nodes = html.css(".mainArea .match_item").nodes();
for (int i = 0; i < nodes.size(); i++) {
BasketballMatchDO basketballMatch = new BasketballMatchDO();
basketballMatch.setNumber(nodes.get(i).css(".list_main .matchNo", "text").toString());
// footballMatch.setHomeTeam(nodes.get(i).css(".list_main .analysis home", "text").toString());
// footballMatch.setVisitingTeam(nodes.get(i).css(".list_main .analysis guest", "text").toString());
String clk = html.xpath("//*[@id=\"divPage_Main\"]/div[2]/div[" + (i + 1) + "]/div[1]/div[2]/div[1]/@onclick").toString();
basketballMatch.setAnalysis("
http://wap.310win.com" + clk.substring(clk.indexOf("('") + 2, clk.indexOf("')")));
basketballMatchList.add(basketballMatch);
}