|
|
@ -3,6 +3,10 @@ package com.baiye.task;
|
|
|
|
import cn.hutool.core.date.DateUtil;
|
|
|
|
import cn.hutool.core.date.DateUtil;
|
|
|
|
import com.alibaba.excel.EasyExcelFactory;
|
|
|
|
import com.alibaba.excel.EasyExcelFactory;
|
|
|
|
import com.baiye.module.constant.FileConstant;
|
|
|
|
import com.baiye.module.constant.FileConstant;
|
|
|
|
|
|
|
|
import com.baiye.module.dao.ClueJpa;
|
|
|
|
|
|
|
|
import com.baiye.module.dao.ClueMiddleRepository;
|
|
|
|
|
|
|
|
import com.baiye.module.dao.ClueRepository;
|
|
|
|
|
|
|
|
import com.baiye.module.entity.Clue;
|
|
|
|
import com.baiye.module.entity.ClueRecord;
|
|
|
|
import com.baiye.module.entity.ClueRecord;
|
|
|
|
import com.baiye.module.entity.vo.BaseExcelVo;
|
|
|
|
import com.baiye.module.entity.vo.BaseExcelVo;
|
|
|
|
import com.baiye.module.entity.vo.KsVo;
|
|
|
|
import com.baiye.module.entity.vo.KsVo;
|
|
|
@ -16,10 +20,12 @@ import lombok.extern.slf4j.Slf4j;
|
|
|
|
import org.apache.commons.lang.StringUtils;
|
|
|
|
import org.apache.commons.lang.StringUtils;
|
|
|
|
import org.springframework.scheduling.annotation.Async;
|
|
|
|
import org.springframework.scheduling.annotation.Async;
|
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
|
|
import org.springframework.transaction.annotation.Transactional;
|
|
|
|
|
|
|
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
import java.io.FileInputStream;
|
|
|
|
import java.io.FileInputStream;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.*;
|
|
|
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Component
|
|
|
|
@Component
|
|
|
@ -29,10 +35,20 @@ public class FileAnalysisTask {
|
|
|
|
@Resource
|
|
|
|
@Resource
|
|
|
|
private ClueService clueService;
|
|
|
|
private ClueService clueService;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
|
|
|
private ClueJpa clueJpa;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
|
|
|
private ClueRepository clueRepository;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
|
|
|
private ClueMiddleRepository clueMiddleRepository;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 读取文件处理数据
|
|
|
|
* 读取文件处理数据
|
|
|
|
* @param clueRecords
|
|
|
|
* @param clueRecords
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
@Async(value = "SendBigDataTaskExecutor")
|
|
|
|
@Async(value = "SendBigDataTaskExecutor")
|
|
|
|
@SneakyThrows //处理异常try
|
|
|
|
@SneakyThrows //处理异常try
|
|
|
|
public void runFileAnalysisTask(List<ClueRecord> clueRecords,String oneFileName) {
|
|
|
|
public void runFileAnalysisTask(List<ClueRecord> clueRecords,String oneFileName) {
|
|
|
@ -67,7 +83,9 @@ public class FileAnalysisTask {
|
|
|
|
Long taskId = clueRecords.get(0).getTaskId();
|
|
|
|
Long taskId = clueRecords.get(0).getTaskId();
|
|
|
|
Long userId = clueRecords.get(0).getCreateBy();
|
|
|
|
Long userId = clueRecords.get(0).getCreateBy();
|
|
|
|
if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)){
|
|
|
|
if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)){
|
|
|
|
log.info("================创建任务开始:{}================", DateUtil.now());
|
|
|
|
// 任务内手机号去重,先去重然后创建任务(统计任务数量)
|
|
|
|
|
|
|
|
distinctNid(taskId);
|
|
|
|
|
|
|
|
// 创建任务
|
|
|
|
BaseExcelVo baseExcelVo = new BaseExcelVo();
|
|
|
|
BaseExcelVo baseExcelVo = new BaseExcelVo();
|
|
|
|
baseExcelVo.setUserId(userId);
|
|
|
|
baseExcelVo.setUserId(userId);
|
|
|
|
baseExcelVo.setTaskId(taskId);
|
|
|
|
baseExcelVo.setTaskId(taskId);
|
|
|
@ -75,4 +93,26 @@ public class FileAnalysisTask {
|
|
|
|
clueService.saveTask(baseExcelVo);
|
|
|
|
clueService.saveTask(baseExcelVo);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
|
|
* 同一个任务内去重手机号
|
|
|
|
|
|
|
|
* @param taskId
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
|
|
|
public void distinctNid(Long taskId){
|
|
|
|
|
|
|
|
List<Clue> clueList = clueJpa.findClue(taskId);
|
|
|
|
|
|
|
|
Map<String, List<Clue>> map = clueList.stream().collect(Collectors.groupingBy(Clue::getNid));
|
|
|
|
|
|
|
|
Set<String> keySet = map.keySet();
|
|
|
|
|
|
|
|
for (String key : keySet) {
|
|
|
|
|
|
|
|
if (map.get(key).size() > 1){
|
|
|
|
|
|
|
|
List<Clue> clues = map.get(key);
|
|
|
|
|
|
|
|
clues.remove(0);
|
|
|
|
|
|
|
|
Set<Long> clueIdSet = new HashSet<>();
|
|
|
|
|
|
|
|
clues.forEach(cs -> clueIdSet.add(cs.getId()));
|
|
|
|
|
|
|
|
clueRepository.deleteAllByIdIn(clueIdSet);
|
|
|
|
|
|
|
|
log.info("===========要删除的id:{}============",clueIdSet);
|
|
|
|
|
|
|
|
clueMiddleRepository.deleteAllByClueIdIn(clueIdSet);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|