diff --git a/manage/ad-platform-management/src/main/java/com/baiye/modules/security/config/SpringSecurityConfig.java b/manage/ad-platform-management/src/main/java/com/baiye/modules/security/config/SpringSecurityConfig.java index 289ba500..a1652d31 100644 --- a/manage/ad-platform-management/src/main/java/com/baiye/modules/security/config/SpringSecurityConfig.java +++ b/manage/ad-platform-management/src/main/java/com/baiye/modules/security/config/SpringSecurityConfig.java @@ -140,6 +140,7 @@ public class SpringSecurityConfig extends WebSecurityConfigurerAdapter { .antMatchers("/api/task/saveTask").permitAll() .antMatchers( "/api/users/admin").permitAll() .antMatchers( "/api/task/query").permitAll() + .antMatchers( "/api/task/sendErrMessage").permitAll() .antMatchers( "/api/organize/queryAll").permitAll() .antMatchers( "/api/back/cdrUrl").permitAll() .antMatchers( "/api/back/status").permitAll() diff --git a/manage/ad-platform-management/src/main/java/com/baiye/modules/system/rest/TaskController.java b/manage/ad-platform-management/src/main/java/com/baiye/modules/system/rest/TaskController.java index b20132b7..8b063e8a 100644 --- a/manage/ad-platform-management/src/main/java/com/baiye/modules/system/rest/TaskController.java +++ b/manage/ad-platform-management/src/main/java/com/baiye/modules/system/rest/TaskController.java @@ -1,8 +1,13 @@ package com.baiye.modules.system.rest; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import com.baiye.constant.DefaultNumberConstants; import com.baiye.model.dto.TaskQueryCriteria; import com.baiye.modules.system.domain.Task; import com.baiye.modules.system.service.TaskService; +import com.baiye.socket.WebSocketServer; +import com.baiye.util.SecurityUtils; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.RequiredArgsConstructor; @@ -11,6 +16,10 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + /** * @author YQY @@ -23,6 +32,7 @@ import org.springframework.web.bind.annotation.*; public class TaskController { private final TaskService taskService; + private final WebSocketServer webSocketServer; @ApiOperation("新建任务") @PostMapping("/saveTask") @@ -41,4 +51,18 @@ public class TaskController { public ResponseEntity query(@RequestBody TaskQueryCriteria taskQueryCriteria){ return new ResponseEntity<>(taskService.query(taskQueryCriteria),HttpStatus.OK); } + + @ApiOperation("websocket发送错误信息") + @GetMapping("/sendErrMessage") + public void sendErrMessage(String errorMessage, Long userId) { + Map map = new HashMap<>(); + map.put("message", errorMessage); + map.put("code", 500); + map.put("type", "source_error"); + + JSONObject jsonData = new JSONObject(); + jsonData.putOpt("data",map); + String message = JSONUtil.toJsonStr(jsonData); + webSocketServer.sendMessage(message, userId.toString()); + } } \ No newline at end of file diff --git a/manage/ad-platform-management/src/main/java/com/baiye/socket/WebSocketServer.java b/manage/ad-platform-management/src/main/java/com/baiye/socket/WebSocketServer.java index 6f5c8f78..1b321860 100644 --- a/manage/ad-platform-management/src/main/java/com/baiye/socket/WebSocketServer.java +++ b/manage/ad-platform-management/src/main/java/com/baiye/socket/WebSocketServer.java @@ -196,7 +196,7 @@ public class WebSocketServer { * @param sessionId * @throws IOException */ - public void sendMessage(String message, String sessionId) throws IOException { + public void sendMessage(String message, String sessionId){ log.info("发送web信息 {}", message); Session session = null; if (SESSIONS.get(sessionId) != null) { diff --git a/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java b/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java index 63cbc1a5..aa9a9ae9 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java +++ b/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java @@ -3,10 +3,13 @@ package com.baiye.feign; import com.baiye.feign.fallback.TaskClientFallback; import com.baiye.model.dto.TaskQueryCriteria; import com.baiye.module.entity.Task; +import io.swagger.annotations.ApiOperation; import org.springframework.cloud.openfeign.FeignClient; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; @FeignClient(value = "ad-platform-management",fallback = TaskClientFallback.class) public interface TaskClient { @@ -23,4 +26,8 @@ public interface TaskClient { @PostMapping(PAY_PREFIX + "/query") ResponseEntity query(@RequestBody TaskQueryCriteria taskQueryCriteria); + + @ApiOperation("websocket发送错误信息") + @GetMapping(PAY_PREFIX + "/sendErrMessage") + void sendErrMessage(@RequestParam("errorMessage") String errorMessage, @RequestParam("userId") Long userId); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java b/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java index 849b8364..683aae98 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java +++ b/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java @@ -17,4 +17,9 @@ public class TaskClientFallback implements TaskClient { public ResponseEntity query(TaskQueryCriteria taskQueryCriteria) { return null; } + + @Override + public void sendErrMessage(String errorMessage, Long userId) { + + } } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java index bdf6ddfd..489c0616 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java @@ -97,4 +97,12 @@ public interface ClueMiddleRepository extends JpaRepository, J @Query(value = "select clue_id from tb_clue_middle where member_id = ?1 limit 0 ,1 ", nativeQuery = true) Long judgeMember(Long memberId); + + /** + * 查询任务总数 + * @param taskId + * @return + */ + @Query(value = "select count(*) from tb_clue_middle where task_id = ?1", nativeQuery = true) + Integer findTaskNum(Long taskId); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java index 711a9d1d..45110a1a 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java @@ -1,5 +1,7 @@ package com.baiye.module.entity; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import javax.persistence.*; @@ -34,6 +36,7 @@ public class ClueRecord implements Serializable { */ @Id @Column(name = "id", nullable = false) + @JsonSerialize(using = ToStringSerializer.class) private Long id; /** diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java index e775aa51..0b9206fc 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java @@ -21,4 +21,7 @@ public class BaseExcelVo { @ApiModelProperty(value = "任务id") private Long taskId; + + @ApiModelProperty(value = "文件名") + private String oneFileName; } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java index ed0542fc..0309e3a8 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java @@ -8,6 +8,7 @@ import com.baiye.module.entity.vo.BaseExcelVo; import com.baiye.module.entity.vo.KsVo; import com.baiye.module.service.ClueService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.BeanUtils; @@ -23,6 +24,7 @@ public class KsExcelListener extends AnalysisEventListener { * 监听类不被spring管理，选择手动注入service */ private static ClueService clueService; + public KsExcelListener(ClueService clueService, BaseExcelVo baseExcelVo) { this.clueService = clueService; this.baseExcelVo = baseExcelVo; @@ -30,13 +32,14 @@ public class KsExcelListener extends AnalysisEventListener { /** * 每条数据都会回调此函数，1000条处理一次 + * * @param ksVo * @param analysisContext */ @Override public void invoke(KsVo ksVo, AnalysisContext analysisContext) { rows.add(ksVo); - if (rows.size() >= FileConstant.ONE_THOUSAND_NUMBER){ + if (rows.size() >= FileConstant.ONE_THOUSAND_NUMBER) { saveData(); rows.clear(); } @@ -44,6 +47,7 @@ public class KsExcelListener extends AnalysisEventListener { /** * 读取文件完成回调函数，清空集合剩余数据 + * * @param analysisContext */ @Override @@ -55,24 +59,20 @@ public class KsExcelListener extends AnalysisEventListener { /** * 插入数据库 */ - private void saveData(){ + private void saveData() { List clues = new ArrayList<>(); for (KsVo row : rows) { - Clue clue = new Clue(); - BeanUtils.copyProperties(row,clue); -// clue.setName(row.getName()); -// clue.setNid(row.getNid()); -// clue.setWx(row.getWx()); -// clue.setAmount(row.getAmount()); -// clue.setCollectTime(row.getCollectTime()); -// clue.setAddress(row.getAddress()); - clue.setOrigin(baseExcelVo.getOrigin()); - clue.setRecordId(baseExcelVo.getClueRecordId()); - clue.setCreateBy(baseExcelVo.getUserId()); - clues.add(clue); + if (StringUtils.isNotBlank(row.getNid())) { + Clue clue = new Clue(); + BeanUtils.copyProperties(row, clue); + clue.setOrigin(baseExcelVo.getOrigin()); + clue.setRecordId(baseExcelVo.getClueRecordId()); + clue.setCreateBy(baseExcelVo.getUserId()); + clues.add(clue); + } } if (clues.size() > 0) { - clueService.saveClue(clues,baseExcelVo.getTaskId()); + clueService.saveClue(clues, baseExcelVo.getTaskId()); } } } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java index ff5f9ceb..e54bd44d 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java @@ -10,6 +10,7 @@ import com.baiye.module.constant.FileConstant; import com.baiye.module.entity.Clue; import com.baiye.module.entity.vo.BaseExcelVo; import com.baiye.module.service.ClueService; +import org.apache.commons.lang.StringUtils; import java.nio.charset.Charset; import java.util.ArrayList; @@ -40,23 +41,22 @@ public class UcCsvListener { //遍历行 int index = 0; for (CsvRow csvRow : rows) { - //判断去除表头 - if (index != 0){ + String nid = csvRow.getRawList().get(4); + //判断去除表头，手机号为空不添加 + if (index != 0 && StringUtils.isNotBlank(nid)){ //getRawList返回一个List列表，列表的每一项为CSV中的一个单元格（既逗号分隔部分） Clue clue = new Clue(); clue.setName(csvRow.getRawList().get(3)); - clue.setNid(csvRow.getRawList().get(4)); + clue.setNid(nid); clue.setAmount(csvRow.getRawList().get(5)); clue.setCollectTime(DateUtil.parse(csvRow.getRawList().get(2))); clue.setOrigin(baseExcelVo.getOrigin()); clue.setRecordId(baseExcelVo.getClueRecordId()); clue.setCreateBy(baseExcelVo.getUserId()); -// clue.setMemberStatus(DefaultNumberConstants.ZERO_NUMBER); clues.add(clue); } //1000条插入一次 if (clues.size() > FileConstant.ONE_THOUSAND_NUMBER){ -// if (clues.size() > 2){ clueService.saveClue(clues,baseExcelVo.getTaskId()); clues.clear(); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java index 37416ce8..bfe33cfd 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java @@ -8,6 +8,7 @@ import com.baiye.module.entity.vo.BaseExcelVo; import com.baiye.module.entity.vo.ZdyVo; import com.baiye.module.service.ClueService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.BeanUtils; import java.util.ArrayList; @@ -57,18 +58,14 @@ public class ZdyExcelListener extends AnalysisEventListener { private void saveData(){ List clues = new ArrayList<>(); for (ZdyVo row : rows) { - Clue clue = new Clue(); - BeanUtils.copyProperties(row,clue); -// clue.setName(row.getName()); -// clue.setNid(row.getNid()); -// clue.setWx(row.getWx()); -// clue.setAmount(row.getAmount()); -// clue.setCollectTime(row.getCollectTime()); -// clue.setAddress(row.getAddress()); - clue.setOrigin(baseExcelVo.getOrigin()); - clue.setRecordId(baseExcelVo.getClueRecordId()); - clue.setCreateBy(baseExcelVo.getUserId()); - clues.add(clue); + if (StringUtils.isNotBlank(row.getNid())) { + Clue clue = new Clue(); + BeanUtils.copyProperties(row, clue); + clue.setOrigin(baseExcelVo.getOrigin()); + clue.setRecordId(baseExcelVo.getClueRecordId()); + clue.setCreateBy(baseExcelVo.getUserId()); + clues.add(clue); + } } if (clues.size() > 0) { clueService.saveClue(clues,baseExcelVo.getTaskId()); diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java index cc72650e..41e78585 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java @@ -7,6 +7,7 @@ import com.baiye.model.vo.ResSourceLabel; import com.baiye.module.entity.Clue; import com.baiye.module.entity.ClueMiddle; import com.baiye.module.entity.ClueRecord; +import com.baiye.module.entity.vo.BaseExcelVo; import com.baiye.module.service.dto.ClueRecordCriteria; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -137,4 +138,9 @@ public interface ClueService { * @return */ Boolean judgeMember(Long memberId); + + /** + * 创建任务 + */ + void saveTask(BaseExcelVo baseExcelVo); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java index bee30610..8b94f114 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java @@ -3,6 +3,7 @@ package com.baiye.module.service.impl; import com.baiye.constant.DefaultNumberConstants; import com.baiye.exception.BadRequestException; import com.baiye.feign.OrganizeClient; +import com.baiye.feign.TaskClient; import com.baiye.model.dto.*; import com.baiye.model.vo.ResSourceLabel; import com.baiye.module.dao.ClueJpa; @@ -12,6 +13,8 @@ import com.baiye.module.dao.ClueRepository; import com.baiye.module.entity.Clue; import com.baiye.module.entity.ClueMiddle; import com.baiye.module.entity.ClueRecord; +import com.baiye.module.entity.Task; +import com.baiye.module.entity.vo.BaseExcelVo; import com.baiye.module.service.ClueService; import com.baiye.module.service.dto.*; import com.baiye.module.service.mapstruct.ClueRecordMapper; @@ -28,6 +31,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.servlet.http.HttpServletResponse; +import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; @@ -43,6 +47,9 @@ public class ClueServiceImpl implements ClueService { private final ClueJpa clueJpa; private final OrganizeClient organizeClient; private final RedisUtils redisUtils; + private final TaskClient taskClient; + + private static SimpleDateFormat timeOne = new SimpleDateFormat("yyyyMMddHHmmssSSS"); @Override @Transactional(rollbackFor = Exception.class) @@ -278,4 +285,29 @@ public class ClueServiceImpl implements ClueService { return false; } + /** + * 创建任务 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void saveTask(BaseExcelVo baseExcelVo) { + String oneFileName = baseExcelVo.getOneFileName(); + Long taskId = baseExcelVo.getTaskId(); + Long userId = baseExcelVo.getUserId(); + Integer taskNum = clueMiddleRepository.findTaskNum(taskId); + if (taskNum == null || taskNum == 0){ + taskClient.sendErrMessage("导入资源合规条数0条,请检查文件重新导入",userId); + log.info("=======================导入资源合规条数为0条,请检查文件重新导入=============================="); + } + int lastIndexOf = oneFileName.lastIndexOf("."); + String nameStr = oneFileName.substring(0, lastIndexOf); + String taskName = nameStr + timeOne.format(new Date()); + Task task = new Task(); + task.setId(taskId); + task.setCreateBy(userId); + task.setTaskName(taskName); + task.setTotalNumber(taskNum); + taskClient.saveTask(task); + } + } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java index 66d4bc44..064242f6 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java @@ -10,18 +10,20 @@ import cn.hutool.core.util.IdUtil; import cn.hutool.poi.excel.ExcelReader; import cn.hutool.poi.excel.ExcelUtil; import com.baiye.exception.BadRequestException; -import com.baiye.feign.TaskClient; import com.baiye.http.CommonResponse; import com.baiye.http.ResponseCode; import com.baiye.module.dao.ClueRecordRepository; import com.baiye.module.entity.ClueRecord; -import com.baiye.module.entity.Task; +import com.baiye.module.entity.vo.BaseExcelVo; +import com.baiye.module.service.ClueService; import com.baiye.module.service.UploadFileService; import com.baiye.task.FileAnalysisTask; import com.baiye.module.constant.FileConstant; import com.baiye.util.FileZipUtil; +import com.baiye.util.TestingUtil; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -48,7 +50,7 @@ public class UploadFileServiceImpl implements UploadFileService { @Resource private ClueRecordRepository clueRecordRepository; @Resource - private TaskClient taskClient; + private ClueService clueService; @Value(value = "${storage.url}") private String URL; @@ -77,52 +79,19 @@ public class UploadFileServiceImpl implements UploadFileService { @Transactional(rollbackFor = Exception.class) public ResponseEntity singleFileUpload(MultipartFile[] files, Integer uploadType, Long userId) { - List clueRecords = new ArrayList<>(); - int totalNum = 0; + List clueRecordList = new ArrayList<>(); String oneFileName = files[0].getOriginalFilename(); - long taskId = IdUtil.getSnowflake(workerId, datacenterId).nextId(); + Long taskId = IdUtil.getSnowflake(workerId, datacenterId).nextId(); //处理文件数据 for (MultipartFile file : files) { String name = file.getOriginalFilename(); int lastIndexOf = name.lastIndexOf("."); String nameStr = name.substring(lastIndexOf); // 校验文件格式 - switch (uploadType) { - case FileConstant.ONE_NUMBER: - if (!nameStr.equals(".csv")) { - throw new BadRequestException("文件格式错误！, 请上传csv格式"); - } - break; - default: - if (!((nameStr.equals(".xlsx") || nameStr.equals(".xls")))) { - throw new BadRequestException("文件格式错误！, 请上传xlsx、xls、csv格式"); - } - break; - } + testingSuffix(uploadType, nameStr); //检测excel if (nameStr.equals(".xlsx") || nameStr.equals(".xls")) { - ExcelReader reader = ExcelUtil.getReader(file.getInputStream()); - //检验excel内容格式(表头信息) - List excelHeadList = reader.readRow(0); - Boolean flag = testingFormat(excelHeadList, uploadType, FileConstant.ZERO_NUMBER); - if (!flag) { - throw new BadRequestException("文件内容格式上传错误"); - } - //检测文件不超过100w行 - int rowCount = reader.getRowCount() - FileConstant.ONE_NUMBER; - if (reader.getSheetCount() > FileConstant.ONE_NUMBER) { - // 检测多个sheet，i从1开始，自动过滤第一个sheet(第一个上面已经读过一次) - for (int i = 1; i < reader.getSheetCount(); i++) { - ExcelReader readerSheet = ExcelUtil.getReader(file.getInputStream(), i); - //-1减去表头行计算 - rowCount = readerSheet.getRowCount() - FileConstant.ONE_NUMBER + rowCount; - } - } - if (rowCount > FileConstant.ONE_MILLION_NUMBER) { - log.info("============excel行数大于100w====================="); - throw new BadRequestException("文件行数不能超过100w行"); - } - totalNum += rowCount; + testingExcel(file, uploadType); } //存入本地 String currentDate = timeTwo.format(new Date()); @@ -134,24 +103,9 @@ public class UploadFileServiceImpl implements UploadFileService { String newName = randomFileName(name); File newFile = new File(newRealPathFile, newName); file.transferTo(newFile); - - //检测csv文件(不满足记得delete临时文件) + //检测csv文件 if (nameStr.equals(".csv")) { - CsvData data = CsvUtil.getReader().read(newFile, Charset.forName("gbk")); - List rows = data.getRows(); - //检验csv内容格式(表头信息),传的是行数，下面取第一行判断 - Boolean flag = testingFormat(rows, uploadType, FileConstant.ONE_NUMBER); - if (!flag) { - newFile.delete(); - throw new BadRequestException("文件内容格式上传错误"); - } - //行数不能超过100w行 - if (rows.size() > FileConstant.ONE_MILLION_NUMBER) { - log.info("============csv行数大于100w====================="); - newFile.delete(); - throw new BadRequestException("文件行数不能超过100w行"); - } - totalNum += (rows.size() - 1); + testingCsv(newFile, uploadType); } //保存上传记录信息 long clueRecordId = IdUtil.getSnowflake(workerId, datacenterId).nextId(); @@ -163,116 +117,97 @@ public class UploadFileServiceImpl implements UploadFileService { clueRecord.setUploadType(uploadType); clueRecord.setCreateBy(userId); clueRecord.setTaskId(taskId); - - clueRecords.add(clueRecord); + clueRecordList.add(clueRecord); clueRecordRepository.save(clueRecord); } - //创建任务 - saveTask(userId, taskId, oneFileName, totalNum); - //异步读取文件 - fileAnalysisTask.runFileAnalysisTask(clueRecords); + //异步读取文件并创建任务 + fileAnalysisTask.runFileAnalysisTask(clueRecordList, oneFileName); return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.SUCCESS), HttpStatus.OK); } /** - * 创建任务 + * 检测导入类型文件后缀是否合法 * - * @param userId - * @param taskId - * @param oneFileName - * @param totalNum + * @param uploadType */ - public void saveTask(Long userId, long taskId, String oneFileName, int totalNum) { - int lastIndexOf = oneFileName.lastIndexOf("."); - String nameStr = oneFileName.substring(0, lastIndexOf); - String taskName = nameStr + timeOne.format(new Date()); - Task task = new Task(); - task.setId(taskId); - task.setCreateBy(userId); - task.setTaskName(taskName); - task.setTotalNumber(totalNum); - taskClient.saveTask(task); + public void testingSuffix(Integer uploadType, String nameStr) { + switch (uploadType) { + case FileConstant.ONE_NUMBER: + if (!nameStr.equals(".csv")) { + throw new BadRequestException("文件格式错误！, 请上传csv格式"); + } + break; + default: + if (!((nameStr.equals(".xlsx") || nameStr.equals(".xls")))) { + throw new BadRequestException("文件格式错误！, 请上传xlsx、xls格式"); + } + break; + } } /** - * 检测文件内容格式 + * 检测Excel文件格式，内容，量 * - * @param list - * @param uploadType - * @param format - * @return + * @param file 文件 + * @param uploadType 线索文件类型(UC..) */ - public Boolean testingFormat(List list, Integer uploadType, Integer format) { - //0检测excel - if (format == FileConstant.ZERO_NUMBER) { - switch (uploadType) { - //检测自定义表单头 - case FileConstant.ZERO_NUMBER: - if (list.size() >= 6) { - String zdyName = String.valueOf(list.get(0)); - String zdyNid = String.valueOf(list.get(1)); - String zdyWx = String.valueOf(list.get(2)); - String zdyAmount = String.valueOf(list.get(3)); - String zdyCollectTime = String.valueOf(list.get(4)); - String zdyAddress = String.valueOf(list.get(5)); - if (!(zdyName.equals("姓名") && zdyNid.equals("电话") && zdyWx.equals("微信") && zdyCollectTime.equals("日期") - && zdyAddress.equals("线索归属地") && zdyAmount.equals("金额"))) { - log.info("======================自定义单内容格式不正确====================="); - return false; - } - } else { - return false; - } - break; - //检测快手表单头 - case FileConstant.TWO_NUMBER: - if (list.size() >= 25) { - String ksName = String.valueOf(list.get(3)); - String ksNid = String.valueOf(list.get(4)); - String ksWx = String.valueOf(list.get(5)); - String ksCollectTime = String.valueOf(list.get(11)); - String ksAddress = String.valueOf(list.get(24)); - if (!(ksName.equals("姓名") && ksNid.equals("电话") && ksWx.equals("微信") && ksCollectTime.equals("收集日期") - && ksAddress.equals("线索归属地"))) { - log.info("======================快手表单内容格式不正确====================="); - return false; - } - } else { - return false; - } - break; - default: - break; - } + public void testingExcel(MultipartFile file, Integer uploadType) throws IOException { + ExcelReader reader = ExcelUtil.getReader(file.getInputStream()); + //检验excel内容格式(表头信息) + List excelHeadList = reader.readRow(0); + Boolean flag = TestingUtil.testingFormat(excelHeadList, uploadType, FileConstant.ZERO_NUMBER); + if (!flag) { + throw new BadRequestException("文件内容格式上传错误"); } - //1检测csv - if (format == FileConstant.ONE_NUMBER) { - switch (uploadType) { - //检测UC表单头 - case FileConstant.ONE_NUMBER: - CsvRow csvRow = (CsvRow) list.get(0); - if (csvRow.size() >= 6) { - String collectTime = csvRow.get(2); - String name = csvRow.get(3); - String nid = csvRow.get(4); - String amount = csvRow.get(5); - if (!(name.equals("第一行") && nid.equals("第二行") && collectTime.equals("时间") && amount.equals("第三行"))) { - log.info("======================UC表单内容格式不正确====================="); - return false; - } - } else { - return false; - } - break; - default: - break; + //检测文件不超过100w行 + int rowCount = reader.getRowCount() - FileConstant.ONE_NUMBER; + if (reader.getSheetCount() > FileConstant.ONE_NUMBER) { + // 检测多个sheet，i从1开始，自动过滤第一个sheet(第一个上面已经读过一次) + for (int i = 1; i < reader.getSheetCount(); i++) { + ExcelReader readerSheet = ExcelUtil.getReader(file.getInputStream(), i); + //-1减去表头行计算 + rowCount = readerSheet.getRowCount() - FileConstant.ONE_NUMBER + rowCount; } } - return true; + if (rowCount <= FileConstant.ONE_NUMBER) { + throw new BadRequestException("文件没有内容，无法生成任务"); + } + if (rowCount > FileConstant.ONE_MILLION_NUMBER) { + log.info("============excel行数大于100w====================="); + throw new BadRequestException("文件行数不能超过100w行"); + } + } + + /** + * 检测CSV文件格式，内容，量(不满足要delete临时文件) + * + * @param newFile 文件 + * @param uploadType 线索文件类型(UC..) + */ + public void testingCsv(File newFile, Integer uploadType) { + CsvData data = CsvUtil.getReader().read(newFile, Charset.forName("gbk")); + List rows = data.getRows(); + //检验csv内容格式(表头信息),传的是行数，下面取第一行判断 + Boolean flag = TestingUtil.testingFormat(rows, uploadType, FileConstant.ONE_NUMBER); + if (!flag) { + newFile.delete(); + throw new BadRequestException("文件内容格式上传错误"); + } + if (rows.size() <= FileConstant.ONE_NUMBER) { + newFile.delete(); + throw new BadRequestException("文件没有内容，无法生成任务"); + } + //行数不能超过100w行 + if (rows.size() > FileConstant.ONE_MILLION_NUMBER) { + log.info("============csv行数大于100w====================="); + newFile.delete(); + throw new BadRequestException("文件行数不能超过100w行"); + } } /** * 压缩前一天文件夹(每天3点) + * * @throws Exception */ @Scheduled(cron = "0 0 3 * * ? ") @@ -281,8 +216,8 @@ public class UploadFileServiceImpl implements UploadFileService { DateTime date = DateUtil.yesterday(); String path = URL + dateFormat.format(date); boolean directory = FileUtil.isDirectory(path); - if (directory){ - FileZipUtil.zipFile(new File(path),"zip"); + if (directory) { + FileZipUtil.zipFile(new File(path), "zip"); } log.info("-----------------------------压缩文件夹开始时间{}-----------------------------", DateUtil.now()); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java b/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java index d0c9d582..3cb7e433 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java +++ b/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java @@ -1,5 +1,6 @@ package com.baiye.task; +import cn.hutool.core.date.DateUtil; import com.alibaba.excel.EasyExcelFactory; import com.baiye.module.constant.FileConstant; import com.baiye.module.entity.ClueRecord; @@ -12,6 +13,7 @@ import com.baiye.module.listener.ZdyExcelListener; import com.baiye.module.service.ClueService; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; @@ -33,8 +35,7 @@ public class FileAnalysisTask { */ @Async(value = "SendBigDataTaskExecutor") @SneakyThrows //处理异常try - public void runFileAnalysisTask(List clueRecords) { - + public void runFileAnalysisTask(List clueRecords,String oneFileName) { for (ClueRecord clueRecord : clueRecords) { BaseExcelVo baseExcelVo = new BaseExcelVo(); @@ -42,6 +43,7 @@ public class FileAnalysisTask { baseExcelVo.setClueRecordId(clueRecord.getId()); baseExcelVo.setOrigin(clueRecord.getUploadType()); baseExcelVo.setTaskId(clueRecord.getTaskId()); + baseExcelVo.setOneFileName(oneFileName); FileInputStream file = new FileInputStream(clueRecord.getNewFileUrl()); switch (clueRecord.getUploadType()){ @@ -50,8 +52,7 @@ public class FileAnalysisTask { EasyExcelFactory.read(file, ZdyVo.class, new ZdyExcelListener(clueService,baseExcelVo)).build().readAll(); break; case FileConstant.ONE_NUMBER: - log.info("==============开始导入UC表单============",Thread.currentThread().getName()); - //EasyExcelFactory.read(file, UcVo.class, new UcExcelListener(clueService,baseExcelVo)).build().readAll(); + log.info("==============开始导入UC表单============{}",Thread.currentThread().getName()); new UcCsvListener(clueService,baseExcelVo).readUcCsv(clueRecord.getNewFileUrl()); break; case FileConstant.TWO_NUMBER: @@ -62,6 +63,16 @@ public class FileAnalysisTask { break; } } - + //创建任务 + Long taskId = clueRecords.get(0).getTaskId(); + Long userId = clueRecords.get(0).getCreateBy(); + if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)){ + log.info("================创建任务开始:{}================", DateUtil.now()); + BaseExcelVo baseExcelVo = new BaseExcelVo(); + baseExcelVo.setUserId(userId); + baseExcelVo.setTaskId(taskId); + baseExcelVo.setOneFileName(oneFileName); + clueService.saveTask(baseExcelVo); + } } } diff --git a/services/ad-platform-source/src/main/java/com/baiye/util/TestingUtil.java b/services/ad-platform-source/src/main/java/com/baiye/util/TestingUtil.java new file mode 100644 index 00000000..5475c516 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/util/TestingUtil.java @@ -0,0 +1,91 @@ +package com.baiye.util; + +import cn.hutool.core.text.csv.CsvRow; +import com.baiye.module.constant.FileConstant; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +/** + * 检测问价格式内容 + */ +@Slf4j +public class TestingUtil { + + /** + * 检测文件内容格式是否对应 + * + * @param list + * @param uploadType 线索类型(UC,快手,自定义) + * @param format 文件格式(excel,csv) + * @return + */ + public static Boolean testingFormat(List list, Integer uploadType, Integer format) { + //0检测excel + if (format == FileConstant.ZERO_NUMBER) { + switch (uploadType) { + //检测自定义表单头 + case FileConstant.ZERO_NUMBER: + if (list.size() >= 6) { + String zdyName = String.valueOf(list.get(0)); + String zdyNid = String.valueOf(list.get(1)); + String zdyWx = String.valueOf(list.get(2)); + String zdyAmount = String.valueOf(list.get(3)); + String zdyCollectTime = String.valueOf(list.get(4)); + String zdyAddress = String.valueOf(list.get(5)); + if (!(zdyName.equals("姓名") && zdyNid.equals("电话") && zdyWx.equals("微信") && zdyCollectTime.equals("日期") + && zdyAddress.equals("线索归属地") && zdyAmount.equals("金额"))) { + log.info("======================自定义单内容格式不正确====================="); + return false; + } + } else { + return false; + } + break; + //检测快手表单头 + case FileConstant.TWO_NUMBER: + if (list.size() >= 25) { + String ksName = String.valueOf(list.get(3)); + String ksNid = String.valueOf(list.get(4)); + String ksWx = String.valueOf(list.get(5)); + String ksCollectTime = String.valueOf(list.get(11)); + String ksAddress = String.valueOf(list.get(24)); + if (!(ksName.equals("姓名") && ksNid.equals("电话") && ksWx.equals("微信") && ksCollectTime.equals("收集日期") + && ksAddress.equals("线索归属地"))) { + log.info("======================快手表单内容格式不正确====================="); + return false; + } + } else { + return false; + } + break; + default: + break; + } + } + //1检测csv + if (format == FileConstant.ONE_NUMBER) { + switch (uploadType) { + //检测UC表单头 + case FileConstant.ONE_NUMBER: + CsvRow csvRow = (CsvRow) list.get(0); + if (csvRow.size() >= 6) { + String collectTime = csvRow.get(2); + String name = csvRow.get(3); + String nid = csvRow.get(4); + String amount = csvRow.get(5); + if (!(name.equals("第一行") && nid.equals("第二行") && collectTime.equals("时间") && amount.equals("第三行"))) { + log.info("======================UC表单内容格式不正确====================="); + return false; + } + } else { + return false; + } + break; + default: + break; + } + } + return true; + } +}