From e8d45b648f1afae537ad01396a723d612af5a107 Mon Sep 17 00:00:00 2001 From: yqy Date: Thu, 23 Dec 2021 17:22:13 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B5=84=E6=BA=90=E6=9C=8D=E5=8A=A1=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/ad-platform-source/pom.xml | 49 +++++ .../baiye/AdPlatformSourceApplication.java | 17 +- .../com/baiye/config/ThreadPoolConfig.java | 53 ++++++ .../com/baiye/controller/TestController.java | 19 -- .../main/java/com/baiye/feign/TaskClient.java | 22 +++ .../feign/fallback/TaskClientFallback.java | 14 ++ .../baiye/module/constant/FileConstant.java | 74 ++++++++ .../module/controller/ClueController.java | 78 ++++++++ .../controller/UploadFileController.java | 51 +++++ .../module/dao/ClueRecordRepository.java | 19 ++ .../com/baiye/module/dao/ClueRepository.java | 46 +++++ .../java/com/baiye/module/entity/Clue.java | 32 ++++ .../com/baiye/module/entity/ClueRecord.java | 111 +++++++++++ .../java/com/baiye/module/entity/Task.java | 25 +++ .../baiye/module/entity/vo/BaseExcelVo.java | 24 +++ .../java/com/baiye/module/entity/vo/KsVo.java | 44 +++++ .../java/com/baiye/module/entity/vo/UcVo.java | 35 ++++ .../com/baiye/module/entity/vo/ZdyVo.java | 44 +++++ .../module/listener/KsExcelListener.java | 80 ++++++++ .../baiye/module/listener/UcCsvListener.java | 71 +++++++ .../module/listener/UcExcelListener.java | 75 ++++++++ .../module/listener/ZdyExcelListener.java | 79 ++++++++ .../com/baiye/module/service/ClueService.java | 62 ++++++ .../module/service/UploadFileService.java | 16 ++ .../service/dto/ClueRecordCriteria.java | 23 +++ .../module/service/impl/ClueServiceImpl.java | 104 ++++++++++ .../module/service/mapstruct/ClueMapper.java | 11 ++ .../java/com/baiye/task/FileAnalysisTask.java | 66 +++++++ .../main/java/com/baiye/util/QueryHelp.java | 178 ++++++++++++++++++ .../src/main/resources/application-dev.yml | 80 ++++++++ .../src/main/resources/application-prod.yml | 76 ++++++++ .../src/main/resources/application.yml | 39 +++- .../src/main/resources/logback.xml | 88 +++++++++ 33 files changed, 1775 insertions(+), 30 deletions(-) create mode 100644 services/ad-platform-source/src/main/java/com/baiye/config/ThreadPoolConfig.java delete mode 100644 services/ad-platform-source/src/main/java/com/baiye/controller/TestController.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/constant/FileConstant.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/controller/ClueController.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/controller/UploadFileController.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRecordRepository.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/Clue.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/Task.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/KsVo.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/UcVo.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/ZdyVo.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/listener/UcExcelListener.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/service/UploadFileService.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/service/dto/ClueRecordCriteria.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/service/mapstruct/ClueMapper.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java create mode 100644 services/ad-platform-source/src/main/java/com/baiye/util/QueryHelp.java create mode 100644 services/ad-platform-source/src/main/resources/application-dev.yml create mode 100644 services/ad-platform-source/src/main/resources/application-prod.yml create mode 100644 services/ad-platform-source/src/main/resources/logback.xml diff --git a/services/ad-platform-source/pom.xml b/services/ad-platform-source/pom.xml index e893000c..f5d5b342 100644 --- a/services/ad-platform-source/pom.xml +++ b/services/ad-platform-source/pom.xml @@ -24,12 +24,61 @@ ad-platform-common 1.0-SNAPSHOT + com.baiye ad-platform-pojo 1.0-SNAPSHOT + + + com.alibaba + easyexcel + 2.2.7 + + + + + cn.hutool + hutool-all + 5.4.7 + + + + + com.alibaba + druid-spring-boot-starter + + + + + mysql + mysql-connector-java + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + + org.mapstruct + mapstruct + + + org.mapstruct + mapstruct-processor + provided + diff --git a/services/ad-platform-source/src/main/java/com/baiye/AdPlatformSourceApplication.java b/services/ad-platform-source/src/main/java/com/baiye/AdPlatformSourceApplication.java index 350ed20d..dd3017c2 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/AdPlatformSourceApplication.java +++ b/services/ad-platform-source/src/main/java/com/baiye/AdPlatformSourceApplication.java @@ -1,18 +1,25 @@ package com.baiye; - +import com.baiye.util.SpringContextHolder; import com.spring4all.swagger.EnableSwagger2Doc; +import org.mapstruct.Mapper; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.context.annotation.Bean; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; +import org.springframework.scheduling.annotation.EnableAsync; /** - * @author q + * @author YQY */ -@EnableSwagger2Doc +@SpringBootApplication +@EnableAsync +@EnableJpaAuditing +@EnableFeignClients @EnableDiscoveryClient -@SpringBootApplication(exclude = DataSourceAutoConfiguration.class) +@EnableSwagger2Doc public class AdPlatformSourceApplication { public static void main(String[] args) { SpringApplication.run(AdPlatformSourceApplication.class, args); diff --git a/services/ad-platform-source/src/main/java/com/baiye/config/ThreadPoolConfig.java b/services/ad-platform-source/src/main/java/com/baiye/config/ThreadPoolConfig.java new file mode 100644 index 00000000..ec7fbb2a --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/config/ThreadPoolConfig.java @@ -0,0 +1,53 @@ +package com.baiye.config; + +import cn.hutool.core.thread.NamedThreadFactory; +import org.apache.tomcat.util.threads.ThreadPoolExecutor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; + +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; + +/** + * 线程池配置总类,用于配置线程池 + */ +@Configuration +@EnableAsync +public class ThreadPoolConfig { + + /** + * 从配置文件中加载线程池相关的配置 + */ + + //常驻线程数量 + @Value(value = "${save.task.thread_pool.corePoolSize}") + private int saveCorePoolSize = 4; + //最大线程数量 + @Value(value = "${save.task.thread_pool.maxPoolSize}") + private int saveMaxPoolSize = 32; + //阻塞队列数量 + @Value(value = "${save.task.thread_pool.queueCapacity}") + private int saveQueueCapacity = 100; + //线程名称前缀 + @Value(value = "${save.task.thread_pool.ThreadNamePrefix}") + private String saveThreadNamePrefix = "SourceSaveFileTaskExecutor-"; + + + @Bean(value = "SendBigDataTaskExecutor") + public Executor SendBigDataTaskExecutor(){ + ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor( + saveCorePoolSize, + saveMaxPoolSize, + 3, + TimeUnit.SECONDS, + new LinkedBlockingDeque<>(saveQueueCapacity), + new NamedThreadFactory(saveThreadNamePrefix, false), + new ThreadPoolExecutor.DiscardOldestPolicy() + ); + return threadPoolExecutor; + } + +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/controller/TestController.java b/services/ad-platform-source/src/main/java/com/baiye/controller/TestController.java deleted file mode 100644 index 1bb49cc3..00000000 --- a/services/ad-platform-source/src/main/java/com/baiye/controller/TestController.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.baiye.controller; - -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @author Enzo - * @date : 2021/12/16 - */ -@RequestMapping("/test") -@RestController -public class TestController { - - @GetMapping - public String helloWorld(){ - return "Hello Source"; - } -} diff --git a/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java b/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java new file mode 100644 index 00000000..258c8f86 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/feign/TaskClient.java @@ -0,0 +1,22 @@ +package com.baiye.feign; + +import com.baiye.feign.fallback.TaskClientFallback; +import com.baiye.module.entity.Task; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +@FeignClient(value = "ad-platform-management",fallback = TaskClientFallback.class) +public interface TaskClient { + String PAY_PREFIX = "/api/task"; + + + /** + * 添加任务 + * @param task + * @return + */ + @PostMapping(value = PAY_PREFIX + "/saveTask") + ResponseEntity saveTask(@RequestBody Task task); +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java b/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java new file mode 100644 index 00000000..c21f5182 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/feign/fallback/TaskClientFallback.java @@ -0,0 +1,14 @@ +package com.baiye.feign.fallback; + +import com.baiye.feign.TaskClient; +import com.baiye.module.entity.Task; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; + +@Component +public class TaskClientFallback implements TaskClient { + @Override + public ResponseEntity saveTask(Task task) { + return null; + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/constant/FileConstant.java b/services/ad-platform-source/src/main/java/com/baiye/module/constant/FileConstant.java new file mode 100644 index 00000000..d5ddf365 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/constant/FileConstant.java @@ -0,0 +1,74 @@ +package com.baiye.module.constant; + +/** + * @author + * 文件相关 常量类 + */ +public class FileConstant { + + /** + * 0，(自定义模板) + */ + public static final int ZERO_NUMBER = 0; + + /** + * 1，(UC) + */ + public static final int ONE_NUMBER = 1; + + /** + * 2，(快手) + */ + public static final int TWO_NUMBER = 2; + + /** + * 3，(抖音) + */ + public static final int THREE_NUMBER = 3; + + /** + * 4，(百度) + */ + public static final int FOUR_NUMBER = 4; + + /** + * 5 + */ + public static final int FIVE_NUMBER = 5; + + /** + * 6 + */ + public static final int SIX_NUMBER = 6; + + /** + * 1000 + */ + public static final int ONE_THOUSAND_NUMBER = 1000; + + /** + * 1000000 + */ + public static final int ONE_MILLION_NUMBER = 1000000; + + /** + * uc + */ + public static final String UC = "uc"; + + /** + * 快手 + */ + public static final String KS = "ks"; + + /** + * 抖音 + */ + public static final String DY = "dy"; + + /** + * 其它 + */ + public static final String QT = "qt"; + +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/controller/ClueController.java b/services/ad-platform-source/src/main/java/com/baiye/module/controller/ClueController.java new file mode 100644 index 00000000..8f43c0b6 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/controller/ClueController.java @@ -0,0 +1,78 @@ +package com.baiye.module.controller; + +import com.baiye.http.CommonResponse; +import com.baiye.http.ResponseCode; +import com.baiye.model.dto.ClueQueryCriteria; +import com.baiye.model.dto.DistributeResponseDTO; +import com.baiye.module.entity.Clue; +import com.baiye.module.service.ClueService; +import com.baiye.module.service.dto.ClueRecordCriteria; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Map; + +@RequiredArgsConstructor +@RestController +@Api(tags = "资源管理") +@RequestMapping("/source/clue") +@Slf4j +public class ClueController { + + private final ClueService clueService; + + @ApiOperation("任务id查询资源列表") + @GetMapping("/queryList") + public ResponseEntity> queryList(@RequestParam("taskId") Long taskId){ + if (taskId != null){ + return new ResponseEntity<>(clueService.queryList(taskId), HttpStatus.OK); + } + return new ResponseEntity<>(HttpStatus.OK); + } + + @ApiOperation("查询资源记录列表") + @GetMapping("/queryRecordList") + public ResponseEntity queryRecordList(ClueRecordCriteria clueRecordCriteria, Pageable pageable){ + return new ResponseEntity<>(clueService.queryRecordList(clueRecordCriteria,pageable), HttpStatus.OK); + } + + @ApiOperation("批量更新组信息") + @PostMapping("/batchUpdateOrganize") + public ResponseEntity batchUpdateOrganize(@RequestBody DistributeResponseDTO distributeResponseDTO){ + if (distributeResponseDTO != null){ + clueService.batchUpdateOrganize(distributeResponseDTO); + } + return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.EMPTY_ARGUMENT), HttpStatus.OK); + } + + @ApiOperation("小组id查询") + @GetMapping("/queryOrganizeIdList") + public ResponseEntity> queryOrganizeIdList(@RequestParam("organizeId") Long organizeId){ + if (organizeId != null){ + return new ResponseEntity<>(clueService.queryOrganizeIdList(organizeId), HttpStatus.OK); + } + return new ResponseEntity<>(HttpStatus.OK); + } + + @ApiOperation("批量更新组员信息") + @PostMapping("/batchUpdateUserId") + public ResponseEntity batchUpdateUserId(@RequestBody List>> mapList){ + if ( mapList.size() > 0){ + clueService.batchUpdateUserId(mapList); + } + return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.EMPTY_ARGUMENT), HttpStatus.OK); + } + + @ApiOperation("分页查询资源信息") + @PostMapping("/queryAll") + public ResponseEntity> queryAll(@RequestBody ClueQueryCriteria clueQueryCriteria){ + return new ResponseEntity<>(clueService.queryAll(clueQueryCriteria),HttpStatus.OK); + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/controller/UploadFileController.java b/services/ad-platform-source/src/main/java/com/baiye/module/controller/UploadFileController.java new file mode 100644 index 00000000..43d11f80 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/controller/UploadFileController.java @@ -0,0 +1,51 @@ +package com.baiye.module.controller; + +import com.baiye.http.CommonResponse; +import com.baiye.http.ResponseCode; +import com.baiye.module.service.UploadFileService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +@RequiredArgsConstructor +@RestController +@Api(tags = "文件上传管理") +@RequestMapping("/source/uploadFile") +@Slf4j +public class UploadFileController { + + private final UploadFileService uploadFileService; + @Value("${storage.download-template}") + private String rootPath; + + /** + * 文件上传接口 + * + */ + @ApiOperation("文件上传") + @PostMapping("/fileUpload") + public ResponseEntity singleFileUpload(@RequestParam("file") MultipartFile[] file, + @RequestParam(value = "uploadType") Integer uploadType, + @RequestParam(value = "userId") Long userId) { + + if (file.length > 0 && uploadType != null && userId != null){ + return uploadFileService.singleFileUpload(file,uploadType,userId); + } + return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.EMPTY_ARGUMENT.getDesc()), HttpStatus.OK); + } + + @ApiOperation("模板下载") + @PostMapping("/downLoadTemplate") + public ResponseEntity downloadFileAction() { + return new ResponseEntity<>(rootPath, HttpStatus.OK); + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRecordRepository.java b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRecordRepository.java new file mode 100644 index 00000000..9b3ae517 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRecordRepository.java @@ -0,0 +1,19 @@ +package com.baiye.module.dao; + +import com.baiye.module.entity.ClueRecord; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface ClueRecordRepository extends JpaRepository, JpaSpecificationExecutor { + + /** + * 任务id查询 + * @param taskId + * @return + */ + List findByTaskId(Long taskId); +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java new file mode 100644 index 00000000..24911971 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java @@ -0,0 +1,46 @@ +package com.baiye.module.dao; + +import com.baiye.module.entity.Clue; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface ClueRepository extends JpaRepository, JpaSpecificationExecutor { + + /** + * 根据recordId查询 + * @param recordId + * @return + */ + List findByRecordId(Long recordId); + + /** + * 批量更新组信息 + * @param id + * @param deptId + */ + @Modifying + @Query(value = " update tb_clue set organize_id = ?2 where id = ?1 ",nativeQuery = true) + void updateOrganizeIdById(Long id,Long deptId); + + /** + * 小组id查询 + * @param organizeId + * @return + */ + List findByOrganizeId(Long organizeId); + + /** + * 批量更新组员信息 + * @param clueId + * @param userId + */ + @Modifying + @Query(value = " update tb_clue set member_id = ?3,member_status = ?2 where id = ?1 ",nativeQuery = true) + void updateUserId(Long clueId, Integer memberStatus,Long userId); +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/Clue.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/Clue.java new file mode 100644 index 00000000..679b28ca --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/Clue.java @@ -0,0 +1,32 @@ +package com.baiye.module.entity; + +import com.baiye.model.entity.ClueBase; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import javax.persistence.*; +import lombok.Data; + + +/** + * Clue + * + * @author yqy + * @date 2021-12-07 10:40:37 + */ +@Data +@Entity +@Table(name = "tb_clue") +@ApiModel(value = "Clue") +public class Clue extends ClueBase { + + /** + * 线索表，主键id（自动递增） + * nullable : false + * default : null + */ + @Id + @ApiModelProperty(value = "线索表，主键id（自动递增）") + @Column(name = "id", nullable = true) + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java new file mode 100644 index 00000000..711a9d1d --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/ClueRecord.java @@ -0,0 +1,111 @@ +package com.baiye.module.entity; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import javax.persistence.*; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.CreationTimestamp; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.io.Serializable; +import java.util.Date; + + +/** + * ClueRecord + * + * @author yqy + * @date 2021-12-07 10:40:53 + */ +@Setter +@Getter +@Entity +@Table(name = "tb_clue_record") +@ApiModel(value = "ClueRecord") +@EntityListeners(AuditingEntityListener.class) +public class ClueRecord implements Serializable { + + private static final long serialVersionUID = 1271571231859316736L; + /** + * 导入资源记录表，主键ID + * nullable : false + * default : null + */ + @Id + @Column(name = "id", nullable = false) + private Long id; + + /** + * 上传状态 0:上传成功 1:上传失败 (默认0) + * nullable : true + * default : 0 + */ + @ApiModelProperty(value = "上传状态 0:上传成功 1:上传失败 (默认0)") + @Column(name = "status", nullable = true, length = 1) + private Boolean status; + + /** + * 上传类型 0:其它 1:UC 2:快手 3:抖音 4:百度 + * nullable : true + * default : null + */ + @ApiModelProperty(value = "上传类型 0:自定义模板 1:UC 2:快手 3:抖音 4:百度") + @Column(name = "upload_type", nullable = true) + private Integer uploadType; + + /** + * 任务id + * nullable : false + * default : null + */ + @ApiModelProperty(value = "任务id") + @Column(name = "task_id", nullable = true) + private Long taskId; + + /** + * 上传文件名 + * nullable : true + * default : null + */ + @ApiModelProperty(value = "上传文件名") + @Column(name = "old_file", nullable = true, length = 255) + private String oldFile; + + /** + * 存入路径 + * nullable : true + * default : null + */ + @ApiModelProperty(value = "存入路径") + @Column(name = "new_file_url", nullable = true, length = 255) + private String newFileUrl; + + /** + * 创建人 + * nullable : true + * default : null + */ + @ApiModelProperty(value = "创建人") + @Column(name = "create_by", nullable = true) + private Long createBy; + + /** + * 创建时间 + * nullable : true + * default : null + */ + @ApiModelProperty(value = "创建时间") + @Column(name = "create_time", nullable = true) + @CreationTimestamp + private Date createTime; + + /** + * 备注 + * nullable : true + * default : null + */ + @ApiModelProperty(value = "备注") + @Column(name = "remark", nullable = true, length = 500) + private String remark; +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/Task.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/Task.java new file mode 100644 index 00000000..b8423b01 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/Task.java @@ -0,0 +1,25 @@ +package com.baiye.module.entity; + +import com.baiye.model.entity.TaskBase; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.persistence.*; + +@Data +@Entity +@Table(name = "tb_task") +@ApiModel(value = "Task") +public class Task extends TaskBase { + + /** + * 任务表，主键ID + * nullable : false + * default : null + */ + @Id + @ApiModelProperty(value = "任务表，主键ID") + @Column(name = "id", nullable = true) + private Long id; +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java new file mode 100644 index 00000000..e775aa51 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/BaseExcelVo.java @@ -0,0 +1,24 @@ +package com.baiye.module.entity.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class BaseExcelVo { + + @ApiModelProperty(value = "记录表id") + private Long clueRecordId; + + @ApiModelProperty(value = "用户id") + private Long userId; + + @ApiModelProperty(value = "数据来源类型") + private Integer origin; + + @ApiModelProperty(value = "任务id") + private Long taskId; +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/KsVo.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/KsVo.java new file mode 100644 index 00000000..4f58bbee --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/KsVo.java @@ -0,0 +1,44 @@ +package com.baiye.module.entity.vo; + +import com.alibaba.excel.annotation.ExcelProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.Date; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(value = "快手表") +@ToString +public class KsVo { + + @ExcelProperty(value = "姓名", index = 3) + @ApiModelProperty(value = "姓名") + private String name; + + @ExcelProperty(value = "电话", index = 4) + @ApiModelProperty(value = "nid") + private String nid; + + @ExcelProperty(value = "微信", index = 5) + @ApiModelProperty(value = "微信") + private String wx; + + @ExcelProperty(value = "收集日期", index = 11) + @ApiModelProperty(value = "线索获取时间") + private Date collectTime; + + @ExcelProperty(value = "线索归属地", index = 24) + @ApiModelProperty(value = "线索归属地") + private String address; + + @ExcelProperty(value = "金额", index = 32) + @ApiModelProperty(value = "金额") + private String amount; + +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/UcVo.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/UcVo.java new file mode 100644 index 00000000..4e066e25 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/UcVo.java @@ -0,0 +1,35 @@ +package com.baiye.module.entity.vo; + +import com.alibaba.excel.annotation.ExcelProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.Date; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(value = "UC表") +@ToString +public class UcVo{ + + @ExcelProperty(value = "时间", index = 2) + @ApiModelProperty(value = "线索获取时间") + private Date collectTime; + + @ExcelProperty(value = "第一行", index = 3) + @ApiModelProperty(value = "姓名") + private String name; + + @ExcelProperty(value = "第二行", index = 4) + @ApiModelProperty(value = "nid") + private String nid; + + @ExcelProperty(value = "第三行", index = 5) + @ApiModelProperty(value = "金额") + private String amount; +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/ZdyVo.java b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/ZdyVo.java new file mode 100644 index 00000000..15e22443 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/entity/vo/ZdyVo.java @@ -0,0 +1,44 @@ +package com.baiye.module.entity.vo; + +import com.alibaba.excel.annotation.ExcelProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.Date; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(value = "自定义表") +@ToString +public class ZdyVo { + + @ExcelProperty(value = "姓名", index = 0) + @ApiModelProperty(value = "姓名") + private String name; + + @ExcelProperty(value = "电话", index = 1) + @ApiModelProperty(value = "nid") + private String nid; + + @ExcelProperty(value = "微信", index = 2) + @ApiModelProperty(value = "微信") + private String wx; + + @ExcelProperty(value = "金额", index = 3) + @ApiModelProperty(value = "金额") + private String amount; + + @ExcelProperty(value = "日期", index = 4) + @ApiModelProperty(value = "线索获取时间") + private Date collectTime; + + @ExcelProperty(value = "线索归属地", index = 5) + @ApiModelProperty(value = "线索归属地") + private String address; + +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java new file mode 100644 index 00000000..2936a0ec --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/KsExcelListener.java @@ -0,0 +1,80 @@ +package com.baiye.module.listener; + +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.baiye.constant.DefaultNumberConstants; +import com.baiye.module.constant.FileConstant; +import com.baiye.module.entity.Clue; +import com.baiye.module.entity.vo.BaseExcelVo; +import com.baiye.module.entity.vo.KsVo; +import com.baiye.module.service.ClueService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; + + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class KsExcelListener extends AnalysisEventListener { + + public final List rows = new ArrayList<>(); + private BaseExcelVo baseExcelVo; + /** + * 监听类不被spring管理，选择手动注入service + */ + private static ClueService clueService; + public KsExcelListener(ClueService clueService, BaseExcelVo baseExcelVo) { + this.clueService = clueService; + this.baseExcelVo = baseExcelVo; + } + + /** + * 每条数据都会回调此函数，1000条处理一次 + * @param ksVo + * @param analysisContext + */ + @Override + public void invoke(KsVo ksVo, AnalysisContext analysisContext) { + rows.add(ksVo); + if (rows.size() >= FileConstant.ONE_THOUSAND_NUMBER){ + saveData(); + rows.clear(); + } + } + + /** + * 读取文件完成回调函数，清空集合剩余数据 + * @param analysisContext + */ + @Override + public void doAfterAllAnalysed(AnalysisContext analysisContext) { + saveData(); + rows.clear(); + } + + /** + * 插入数据库 + */ + private void saveData(){ + List clues = new ArrayList<>(); + for (KsVo row : rows) { + Clue clue = new Clue(); + BeanUtils.copyProperties(row,clue); +// clue.setName(row.getName()); +// clue.setNid(row.getNid()); +// clue.setWx(row.getWx()); +// clue.setAmount(row.getAmount()); +// clue.setCollectTime(row.getCollectTime()); +// clue.setAddress(row.getAddress()); + clue.setOrigin(baseExcelVo.getOrigin()); + clue.setRecordId(baseExcelVo.getClueRecordId()); + clue.setCreateBy(baseExcelVo.getUserId()); + clue.setMemberStatus(DefaultNumberConstants.ZERO_NUMBER); + clues.add(clue); + } + if (clues.size() > 0) { + clueService.saveClue(clues); + } + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java new file mode 100644 index 00000000..194ee85b --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcCsvListener.java @@ -0,0 +1,71 @@ +package com.baiye.module.listener; + +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.text.csv.CsvData; +import cn.hutool.core.text.csv.CsvReader; +import cn.hutool.core.text.csv.CsvRow; +import cn.hutool.core.text.csv.CsvUtil; +import com.baiye.constant.DefaultNumberConstants; +import com.baiye.module.constant.FileConstant; +import com.baiye.module.entity.Clue; +import com.baiye.module.entity.vo.BaseExcelVo; +import com.baiye.module.service.ClueService; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +public class UcCsvListener { + + private final List clues = new ArrayList<>(); + private BaseExcelVo baseExcelVo; + /** + * 监听类不被spring管理，选择手动注入service + */ + private static ClueService clueService; + public UcCsvListener(ClueService clueService, BaseExcelVo baseExcelVo) { + this.clueService = clueService; + this.baseExcelVo = baseExcelVo; + } + + /** + * 处理UC的csv文件 + * @param filePath + */ + public void readUcCsv(String filePath){ + CsvReader reader = CsvUtil.getReader(); + //从文件中读取CSV数据，编码要gbk解决乱码 + CsvData data = reader.read(FileUtil.file(filePath), Charset.forName("gbk")); + List rows = data.getRows(); + //遍历行 + int index = 0; + for (CsvRow csvRow : rows) { + //判断去除表头 + if (index != 0){ + //getRawList返回一个List列表，列表的每一项为CSV中的一个单元格（既逗号分隔部分） + Clue clue = new Clue(); + clue.setName(csvRow.getRawList().get(3)); + clue.setNid(csvRow.getRawList().get(4)); + clue.setAmount(csvRow.getRawList().get(5)); + clue.setCollectTime(DateUtil.parse(csvRow.getRawList().get(2))); + clue.setOrigin(baseExcelVo.getOrigin()); + clue.setRecordId(baseExcelVo.getClueRecordId()); + clue.setCreateBy(baseExcelVo.getUserId()); + clue.setMemberStatus(DefaultNumberConstants.ZERO_NUMBER); + clues.add(clue); + } + //1000条插入一次 + if (clues.size() > FileConstant.ONE_THOUSAND_NUMBER){ +// if (clues.size() > 2){ + clueService.saveClue(clues); + clues.clear(); + } + index++; + } + //清除集合数据 + if (clues.size() > 0) { + clueService.saveClue(clues); + } + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcExcelListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcExcelListener.java new file mode 100644 index 00000000..42d4c026 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/UcExcelListener.java @@ -0,0 +1,75 @@ +package com.baiye.module.listener; + +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.baiye.constant.DefaultNumberConstants; +import com.baiye.module.constant.FileConstant; +import com.baiye.module.entity.Clue; +import com.baiye.module.entity.vo.BaseExcelVo; +import com.baiye.module.entity.vo.UcVo; +import com.baiye.module.service.ClueService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class UcExcelListener extends AnalysisEventListener { + + private final List rows = new ArrayList<>(); + private BaseExcelVo baseExcelVo; + + /** + * 监听类不被spring管理，选择手动注入service + */ + private static ClueService clueService; + public UcExcelListener(ClueService clueService, BaseExcelVo baseExcelVo) { + this.clueService = clueService; + this.baseExcelVo = baseExcelVo; + } + + /** + * 每条数据都会回调此函数，1000条处理一次 + * @param ucVo + * @param analysisContext + */ + @Override + public void invoke(UcVo ucVo, AnalysisContext analysisContext) { + rows.add(ucVo); + if (rows.size() >= FileConstant.ONE_THOUSAND_NUMBER){ + saveData(); + rows.clear(); + } + } + + /** + * 读取文件完成回调函数，清空集合剩余数据 + * @param analysisContext + */ + @Override + public void doAfterAllAnalysed(AnalysisContext analysisContext) { + saveData(); + rows.clear(); + } + + private void saveData(){ + List clues = new ArrayList<>(); + for (UcVo row : rows) { + Clue clue = new Clue(); +// clue.setName(row.getName()); +// clue.setNid(row.getNid()); +// clue.setAmount(row.getAmount()); +// clue.setCollectTime(row.getCollectTime()); + BeanUtils.copyProperties(row,clue); + clue.setOrigin(baseExcelVo.getOrigin()); + clue.setRecordId(baseExcelVo.getClueRecordId()); + clue.setCreateBy(baseExcelVo.getUserId()); + clue.setMemberStatus(DefaultNumberConstants.ZERO_NUMBER); + clues.add(clue); + } + if (clues.size() > 0) { + clueService.saveClue(clues); + } + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java b/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java new file mode 100644 index 00000000..03c6854e --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/listener/ZdyExcelListener.java @@ -0,0 +1,79 @@ +package com.baiye.module.listener; + +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.baiye.constant.DefaultNumberConstants; +import com.baiye.module.constant.FileConstant; +import com.baiye.module.entity.Clue; +import com.baiye.module.entity.vo.BaseExcelVo; +import com.baiye.module.entity.vo.ZdyVo; +import com.baiye.module.service.ClueService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class ZdyExcelListener extends AnalysisEventListener { + + public final List rows = new ArrayList<>(); + private BaseExcelVo baseExcelVo; + /** + * 监听类不被spring管理，选择手动注入service + */ + private static ClueService clueService; + public ZdyExcelListener(ClueService clueService, BaseExcelVo baseExcelVo) { + this.clueService = clueService; + this.baseExcelVo = baseExcelVo; + } + + /** + * 每条数据都会回调此函数，1000条处理一次 + * @param zdyVo + * @param analysisContext + */ + @Override + public void invoke(ZdyVo zdyVo, AnalysisContext analysisContext) { + rows.add(zdyVo); + if (rows.size() >= FileConstant.ONE_THOUSAND_NUMBER){ + saveData(); + rows.clear(); + } + } + + /** + * 读取文件完成回调函数，清空集合剩余数据 + * @param analysisContext + */ + @Override + public void doAfterAllAnalysed(AnalysisContext analysisContext) { + saveData(); + rows.clear(); + } + + /** + * 插入数据库 + */ + private void saveData(){ + List clues = new ArrayList<>(); + for (ZdyVo row : rows) { + Clue clue = new Clue(); + BeanUtils.copyProperties(row,clue); +// clue.setName(row.getName()); +// clue.setNid(row.getNid()); +// clue.setWx(row.getWx()); +// clue.setAmount(row.getAmount()); +// clue.setCollectTime(row.getCollectTime()); +// clue.setAddress(row.getAddress()); + clue.setOrigin(baseExcelVo.getOrigin()); + clue.setRecordId(baseExcelVo.getClueRecordId()); + clue.setCreateBy(baseExcelVo.getUserId()); + clue.setMemberStatus(DefaultNumberConstants.ZERO_NUMBER); + clues.add(clue); + } + if (clues.size() > 0) { + clueService.saveClue(clues); + } + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java new file mode 100644 index 00000000..dec66afa --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/ClueService.java @@ -0,0 +1,62 @@ +package com.baiye.module.service; + +import com.baiye.model.dto.ClueQueryCriteria; +import com.baiye.model.dto.DistributeResponseDTO; +import com.baiye.module.entity.Clue; +import com.baiye.module.entity.ClueRecord; +import com.baiye.module.service.dto.ClueRecordCriteria; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; + +import java.util.List; +import java.util.Map; + +public interface ClueService { + + /** + * 插入数据 + * @param clueList + */ + void saveClue(List clueList); + + /** + * 任务id查询资源列表 + * @param taskId + * @return + */ + List queryList(Long taskId); + + /** + * 用户id查询资源记录列表 + * @return + */ + Page queryRecordList(ClueRecordCriteria clueRecordCriteria, Pageable pageable); + + /** + * 批量更新组信息 + * @param distributeResponseDTO + */ + void batchUpdateOrganize(DistributeResponseDTO distributeResponseDTO); + + /** + * 小组id查询 + * @param organizeId + * @return + */ + List queryOrganizeIdList(Long organizeId); + + + /** + * 批量更新组员信息 + * Map> ,用户id->所属资源集合 + * @param mapList + */ + void batchUpdateUserId(List>> mapList); + + /** + * 分页查询资源信息 + * @param clueQueryCriteria + * @return + */ + Map queryAll(ClueQueryCriteria clueQueryCriteria); +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/UploadFileService.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/UploadFileService.java new file mode 100644 index 00000000..452fd29b --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/UploadFileService.java @@ -0,0 +1,16 @@ +package com.baiye.module.service; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.multipart.MultipartFile; + +public interface UploadFileService { + + /** + * 文件上传 + * @param file + * @param uploadType + * @param userId + * @return + */ + ResponseEntity singleFileUpload(MultipartFile[] file, Integer uploadType,Long userId); +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/dto/ClueRecordCriteria.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/dto/ClueRecordCriteria.java new file mode 100644 index 00000000..fbe5fe23 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/dto/ClueRecordCriteria.java @@ -0,0 +1,23 @@ +package com.baiye.module.service.dto; + +import com.baiye.annotation.Query; +import lombok.Data; + +import java.sql.Timestamp; +import java.util.List; + +/** + * 小组公共查询类 + */ +@Data +public class ClueRecordCriteria { + + @Query + private Long createBy; + + @Query + private String oldFile; + + @Query(type = Query.Type.BETWEEN) + private List createTime; +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java new file mode 100644 index 00000000..084b2db7 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/ClueServiceImpl.java @@ -0,0 +1,104 @@ +package com.baiye.module.service.impl; + +import com.baiye.constant.DefaultNumberConstants; +import com.baiye.model.dto.ClueDto; +import com.baiye.model.dto.ClueQueryCriteria; +import com.baiye.model.dto.DistributeResponseDTO; +import com.baiye.module.dao.ClueRecordRepository; +import com.baiye.module.dao.ClueRepository; +import com.baiye.module.entity.Clue; +import com.baiye.module.entity.ClueRecord; +import com.baiye.module.service.ClueService; +import com.baiye.module.service.dto.ClueRecordCriteria; +import com.baiye.module.service.mapstruct.ClueMapper; +import com.baiye.util.PageUtil; +import com.baiye.util.QueryHelp; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +@RequiredArgsConstructor +public class ClueServiceImpl implements ClueService { + + private final ClueRepository clueRepository; + private final ClueRecordRepository clueRecordRepository; + private final ClueMapper clueMapper; + + @Override + public void saveClue(List clueList) { + if (clueList.size() > 0){ + clueRepository.saveAll(clueList); + } + } + + @Override + public List queryList(Long taskId) { + List clueLists = new ArrayList<>(); + List clueRecordList = clueRecordRepository.findByTaskId(taskId); + if (clueRecordList.size() > 0) { + for (ClueRecord clueRecord : clueRecordList) { + List clueList = clueRepository.findByRecordId(clueRecord.getId()); + clueLists.addAll(clueList); + } + return clueLists; + } + return null; + } + + @Override + public Page queryRecordList(ClueRecordCriteria clueRecordCriteria, Pageable pageable) { + Page clueAll = clueRecordRepository.findAll((root, criteriaQuery, criteriaBuilder) -> QueryHelp.getPredicate(root, clueRecordCriteria, criteriaBuilder), pageable); + return clueAll; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void batchUpdateOrganize(DistributeResponseDTO distributeResponseDTO) { + List responseList = distributeResponseDTO.getResponseList(); + Long deptId = distributeResponseDTO.getDeptId(); + for (Long id : responseList) { + clueRepository.updateOrganizeIdById(id,deptId); + } + } + + @Override + public List queryOrganizeIdList(Long organizeId) { + + return clueRepository.findByOrganizeId(organizeId); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void batchUpdateUserId(List>> mapList) { + for (Map> map : mapList) { + for (Long userId : map.keySet()) { + List userIdList = map.get(userId); + for (Long clueId : userIdList) { + clueRepository.updateUserId(clueId, DefaultNumberConstants.ONE_NUMBER, userId); + } + } + } + } + + /** + * 分页查询资源信息 + */ + @Override + public Map queryAll(ClueQueryCriteria clueQueryCriteria ) { + Sort sort = Sort.by(Sort.Direction.ASC, "createTime"); + Pageable pageable = PageRequest.of(clueQueryCriteria.getPage(), clueQueryCriteria.getSize(),sort); + Page cluePage = clueRepository.findAll((root, criteriaQuery, criteriaBuilder) -> QueryHelp.getPredicate(root, clueQueryCriteria, criteriaBuilder), pageable); + return PageUtil.toPage(cluePage.map(clueMapper::toDto)); + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/mapstruct/ClueMapper.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/mapstruct/ClueMapper.java new file mode 100644 index 00000000..5020b57e --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/mapstruct/ClueMapper.java @@ -0,0 +1,11 @@ +package com.baiye.module.service.mapstruct; + +import com.baiye.model.base.BaseMapper; +import com.baiye.model.dto.ClueDto; +import com.baiye.module.entity.Clue; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +@Mapper(componentModel = "spring",unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface ClueMapper extends BaseMapper { +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java b/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java new file mode 100644 index 00000000..4c23310f --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java @@ -0,0 +1,66 @@ +package com.baiye.task; + +import com.alibaba.excel.EasyExcelFactory; +import com.baiye.module.constant.FileConstant; +import com.baiye.module.entity.ClueRecord; +import com.baiye.module.entity.vo.BaseExcelVo; +import com.baiye.module.entity.vo.KsVo; +import com.baiye.module.entity.vo.ZdyVo; +import com.baiye.module.listener.KsExcelListener; +import com.baiye.module.listener.UcCsvListener; +import com.baiye.module.listener.ZdyExcelListener; +import com.baiye.module.service.ClueService; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.FileInputStream; +import java.util.List; + + +@Component +@Slf4j +public class FileAnalysisTask { + + @Resource + private ClueService clueService; + + /** + * 读取文件处理数据 + * @param clueRecords + */ + @Async(value = "SendBigDataTaskExecutor") + @SneakyThrows //处理异常try + public void runFileAnalysisTask(List clueRecords) { + + + for (ClueRecord clueRecord : clueRecords) { + BaseExcelVo baseExcelVo = new BaseExcelVo(); + baseExcelVo.setUserId(clueRecord.getCreateBy()); + baseExcelVo.setClueRecordId(clueRecord.getId()); + baseExcelVo.setOrigin(clueRecord.getUploadType()); + + FileInputStream file = new FileInputStream(clueRecord.getNewFileUrl()); + switch (clueRecord.getUploadType()){ + case FileConstant.ZERO_NUMBER: + log.info("==============开始导入自定义表单============{}",Thread.currentThread().getName()); + EasyExcelFactory.read(file, ZdyVo.class, new ZdyExcelListener(clueService,baseExcelVo)).build().readAll(); + break; + case FileConstant.ONE_NUMBER: + log.info("==============开始导入UC表单============",Thread.currentThread().getName()); + //EasyExcelFactory.read(file, UcVo.class, new UcExcelListener(clueService,baseExcelVo)).build().readAll(); + new UcCsvListener(clueService,baseExcelVo).readUcCsv(clueRecord.getNewFileUrl()); + break; + case FileConstant.TWO_NUMBER: + log.info("==============开始导入快手表单============{}",Thread.currentThread().getName()); + EasyExcelFactory.read(file, KsVo.class, new KsExcelListener(clueService,baseExcelVo)).build().readAll(); + break; + default: + break; + } + } + + } +} diff --git a/services/ad-platform-source/src/main/java/com/baiye/util/QueryHelp.java b/services/ad-platform-source/src/main/java/com/baiye/util/QueryHelp.java new file mode 100644 index 00000000..3106f445 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/util/QueryHelp.java @@ -0,0 +1,178 @@ +package com.baiye.util; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import com.baiye.annotation.Query; +import com.baiye.constant.DefaultNumberConstants; +import lombok.extern.slf4j.Slf4j; + +import javax.persistence.criteria.*; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +@Slf4j +@SuppressWarnings({"unchecked", "all"}) +public class QueryHelp { + + public static Predicate getPredicate(Root root, Q query, CriteriaBuilder cb) { + List list = new ArrayList<>(); + + if (query == null) { + return cb.and(list.toArray(new Predicate[0])); + } + try { + List fields = getAllFields(query.getClass(), new ArrayList<>()); + for (Field field : fields) { + boolean accessible = field.isAccessible(); + field.setAccessible(true); + Query q = field.getAnnotation(Query.class); + if (q != null) { + String propName = q.propName(); + String joinName = q.joinName(); + String blurry = q.blurry(); + String attributeName = isBlank(propName) ? field.getName() : propName; + Class fieldType = field.getType(); + Object val = field.get(query); + if (ObjectUtil.isNull(val) || StrUtil.EMPTY.equals(val) || StrUtil.NULL.equals(val) || "undefined".equals(val)) { + continue; + } + Join join = null; + // 模糊多字段 + if (ObjectUtil.isNotEmpty(blurry)) { + String[] blurrys = blurry.split(StrUtil.COMMA); + List orPredicate = new ArrayList<>(); + for (String s : blurrys) { + orPredicate.add(cb.like(root.get(s) + .as(String.class), DefaultNumberConstants.NAT_DELIM + val.toString() + DefaultNumberConstants.NAT_DELIM)); + } + Predicate[] p = new Predicate[orPredicate.size()]; + list.add(cb.or(orPredicate.toArray(p))); + continue; + } + if (ObjectUtil.isNotEmpty(joinName)) { + String[] joinNames = joinName.split(">"); + for (String name : joinNames) { + switch (q.join()) { + case LEFT: + if (ObjectUtil.isNotNull(join)) { + join = join.join(name, JoinType.LEFT); + } else { + join = root.join(name, JoinType.LEFT); + } + break; + case RIGHT: + if (ObjectUtil.isNotNull(join)) { + join = join.join(name, JoinType.RIGHT); + } else { + join = root.join(name, JoinType.RIGHT); + } + break; + case INNER: + if (ObjectUtil.isNotNull(join)) { + join = join.join(name, JoinType.INNER); + } else { + join = root.join(name, JoinType.INNER); + } + break; + default: + break; + } + } + } + switch (q.type()) { + case EQUAL: + list.add(cb.equal(getExpression(attributeName, join, root) + .as((Class) fieldType), val)); + break; + case GREATER_THAN: + list.add(cb.greaterThanOrEqualTo(getExpression(attributeName, join, root) + .as((Class) fieldType), (Comparable) val)); + break; + case LESS_THAN: + list.add(cb.lessThanOrEqualTo(getExpression(attributeName, join, root) + .as((Class) fieldType), (Comparable) val)); + break; + case LESS_THAN_NQ: + list.add(cb.lessThan(getExpression(attributeName, join, root) + .as((Class) fieldType), (Comparable) val)); + break; + case INNER_LIKE: + list.add(cb.like(getExpression(attributeName, join, root) + .as(String.class), DefaultNumberConstants.NAT_DELIM + val.toString() + DefaultNumberConstants.NAT_DELIM)); + break; + case LEFT_LIKE: + list.add(cb.like(getExpression(attributeName, join, root) + .as(String.class), DefaultNumberConstants.NAT_DELIM + val.toString())); + break; + case RIGHT_LIKE: + list.add(cb.like(getExpression(attributeName, join, root) + .as(String.class), val.toString() + DefaultNumberConstants.NAT_DELIM)); + break; + case IN: + if (CollUtil.isNotEmpty((Collection) val)) { + list.add(getExpression(attributeName, join, root).in((Collection) val)); + } + break; + case NOT_IN: + if (CollUtil.isNotEmpty((Collection) val)) { + list.add(cb.not(getExpression(attributeName, join, root).in((Collection) val))); + } + break; + case NOT_EQUAL: + list.add(cb.notEqual(getExpression(attributeName, join, root), val)); + break; + case NOT_NULL: + list.add(cb.isNotNull(getExpression(attributeName, join, root))); + break; + case BETWEEN: + List between = new ArrayList<>((List) val); + list.add(cb.between(getExpression(attributeName, join, root).as((Class) between.get(0).getClass()), + (Comparable) between.get(0), (Comparable) between.get(1))); + break; + default: + break; + } + } + field.setAccessible(accessible); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + int size = list.size(); + return cb.and(list.toArray(new Predicate[size])); + } + + @SuppressWarnings("unchecked") + private static Expression getExpression(String attributeName, Join join, Root root) { + if (ObjectUtil.isNotEmpty(join)) { + return join.get(attributeName); + } else { + return root.get(attributeName); + } + } + + private static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(cs.charAt(i))) { + return false; + } + } + return true; + } + + private static List getAllFields(Class clazz, List fields) { + if (clazz != null) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + getAllFields(clazz.getSuperclass(), fields); + } + return fields; + } +} diff --git a/services/ad-platform-source/src/main/resources/application-dev.yml b/services/ad-platform-source/src/main/resources/application-dev.yml new file mode 100644 index 00000000..8a46666f --- /dev/null +++ b/services/ad-platform-source/src/main/resources/application-dev.yml @@ -0,0 +1,80 @@ +#配置数据源 +spring: + cloud: + nacos: + discovery: + server-addr: ${NACOS_HOST:118.178.137.129}:${NACOS_PORT:8848} + datasource: + druid: + db-type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.jdbc.Driver + url: jdbc:mysql://118.178.137.129:3306/ad-platform?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&zeroDateTimeBehavior=convertToNull + username: root + password: root + # 初始连接数 + initial-size: 5 + # 最小连接数 + min-idle: 15 + # 最大连接数 + max-active: 30 + # 超时时间(以秒数为单位) + remove-abandoned-timeout: 180 + # 获取连接超时时间 + max-wait: 3000 + # 连接有效性检测时间 + time-between-eviction-runs-millis: 60000 + # 连接在池中最小生存的时间 + min-evictable-idle-time-millis: 300000 + # 连接在池中最大生存的时间 + max-evictable-idle-time-millis: 900000 + # 指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除 + test-while-idle: true + # 指明是否在从池中取出连接前进行检验,如果检验失败, 则从池中去除连接并尝试取出另一个 + test-on-borrow: true + # 是否在归还到池中前进行检验 + test-on-return: false + # 检测连接是否有效 + validation-query: select 1 + # 配置监控统计 + webStatFilter: + enabled: true + stat-view-servlet: + enabled: true + url-pattern: /druid/* + reset-enable: false + filter: + stat: + enabled: true + # 记录慢SQL + log-slow-sql: true + slow-sql-millis: 1000 + merge-sql: true + wall: + config: + multi-statement-allow: true + + +# 文件存储 +storage: +# url: /usr/local/webapp/services/source/ +# symbol:/ + url: C:\Users\Admin\Desktop\ + de_symbol: \\ + se_symbol: \ + download-template: /usr/local/webapp/ad-platform/ad-platform-source/custom.xlsx + +# 线程池配置 +save: + task: + thread_pool: + corePoolSize: 2 + maxPoolSize: 16 + queueCapacity: 3 + ThreadNamePrefix: SaveFileTaskExecutor- + +# IP 本地解析 +ip: + local-parsing: true +#密码加密传输，前端公钥加密，后端私钥解密 +rsa: + private_key: MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEA0vfvyTdGJkdbHkB8mp0f3FE0GYP3AYPaJF7jUd1M0XxFSE2ceK3k2kw20YvQ09NJKk+OMjWQl9WitG9pB6tSCQIDAQABAkA2SimBrWC2/wvauBuYqjCFwLvYiRYqZKThUS3MZlebXJiLB+Ue/gUifAAKIg1avttUZsHBHrop4qfJCwAI0+YRAiEA+W3NK/RaXtnRqmoUUkb59zsZUBLpvZgQPfj1MhyHDz0CIQDYhsAhPJ3mgS64NbUZmGWuuNKp5coY2GIj/zYDMJp6vQIgUueLFXv/eZ1ekgz2Oi67MNCk5jeTF2BurZqNLR3MSmUCIFT3Q6uHMtsB9Eha4u7hS31tj1UWE+D+ADzp59MGnoftAiBeHT7gDMuqeJHPL4b+kC+gzV4FGTfhR9q3tTbklZkD2A== diff --git a/services/ad-platform-source/src/main/resources/application-prod.yml b/services/ad-platform-source/src/main/resources/application-prod.yml new file mode 100644 index 00000000..6c281974 --- /dev/null +++ b/services/ad-platform-source/src/main/resources/application-prod.yml @@ -0,0 +1,76 @@ +#配置数据源 +spring: + cloud: + nacos: + discovery: + server-addr: ${NACOS_HOST:118.178.137.129}:${NACOS_PORT:8848} + datasource: + druid: + db-type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.jdbc.Driver + url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:eladmin}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false + username: ${DB_USER:root} + password: ${DB_PWD:123456} + # 初始连接数 + initial-size: 5 + # 最小连接数 + min-idle: 15 + # 最大连接数 + max-active: 30 + # 超时时间(以秒数为单位) + remove-abandoned-timeout: 180 + # 获取连接超时时间 + max-wait: 3000 + # 连接有效性检测时间 + time-between-eviction-runs-millis: 60000 + # 连接在池中最小生存的时间 + min-evictable-idle-time-millis: 300000 + # 连接在池中最大生存的时间 + max-evictable-idle-time-millis: 900000 + # 指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除 + test-while-idle: true + # 指明是否在从池中取出连接前进行检验,如果检验失败, 则从池中去除连接并尝试取出另一个 + test-on-borrow: true + # 是否在归还到池中前进行检验 + test-on-return: false + # 检测连接是否有效 + validation-query: select 1 + # 配置监控统计 + webStatFilter: + enabled: true + stat-view-servlet: + enabled: true + url-pattern: /druid/* + reset-enable: false + filter: + stat: + enabled: true + # 记录慢SQL + log-slow-sql: true + slow-sql-millis: 1000 + merge-sql: true + wall: + config: + multi-statement-allow: true + +# 文件存储 +storage: + url: /usr/local/webapp/services/source/ + de_symbol: / + + +# 线程池配置 +save: + task: + thread_pool: + corePoolSize: 2 + maxPoolSize: 16 + queueCapacity: 3 + ThreadNamePrefix: SaveFileTaskExecutor- + +# IP 本地解析 +ip: + local-parsing: true +#密码加密传输，前端公钥加密，后端私钥解密 +rsa: + private_key: MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEA0vfvyTdGJkdbHkB8mp0f3FE0GYP3AYPaJF7jUd1M0XxFSE2ceK3k2kw20YvQ09NJKk+OMjWQl9WitG9pB6tSCQIDAQABAkA2SimBrWC2/wvauBuYqjCFwLvYiRYqZKThUS3MZlebXJiLB+Ue/gUifAAKIg1avttUZsHBHrop4qfJCwAI0+YRAiEA+W3NK/RaXtnRqmoUUkb59zsZUBLpvZgQPfj1MhyHDz0CIQDYhsAhPJ3mgS64NbUZmGWuuNKp5coY2GIj/zYDMJp6vQIgUueLFXv/eZ1ekgz2Oi67MNCk5jeTF2BurZqNLR3MSmUCIFT3Q6uHMtsB9Eha4u7hS31tj1UWE+D+ADzp59MGnoftAiBeHT7gDMuqeJHPL4b+kC+gzV4FGTfhR9q3tTbklZkD2A== diff --git a/services/ad-platform-source/src/main/resources/application.yml b/services/ad-platform-source/src/main/resources/application.yml index cd01a1af..1ab15b83 100644 --- a/services/ad-platform-source/src/main/resources/application.yml +++ b/services/ad-platform-source/src/main/resources/application.yml @@ -1,10 +1,37 @@ server: - port: 7788 - + port: 7325 spring: - cloud: - nacos: - discovery: - server-addr: ${NACOS_HOST:118.178.137.129}:${NACOS_PORT:8848} + main: + allow-bean-definition-overriding: true + freemarker: + check-template-location: false + profiles: + active: dev application: name: @artifactId@ + jackson: + time-zone: GMT+8 + data: + redis: + repositories: + enabled: false + +# servlet: +# multipart: +# max-file-size: 200MB #单个文件上传大小 +# max-request-size: 600MB #连续上传文件大小 + + #配置 Jpa + jpa: + hibernate: + ddl-auto: none + open-in-view: true + properties: + hibernate: + dialect: org.hibernate.dialect.MySQL5InnoDBDialect + + +#hutool雪花算法 +snowflake: + workerId: 9 + datacenterId: 9 \ No newline at end of file diff --git a/services/ad-platform-source/src/main/resources/logback.xml b/services/ad-platform-source/src/main/resources/logback.xml new file mode 100644 index 00000000..d22f9c88 --- /dev/null +++ b/services/ad-platform-source/src/main/resources/logback.xml @@ -0,0 +1,88 @@ + + + trade-account + + + + + + + + + + %highlight([%-5level]) %cyan(%d{yyyy-MM-dd#HH:mm:ss.SSS}) %yellow([Thread:%thread]) + %magenta([Logger:%logger]) -> %msg%n + + utf-8 + + + + + + ${LOG_DIR}/log.log + + + + ${LOG_DIR}/history/%d{yyyy-MM-dd}.gz + 30 + + + true + + + ${LOG_PATTERN} + utf-8 + + + + + INFO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file