From 3b16f8a531a197ef4b0908d97785d2725d8054cb Mon Sep 17 00:00:00 2001 From: yqy Date: Thu, 3 Mar 2022 14:27:46 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=86=85=E6=89=8B=E6=9C=BA?= =?UTF-8?q?=E5=8F=B7=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/baiye/module/dao/ClueJpa.java | 31 ++++++++++++- .../module/dao/ClueMiddleRepository.java | 8 ++++ .../com/baiye/module/dao/ClueRepository.java | 8 ++++ .../java/com/baiye/task/FileAnalysisTask.java | 44 ++++++++++++++++++- 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueJpa.java b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueJpa.java index 4ca84cff..a3cb2b5f 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueJpa.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueJpa.java @@ -6,10 +6,12 @@ import com.baiye.feign.OrganizeClient; import com.baiye.model.dto.ClueDto; import com.baiye.model.dto.ClueQueryCriteria; import com.baiye.model.vo.ResSourceLabel; +import com.baiye.module.entity.Clue; import com.baiye.util.AESUtils; import com.baiye.util.RedisUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; +import org.hibernate.query.NativeQuery; import org.hibernate.query.internal.NativeQueryImpl; import org.hibernate.transform.Transformers; import org.springframework.beans.factory.annotation.Value; @@ -97,8 +99,8 @@ public class ClueJpa { if (clueQueryCriteria.getMemberStatus() != null) { sql.append("and cm.member_status = :memberStatus "); } - //拼接最后加入时间排序 - sql.append("ORDER BY c.create_time desc "); + //拼接最后加入时间排序,没加id排序出现数据重复问题(注意空格) + sql.append("ORDER BY c.create_time desc ,c.id "); if (pageable != null) { sql.append("LIMIT :number,:size "); @@ -205,4 +207,29 @@ public class ClueJpa { } return list; } + + public List findClue(Long taskId) { + StringBuilder sql = new StringBuilder(); + sql.append("SELECT tc.id as id,tc.nid as nid FROM tb_clue_middle tcm LEFT JOIN tb_clue tc ON tcm.clue_id = tc.id WHERE 1 = 1 "); + if (taskId != null) { + sql.append("AND tcm.task_id = :taskId "); + } + Query query = entityManager.createNativeQuery(sql.toString()); + if (taskId != null) { + query.setParameter("taskId", taskId); + } + query.unwrap(NativeQuery.class).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP); + List resultList = query.getResultList(); + List arrayList = new ArrayList<>(); + for (Object obj : resultList) { + Map row = (Map) obj; + Clue clue = new Clue(); + BigInteger clueId = (BigInteger)row.get("id"); + String phone = (String)row.get("nid"); + clue.setNid(phone); + clue.setId(clueId.longValue()); + arrayList.add(clue); + } + return arrayList; + } } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java index 489c0616..11035921 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java @@ -8,6 +8,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; import java.util.List; +import java.util.Set; @Repository public interface ClueMiddleRepository extends JpaRepository, JpaSpecificationExecutor { @@ -105,4 +106,11 @@ public interface ClueMiddleRepository extends JpaRepository, J */ @Query(value = "select count(*) from tb_clue_middle where task_id = ?1", nativeQuery = true) Integer findTaskNum(Long taskId); + + /** + * Id批量删除 + * + * @param clueIdSet + */ + void deleteAllByClueIdIn(Set clueIdSet); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java index 87c0936c..c2aa6d51 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java +++ b/services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java @@ -5,7 +5,15 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.stereotype.Repository; +import java.util.Set; + @Repository public interface ClueRepository extends JpaRepository, JpaSpecificationExecutor { + /** + * Id批量删除 + * + * @param clueIdList + */ + void deleteAllByIdIn(Set clueIdList); } diff --git a/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java b/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java index 3cb7e433..179f9029 100644 --- a/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java +++ b/services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java @@ -3,6 +3,10 @@ package com.baiye.task; import cn.hutool.core.date.DateUtil; import com.alibaba.excel.EasyExcelFactory; import com.baiye.module.constant.FileConstant; +import com.baiye.module.dao.ClueJpa; +import com.baiye.module.dao.ClueMiddleRepository; +import com.baiye.module.dao.ClueRepository; +import com.baiye.module.entity.Clue; import com.baiye.module.entity.ClueRecord; import com.baiye.module.entity.vo.BaseExcelVo; import com.baiye.module.entity.vo.KsVo; @@ -16,10 +20,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; import java.io.FileInputStream; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; @Component @@ -29,10 +35,20 @@ public class FileAnalysisTask { @Resource private ClueService clueService; + @Resource + private ClueJpa clueJpa; + + @Resource + private ClueRepository clueRepository; + + @Resource + private ClueMiddleRepository clueMiddleRepository; + /** * 读取文件处理数据 * @param clueRecords */ + @Transactional(rollbackFor = Exception.class) @Async(value = "SendBigDataTaskExecutor") @SneakyThrows //处理异常try public void runFileAnalysisTask(List clueRecords,String oneFileName) { @@ -67,7 +83,9 @@ public class FileAnalysisTask { Long taskId = clueRecords.get(0).getTaskId(); Long userId = clueRecords.get(0).getCreateBy(); if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)){ - log.info("================创建任务开始:{}================", DateUtil.now()); + // 任务内手机号去重,先去重然后创建任务(统计任务数量) + distinctNid(taskId); + // 创建任务 BaseExcelVo baseExcelVo = new BaseExcelVo(); baseExcelVo.setUserId(userId); baseExcelVo.setTaskId(taskId); @@ -75,4 +93,26 @@ public class FileAnalysisTask { clueService.saveTask(baseExcelVo); } } + + /** + * 同一个任务内去重手机号 + * @param taskId + */ + @Transactional(rollbackFor = Exception.class) + public void distinctNid(Long taskId){ + List clueList = clueJpa.findClue(taskId); + Map> map = clueList.stream().collect(Collectors.groupingBy(Clue::getNid)); + Set keySet = map.keySet(); + for (String key : keySet) { + if (map.get(key).size() > 1){ + List clues = map.get(key); + clues.remove(0); + Set clueIdSet = new HashSet<>(); + clues.forEach(cs -> clueIdSet.add(cs.getId())); + clueRepository.deleteAllByIdIn(clueIdSet); + log.info("===========要删除的id：{}============",clueIdSet); + clueMiddleRepository.deleteAllByClueIdIn(clueIdSet); + } + } + } }