diff --git a/cdp-iaas/gateway-server/src/main/resources/config/application.yml b/cdp-iaas/gateway-server/src/main/resources/config/application.yml index d535334..0ecac70 100644 --- a/cdp-iaas/gateway-server/src/main/resources/config/application.yml +++ b/cdp-iaas/gateway-server/src/main/resources/config/application.yml @@ -1,7 +1,7 @@ spring: profiles: - active: prod + active: dev application: name: @artifactId@ cloud: diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/EssayHandleController.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/EssayHandleController.java new file mode 100644 index 0000000..ff43c59 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/EssayHandleController.java @@ -0,0 +1,84 @@ +package com.baiye.controller; + +import com.baiye.annotation.Inner; +import com.baiye.core.base.api.Result; +import com.baiye.core.constant.AddGroup; +import com.baiye.core.constant.UpdateGroup; +import com.baiye.dto.EssayDTO; +import com.baiye.entity.LexiconFilter; +import com.baiye.service.EssayHandleService; +import com.baiye.vo.CheckContentVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequestMapping("/xhs/essay") +@RequiredArgsConstructor +@Api(tags = "小红书文章处理") +public class EssayHandleController { + + private final EssayHandleService essayHandleService; + + @GetMapping("/getLexiconFilterList") + @Inner(value = false) + @ApiOperation("获取违禁词列表") + public Result getLexiconFilterList(){ + return Result.data(essayHandleService.getLexiconFilterList()); + } + + @GetMapping("/getLexiconFilterAllList") + @ApiOperation("获取违禁词全部列表") + public Result getLexiconFilterAllList(){ + return Result.data(essayHandleService.getLexiconFilterAllList()); + } + + @PostMapping("/updateLexiconFilter") + @Inner(value = false) + @ApiOperation("更新违禁词") + public Result updateLexiconFilter(@Validated({UpdateGroup.class}) @RequestBody LexiconFilter lexiconFilter){ + essayHandleService.updateLexiconFilter(lexiconFilter); + return Result.success(); + } + + @PostMapping("/addLexiconFilter") + @Inner(value = false) + @ApiOperation("添加违禁词") + public Result addLexiconFilter(@Validated({AddGroup.class}) @RequestBody LexiconFilter lexiconFilter){ + essayHandleService.addLexiconFilter(lexiconFilter); + return Result.success(); + } + + @PostMapping("/checkContent") + @Inner(value = false) + @ApiOperation("检测内容") + public Result checkContent(@RequestBody EssayDTO essayDTO){ + return Result.data(essayHandleService.checkContent(essayDTO)); + } + + @PostMapping("/processArticle") + @Inner(value = false) + @ApiOperation("处理文章") + public Result processArticle(@RequestBody EssayDTO essayDTO){ + return Result.data(essayHandleService.processArticle(essayDTO)); + } + + @GetMapping("/writeInRedis") + @Inner(value = false) + @ApiOperation("向redis写入拼音库") + public Result writeInRedisPinyinLexicon(){ + essayHandleService.writeInRedisPinyinLexicon(); + return Result.success(); + } + + @GetMapping("/writeInForbiddenWords") + @Inner(value = false) + @ApiOperation("文件写入违禁词") + public Result writeInForbiddenWords(@RequestParam("file") MultipartFile file, @RequestParam("organizeId") Long organizeId){ + essayHandleService.writeInForbiddenWords(file, organizeId); + return Result.success(); + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/ProxyController.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/ProxyController.java new file mode 100644 index 0000000..f84db13 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/ProxyController.java @@ -0,0 +1,101 @@ +package com.baiye.controller; + +import com.baiye.annotation.Inner; +import com.baiye.core.base.api.Result; +import com.baiye.core.constant.AddGroup; +import com.baiye.core.constant.UpdateGroup; +import com.baiye.core.page.PageResult; +import com.baiye.dto.ProxyDTO; +import com.baiye.entity.ProxyEntity; +import com.baiye.service.ProxyService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Pageable; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.util.List; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@RestController +@RequestMapping("/xhs/proxy") +@RequiredArgsConstructor +@Api(tags = "小红书代理管理") +public class ProxyController { + + private final ProxyService proxyService; + + @GetMapping("/findProxy") + @Inner(value = false) + @ApiOperation("手机号绑定代理") + public Result findProxy(@RequestParam("phone") String phone){ + return Result.data(proxyService.findProxy(phone)); + } + + @PostMapping("/add") + @Inner(value = false) + @ApiOperation("添加") + public Result add(@Validated({AddGroup.class}) @RequestBody ProxyEntity proxyEntity){ + proxyService.add(proxyEntity); + return Result.success(); + } + + @PostMapping("/update") + @Inner(value = false) + @ApiOperation("修改") + public Result update(@Validated({UpdateGroup.class}) @RequestBody ProxyEntity proxyEntity){ + proxyService.update(proxyEntity); + return Result.success(); + } + + @DeleteMapping("/del") + @Inner(value = false) + @ApiOperation("删除IP-并删除绑定") + public Result del(@RequestParam("id") Long id, @RequestParam("type") Integer type){ + proxyService.del(id, type); + return Result.success(); + } + + @GetMapping("/ipBindingList") + @Inner(value = false) + @ApiOperation("IP绑定手机号列表") + public PageResult ipBindingList(ProxyDTO proxyDTO, Pageable pageable){ + return proxyService.ipBindingList(proxyDTO, pageable); + } + + @GetMapping("/list") + @Inner(value = false) + @ApiOperation("IP列表") + public Result> list(){ + return Result.data(proxyService.list()); + } + + @PostMapping("/delAccount") + @Inner(value = false) + @ApiOperation("删除账号绑定关系") + public Result delAccount(@RequestBody List phoneList){ + proxyService.delAccount(phoneList); + return Result.success(); + } + + @GetMapping("/replaceAccountIp") + @Inner(value = false) + @ApiOperation("替换账号绑定IP") + public Result replaceAccountIp(@RequestParam("proxyId") Long proxyId, @RequestParam("phone") String phone){ + proxyService.replaceAccountIp(proxyId, phone); + return Result.success(); + } + + @GetMapping("/writeProxy") + @Inner(value = false) + @ApiOperation("文件写入代理信息") + public Result writeProxy(@RequestParam("file") MultipartFile file){ + proxyService.writeProxy(file); + return Result.success(); + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/RequestXhsApi.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/RequestXhsApi.java index 4d0af32..cb44b0f 100644 --- a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/RequestXhsApi.java +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/java/com/baiye/controller/RequestXhsApi.java @@ -39,8 +39,32 @@ public class RequestXhsApi { headMap.put("Referer", "https://creator.xiaohongshu.com/creator/home"); headMap.put("Accept-Encoding", "gzip, deflate, br"); headMap.put("Accept-Language", "zh-CN,zh;q=0.9"); -// headMap.put("Cookie","a1=18677d3d9a6vgirqobumcf8567e2apc1iyksxj50j50000143549; webId=1b9718e1a133ca9fdabe28d1ae103034; gid=yYKWWfqfqy3iyYKWWfqfjvV20Khk3FAlD76ST8ViY2KWdM28J0USyC888y4q24j8dfYjjSj8; gid.sign=YckRmYcxiW5ZxdoS3b5F4Smu2eg=; customerBeakerSessionId=187a7799ddc074a0ee4f889e94496c8d94f264e5gAJ9cQAoWBAAAABjdXN0b21lclVzZXJUeXBlcQFLAVgOAAAAX2NyZWF0aW9uX3RpbWVxAkdB2P1t2wd87lgJAAAAYXV0aFRva2VucQNYQQAAADA2ZmQ5NGNlNDUzMzQ5ODg5MmU1NjkwYTMyNGFiNjA2LWI3YTBmNGY5ZTA1OTQ2MGQ4MmVlZWI2ZTNiNjZlMmIxcQRYAwAAAF9pZHEFWCAAAABmYzIwMDIxMjVlOGE0YzY0YWQ3OTI4NmVhYWFlOWM0MXEGWA4AAABfYWNjZXNzZWRfdGltZXEHR0HY/W3bB3zuWAYAAAB1c2VySWRxCFgYAAAANjEyNGY1YzQwMDAwMDAwMDAxMDAzMWUycQl1Lg==; customerClientId=624017522056172; x-user-id-creator.xiaohongshu.com=6124f5c400000000010031e2; access-token=AT-c540766637194806819bc9ba4042e184-f7b4f3f7898a401297ce6177fead94d7; subsystem=ares; sso-type=customer; galaxy.creator.beaker.session.id=1677047660263079219367; xsecappid=creator-creator"); - headMap.put("Cookie", cid); + headMap.put("Cookie", cid); return HttpRequest.get(requestUrl).timeout(10_0000).headerMap(headMap, true).execute().body(); } + + @GetMapping("/xhsApi") + @Inner(value = false) + public String xhsApi(@RequestParam("cid") String cid) { + String requestUrl = "https://edith.xiaohongshu.com/api/sns/web/v1/user/me"; + Map map = new HashMap<>(); + map.put("'Connection'", "keep-alive"); + map.put("Host", "edith.xiaohongshu.com"); + map.put("sec-ch-ua", "\"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Microsoft Edge\";v=\"110\""); + map.put("X-t", "1677822769165"); + map.put("x-b3-traceid", "28a8d74d397d64b7"); + map.put("sec-ch-ua-mobile", "?0"); + map.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.50"); + map.put("Accept", "application/json, text/plain, */*"); + map.put("X-s", "16FCZjMlOls+0jMi1lVU0j9WsBAisBdU1gaUO6ZU0YF3"); + map.put("sec-ch-ua-platform", "\"Windows\""); + map.put("Sec-Fetch-Site", "same-site"); + map.put("Sec-Fetch-Mode", "cors"); + map.put("Sec-Fetch-Dest", "empty"); + map.put("Referer", "https://www.xiaohongshu.com/"); + map.put("Accept-Encoding", "gzip, deflate, br"); + map.put("Accept-Language", "zh-CN,zh;q=0.9"); + map.put("Cookie", cid); + return HttpRequest.get(requestUrl).timeout(10_0000).headerMap(map, true).execute().body(); + } } diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/resources/config/application.yml b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/resources/config/application.yml index f34fe19..8350c66 100644 --- a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/resources/config/application.yml +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-api/src/main/resources/config/application.yml @@ -12,7 +12,7 @@ spring: freemarker: check-template-location: false profiles: - active: prod + active: test jackson: time-zone: GMT+8 data: @@ -33,9 +33,8 @@ spring: bucket: baiye-cdp mybatis-plus: mapper-locations: classpath:mapper/*Mapper.xml - - configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl #控制台打印sql语句 +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl #控制台打印sql语句 snowflake: workerId: 9 diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/LexiconFilterMapper.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/LexiconFilterMapper.java new file mode 100644 index 0000000..3855ce9 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/LexiconFilterMapper.java @@ -0,0 +1,9 @@ +package com.baiye.dao; + +import com.baiye.entity.LexiconFilter; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface LexiconFilterMapper extends BaseMapper { +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/LexiconPinyinMapper.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/LexiconPinyinMapper.java new file mode 100644 index 0000000..a3f2912 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/LexiconPinyinMapper.java @@ -0,0 +1,13 @@ +package com.baiye.dao; + +import com.baiye.entity.LexiconPinyin; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +@Mapper +public interface LexiconPinyinMapper extends BaseMapper { + + void insertAll(List list); +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/ProxyAccountMapper.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/ProxyAccountMapper.java new file mode 100644 index 0000000..74a6a12 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/ProxyAccountMapper.java @@ -0,0 +1,18 @@ +package com.baiye.dao; + +import com.baiye.entity.ProxyAccountEntity; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@Mapper +public interface ProxyAccountMapper extends BaseMapper { + + void insertAll(List list); +} + diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/ProxyMapper.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/ProxyMapper.java new file mode 100644 index 0000000..5bd917e --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dao/ProxyMapper.java @@ -0,0 +1,24 @@ +package com.baiye.dao; + +import com.baiye.entity.ProxyEntity; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@Mapper +public interface ProxyMapper extends BaseMapper { + + List findProxy(@Param("phone") String phone); + + void insertAll(List list); + + void updateExpireDateSubtractOne(); + + void updateExpireDateStatus(); +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dto/EssayDTO.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dto/EssayDTO.java new file mode 100644 index 0000000..b54eb69 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dto/EssayDTO.java @@ -0,0 +1,30 @@ +package com.baiye.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; +import java.util.Set; + +/** + * @Author YQY + * @Date 2023/3/22 + */ +@Data +public class EssayDTO { + + @ApiModelProperty("文章") + private String content; + + @ApiModelProperty("小组ID") + private List organizeId; + + @ApiModelProperty("处理类型:1、拼音首字母替换 2、拼音替换 3、繁体字替换 4、同音字替换") + private Integer type; + + @ApiModelProperty("违禁词") + private Set prohibitedWords; + + @ApiModelProperty("自定义词库") + private String customEssay; +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dto/ProxyDTO.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dto/ProxyDTO.java new file mode 100644 index 0000000..5573fc0 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/dto/ProxyDTO.java @@ -0,0 +1,18 @@ +package com.baiye.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@Data +public class ProxyDTO { + + @ApiModelProperty("手机号") + private String phone; + + @ApiModelProperty("ip地址") + private String ip; +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/LexiconFilter.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/LexiconFilter.java new file mode 100644 index 0000000..01bdb4f --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/LexiconFilter.java @@ -0,0 +1,41 @@ +package com.baiye.entity; + +import com.baiye.core.base.MybatisBaseEntity; +import com.baiye.core.constant.AddGroup; +import com.baiye.core.constant.UpdateGroup; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; + +/** + * @Author YQY + * @Date 2023/3/22 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Accessors(chain = true) +@TableName("tb_lexicon_filter") +@ApiOperation("违禁词库") +public class LexiconFilter extends MybatisBaseEntity { + + @ApiModelProperty(value = "违禁词库主键ID") + @TableId(value = "id", type = IdType.AUTO) + @NotNull(message = "ID不能为空", groups = UpdateGroup.class) + private Long id; + + @NotNull(message = "词不能为空", groups = AddGroup.class) + private String speech; + + @NotNull(message = "组ID不能为空", groups = AddGroup.class) + private Long organizeId; + + @NotNull(message = "组名称不能为空", groups = AddGroup.class) + private String organizeName; +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/LexiconPinyin.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/LexiconPinyin.java new file mode 100644 index 0000000..e29bab4 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/LexiconPinyin.java @@ -0,0 +1,33 @@ +package com.baiye.entity; + +import com.baiye.core.base.MybatisBaseEntity; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +/** + * @Author YQY + * @Date 2023/3/22 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Accessors(chain = true) +@TableName("tb_lexicon_pinyin") +@ApiOperation("拼音词库") +public class LexiconPinyin extends MybatisBaseEntity { + + @ApiModelProperty(value = "拼音词库主键ID") + @TableId(value = "id", type = IdType.ASSIGN_ID) + private Long id; + + @ApiModelProperty("拼音") + private String phoneticTranscription; + + @ApiModelProperty("词") + private String words; +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/ProxyAccountEntity.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/ProxyAccountEntity.java new file mode 100644 index 0000000..7d28f04 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/ProxyAccountEntity.java @@ -0,0 +1,38 @@ +package com.baiye.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@Data +@Accessors(chain = true) +@TableName("tb_proxy_account") +@ApiOperation("代理地址手机号映射表") +@AllArgsConstructor +@NoArgsConstructor +public class ProxyAccountEntity { + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @ApiModelProperty("手机号") + private String nid; + + @ApiModelProperty("映射IP的ID") + private Long proxyId; + + public ProxyAccountEntity(String nid, Long proxyId) { + this.nid = nid; + this.proxyId = proxyId; + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/ProxyEntity.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/ProxyEntity.java new file mode 100644 index 0000000..fb072f6 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/entity/ProxyEntity.java @@ -0,0 +1,109 @@ +package com.baiye.entity; + +import com.baiye.core.constant.AddGroup; +import com.baiye.core.constant.UpdateGroup; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; +import org.springframework.format.annotation.DateTimeFormat; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@Data +@Accessors(chain = true) +@TableName("tb_proxy") +@ApiOperation("代理地址表") +@AllArgsConstructor +@NoArgsConstructor +public class ProxyEntity { + + @TableId(value = "id", type = IdType.AUTO) + @NotNull(groups = UpdateGroup.class, message = "ID不能为空") + private Long id; + + @ApiModelProperty("ip地址") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "ip地址不能为空") + private String ip; + + @ApiModelProperty("端口") + @NotNull(groups = {AddGroup.class, UpdateGroup.class}, message = "端口不能为空") + private Integer port; + + @ApiModelProperty("账户") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "账户不能为空") + private String account; + + @ApiModelProperty("密码") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "密码不能为空") + private String password; + + @ApiModelProperty("地址") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "地址不能为空") + private String address; + + @ApiModelProperty("协议类型(1：HTTP 2：HTTPS 3：Socket5: 4：Socket4)") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "协议类型不能为空") + private String agreementType; + + @ApiModelProperty("剩余使用时间/天") + @NotNull(groups = {AddGroup.class, UpdateGroup.class}, message = "剩余使用时间不能为空") + private Integer surplusDay; + + @ApiModelProperty("是否是境内(1:境内 2:境外)") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "是否是境内不能为空") + private String isDomestic; + + @ApiModelProperty("代理商") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "代理商不能为空") + private String agent; + + @ApiModelProperty("状态(0：可用 1：不可用)") + @NotBlank(groups = {AddGroup.class, UpdateGroup.class}, message = "状态不能为空") + private String status; + + @ApiModelProperty("创建时间") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + + @TableField(exist = false) + private List phoneList = new ArrayList<>(); + + public ProxyEntity(String ip, Integer port, String account, String password, String address, + String agreementType, Integer surplusDay, String isDomestic, String agent, String status) { + this.ip = ip; + this.port = port; + this.account = account; + this.password = password; + this.address = address; + this.agreementType = agreementType; + this.surplusDay = surplusDay; + this.isDomestic = isDomestic; + this.agent = agent; + this.status = status; + } + + public ProxyEntity(String ip, Integer port, String account, String password, String address) { + this.ip = ip; + this.port = port; + this.account = account; + this.password = password; + this.address = address; + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/vo/CheckContentVO.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/vo/CheckContentVO.java new file mode 100644 index 0000000..601caad --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-core/src/main/java/com/baiye/vo/CheckContentVO.java @@ -0,0 +1,38 @@ +package com.baiye.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; + +import java.util.Set; + +/** + * @Author YQY + * @Date 2023/3/22 + */ +@Getter +@Setter +public class CheckContentVO { + + @ApiModelProperty("内容") + private String content; + + @ApiModelProperty("内容长度") + private Integer contentCount; + + @ApiModelProperty("违禁词数量") + private Integer prohibitedWordCount; + + @ApiModelProperty("违禁词") + private Set prohibitedWords; + + public CheckContentVO() { + } + + public CheckContentVO(String content, Integer contentCount, Integer prohibitedWordCount, Set prohibitedWords) { + this.content = content; + this.contentCount = contentCount; + this.prohibitedWordCount = prohibitedWordCount; + this.prohibitedWords = prohibitedWords; + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/pom.xml b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/pom.xml index 1fe94ca..8f46453 100644 --- a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/pom.xml +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/pom.xml @@ -130,10 +130,10 @@ cdp-common-security 1.0-SNAPSHOT - - com.alibaba.cloud - spring-cloud-starter-alicloud-oss - + + + + org.apache.poi poi @@ -149,5 +149,37 @@ poi-ooxml-schemas 4.0.0 + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + com.github.houbb + opencc4j + 1.7.2 + + + + com.belerweb + pinyin4j + 2.5.1 + + + + + com.alibaba + easyexcel + 2.2.7 + + diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/JDKConfig.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/JDKConfig.java new file mode 100644 index 0000000..11173a0 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/JDKConfig.java @@ -0,0 +1,18 @@ +package com.baiye.config; + +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; + +/** + * @Author YQY + * @Date 2023/4/3 + */ +@Configuration +public class JDKConfig { + + @PostConstruct + public void httpsProxyConfig() { + System.setProperty("jdk.http.auth.tunneling.disabledSchemes", ""); + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/OssConfig.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/OssConfig.java index ac2ebc9..a750205 100644 --- a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/OssConfig.java +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/config/OssConfig.java @@ -1,33 +1,33 @@ -package com.baiye.config; - -import com.alibaba.alicloud.context.AliCloudProperties; -import com.alibaba.alicloud.context.oss.OssProperties; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; -import lombok.RequiredArgsConstructor; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; - -/** - * @author Enzo - * @date : 2022/8/15 - */ -@Configuration -@RequiredArgsConstructor -public class OssConfig { - - private final OssProperties ossProperties; - private final AliCloudProperties aliCloudProperties; - - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public OSS ossClient() { - return new OSSClientBuilder().build - (ossProperties.getEndpoint(), - aliCloudProperties.getAccessKey(), - aliCloudProperties.getSecretKey()); - } - -} +//package com.baiye.config; +// +//import com.alibaba.alicloud.context.AliCloudProperties; +//import com.alibaba.alicloud.context.oss.OssProperties; +//import com.aliyun.oss.OSS; +//import com.aliyun.oss.OSSClientBuilder; +//import lombok.RequiredArgsConstructor; +//import org.springframework.beans.factory.config.ConfigurableBeanFactory; +//import org.springframework.context.annotation.Bean; +//import org.springframework.context.annotation.Configuration; +//import org.springframework.context.annotation.Scope; +// +///** +// * @author Enzo +// * @date : 2022/8/15 +// */ +//@Configuration +//@RequiredArgsConstructor +//public class OssConfig { +// +// private final OssProperties ossProperties; +// private final AliCloudProperties aliCloudProperties; +// +// @Bean +// @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +// public OSS ossClient() { +// return new OSSClientBuilder().build +// (ossProperties.getEndpoint(), +// aliCloudProperties.getAccessKey(), +// aliCloudProperties.getSecretKey()); +// } +// +//} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/easyexcel/dto/ProxyDto.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/easyexcel/dto/ProxyDto.java new file mode 100644 index 0000000..803021f --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/easyexcel/dto/ProxyDto.java @@ -0,0 +1,67 @@ +package com.baiye.easyexcel.dto; + +import com.alibaba.excel.annotation.ExcelProperty; +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + +/** + * @Author YQY + * @Date 2023/4/7 + */ +@Data +@ApiModel(value = "代理excel") +public class ProxyDto { + + @ExcelProperty(value = "IP地址", index = 0) + @ApiModelProperty(value = "IP地址") + private String ip; + + @ExcelProperty(value = "端口号", index = 1) + @ApiModelProperty(value = "端口号") + private Integer port; + + @ExcelProperty(value = "用户名", index = 2) + @ApiModelProperty(value = "用户名") + private String account; + + @ExcelProperty(value = "密码", index = 3) + @ApiModelProperty(value = "密码") + private String password; + + @ExcelProperty(value = "城市", index = 4) + @ApiModelProperty(value = "城市") + private String address; + + @ExcelProperty(value = "剩余使用时间/天", index = 5) + @ApiModelProperty(value = "剩余使用时间/天") + private Integer surplusDay; + + @ExcelProperty(value = "是否是境内(1:境内 2:境外)", index = 6) + @ApiModelProperty(value = "是否是境内(1:境内 2:境外)") + private String isDomestic; + + @ExcelProperty(value = "代理商", index = 7) + @ApiModelProperty(value = "代理商") + private String agent; + + @ExcelProperty(value = "状态(0：可用 1：不可用)", index = 8) + @ApiModelProperty(value = "状态(0：可用 1：不可用)") + private String status; + + @ApiModelProperty("协议类型(1：HTTP 2：HTTPS 3：Socket5: 4：Socket4)") + @ExcelProperty(value = "协议类型(1：HTTP 2：HTTPS 3：Socket5: 4：Socket4)", index = 9) + private String agreementType; + + @ApiModelProperty("手机号") + @ExcelProperty(value = "手机号(中文逗号隔断)", index = 10) + private String phone; + + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime = new Date(); +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/easyexcel/listener/ProxyListener.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/easyexcel/listener/ProxyListener.java new file mode 100644 index 0000000..6549f00 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/easyexcel/listener/ProxyListener.java @@ -0,0 +1,63 @@ +package com.baiye.easyexcel.listener; + +import cn.hutool.core.collection.CollUtil; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.baiye.core.util.StringUtils; +import com.baiye.easyexcel.dto.ProxyDto; +import com.baiye.entity.ProxyEntity; +import com.baiye.service.ProxyService; +import com.baiye.util.MobileUtil; +import org.springframework.beans.BeanUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Author YQY + * @Date 2023/4/7 + */ +public class ProxyListener extends AnalysisEventListener { + + public final List rows = new ArrayList<>(); + private final ProxyService proxyService; + + public ProxyListener(ProxyService proxyService) { + this.proxyService = proxyService; + } + + @Override + public void invoke(ProxyDto proxyDto, AnalysisContext analysisContext) { + if (StringUtils.isNotBlank(proxyDto.getIp()) && proxyDto.getPort() != null && StringUtils.isNotBlank(proxyDto.getAccount()) + && StringUtils.isNotBlank(proxyDto.getPassword()) && proxyDto.getSurplusDay() != null + && StringUtils.isNotBlank(proxyDto.getStatus()) && StringUtils.isNotBlank(proxyDto.getAgreementType())) { + rows.add(proxyDto); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext analysisContext) { + + List list = new ArrayList<>(); + if (CollUtil.isNotEmpty(rows)) { + for (ProxyDto row : rows) { + // 解析手机号字符串 + List phoneList = new ArrayList<>(); + String phoneStr = row.getPhone(); + if (StringUtils.isNotBlank(phoneStr)) { + String[] split = phoneStr.split("，"); + for (String phone : split) { + if (MobileUtil.checkPhone(phone)) phoneList.add(phone); + } + } + + ProxyEntity proxyEntity = new ProxyEntity(); + BeanUtils.copyProperties(row, proxyEntity); + proxyEntity.setPhoneList(phoneList); + list.add(proxyEntity); + } + rows.clear(); + } + if (CollUtil.isNotEmpty(list)) proxyService.insertAll(list); + } +} \ No newline at end of file diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/EssayHandleService.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/EssayHandleService.java new file mode 100644 index 0000000..d4f2969 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/EssayHandleService.java @@ -0,0 +1,50 @@ +package com.baiye.service; + +import com.baiye.dto.EssayDTO; +import com.baiye.entity.LexiconFilter; +import com.baiye.vo.CheckContentVO; +import org.springframework.web.multipart.MultipartFile; + +public interface EssayHandleService { + + /** + * 获取违禁过滤词 + */ + Object getLexiconFilterList(); + + /** + * 获取违禁词全部列表 + */ + Object getLexiconFilterAllList(); + + /** + * 修改违禁过滤词 + */ + void updateLexiconFilter(LexiconFilter lexiconFilter); + + /** + * 添加违禁词 + */ + void addLexiconFilter(LexiconFilter lexiconFilter); + + /** + * 内容检测 + */ + CheckContentVO checkContent(EssayDTO essayDTO); + + /** + * 处理文章 + */ + Object processArticle(EssayDTO essayDTO); + + /** + * 向redis写入拼音库 + */ + void writeInRedisPinyinLexicon(); + + /** + * 文件写入违禁词 + * @param organizeId 小组ID + */ + void writeInForbiddenWords(MultipartFile file, Long organizeId); +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/ProxyService.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/ProxyService.java new file mode 100644 index 0000000..b4b9b3b --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/ProxyService.java @@ -0,0 +1,66 @@ +package com.baiye.service; + +import com.baiye.core.page.PageResult; +import com.baiye.dto.ProxyDTO; +import com.baiye.entity.ProxyEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.web.multipart.MultipartFile; + +import java.util.List; + +public interface ProxyService { + + /** + * 手机号绑定代理 + */ + ProxyEntity findProxy(String phone); + + /** + * 添加IP + */ + void add(ProxyEntity proxyEntity); + + /** + * 修改IP-并修改绑定 + */ + void update(ProxyEntity proxyEntity); + + /** + * 删除IP-并删除绑定 + * + * @param type 0:删除IP和所有映射账号 1:删除IP-删除账号重新绑定IP + */ + void del(Long id, Integer type); + + /** + * IP绑定手机号列表 + */ + PageResult ipBindingList(ProxyDTO proxyDTO, Pageable pageable); + + /** + * IP列表 + */ + List list(); + + /** + * 删除手机号绑定关系 + * + * @param phoneList 手机号 + */ + void delAccount(List phoneList); + + /** + * 替换账号绑定IP + */ + void replaceAccountIp(Long proxyId, String phone); + + /** + * 列表插入 + */ + void insertAll(List proxyEntityList); + + /** + * 文件写入代理信息 + */ + void writeProxy(MultipartFile file); +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/impl/EssayHandleServiceImpl.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/impl/EssayHandleServiceImpl.java new file mode 100644 index 0000000..97fa8d6 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/impl/EssayHandleServiceImpl.java @@ -0,0 +1,236 @@ +package com.baiye.service.impl; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.RandomUtil; +import cn.hutool.dfa.WordTree; +import cn.hutool.extra.pinyin.PinyinUtil; +import cn.hutool.http.HtmlUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONUtil; +import com.baiye.BadRequestException; +import com.baiye.core.constant.DefaultNumberConstants; +import com.baiye.core.util.RedisUtils; +import com.baiye.core.util.StringUtils; +import com.baiye.dao.LexiconFilterMapper; +import com.baiye.dao.LexiconPinyinMapper; +import com.baiye.dto.EssayDTO; +import com.baiye.entity.LexiconFilter; +import com.baiye.entity.LexiconPinyin; +import com.baiye.service.EssayHandleService; +import com.baiye.vo.CheckContentVO; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.github.houbb.opencc4j.util.ZhConverterUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @Author YQY + * @Date 2023/3/22 + */ +@Service +@Slf4j +@RequiredArgsConstructor +public class EssayHandleServiceImpl implements EssayHandleService { + + private final LexiconFilterMapper lexiconFilterMapper; + private final LexiconPinyinMapper lexiconPinyinMapper; + private final RedisUtils redisUtils; + + private static final String PINYIN = "xhs:pinyin:lexicon"; + private static final String FILTER_ESSAY_COUNT = "xhs:filter:essay:count"; + + @Override + public Object getLexiconFilterList() { + Map returnMap = new HashMap<>(); + + List lexiconFilterList = lexiconFilterMapper.selectList( + new LambdaQueryWrapper().orderByAsc(LexiconFilter::getOrganizeId)); + Map> map = lexiconFilterList.stream().collect(Collectors.groupingBy(LexiconFilter::getOrganizeId)); + + Map listMap = new HashMap<>(); + for (Long lexiconFilterId : map.keySet()) { + listMap.put(map.get(lexiconFilterId).get(DefaultNumberConstants.ZERO_NUMBER).getOrganizeName(), lexiconFilterId); + } + returnMap.put("list", listMap); + + Object obj = redisUtils.get(FILTER_ESSAY_COUNT); + if (obj != null) returnMap.put("count", Long.parseLong(obj.toString())); + else returnMap.put("count", DefaultNumberConstants.ZERO_NUMBER); + + return returnMap; + } + + @Override + public Object getLexiconFilterAllList() { + List filterList = lexiconFilterMapper.selectList(new LambdaQueryWrapper<>()); + return filterList.stream().collect(Collectors.groupingBy(LexiconFilter::getOrganizeName)); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void updateLexiconFilter(LexiconFilter lexiconFilter) { + lexiconFilterMapper.updateById(lexiconFilter); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void addLexiconFilter(LexiconFilter lexiconFilter) { + lexiconFilterMapper.insert(lexiconFilter); + } + + @Override + public CheckContentVO checkContent(EssayDTO essayDTO) { + String content = essayDTO.getContent(); + int length = content.length(); + List organizeId = essayDTO.getOrganizeId(); + String customEssay = essayDTO.getCustomEssay(); + + //1、查询选择的违禁词 + List lexiconFilters = lexiconFilterMapper.selectList( + new LambdaQueryWrapper().in(LexiconFilter::getOrganizeId, organizeId)); + Set speechList = lexiconFilters.stream().map(LexiconFilter::getSpeech).collect(Collectors.toSet()); + + WordTree tree = new WordTree(); + //2、自定义违禁词添加到WordTree中 + if (StringUtils.isNotBlank(customEssay)) { + String[] split = customEssay.split("，"); + for (String word : split) tree.addWord(word); + } + if (CollUtil.isNotEmpty(speechList)) { + //3、违禁词添加到WordTree中 + speechList.forEach(tree::addWord); + //4、违禁词在文章中匹配,结果->文章内出现的违禁词集合 + List filterSpeechList = tree.matchAll(content, -1, false, false); + //5、文章中的违禁词集合去重 + Set filterSpeechSet = new HashSet<>(filterSpeechList); + //6、文章中的违禁词替换红色样式标红 + for (String prohibitSpeech : filterSpeechSet) { + content = content.replace(prohibitSpeech, "" + prohibitSpeech + ""); + } + //7、检测文件总数统计 + Object obj = redisUtils.get(FILTER_ESSAY_COUNT); + if (obj != null) + redisUtils.set(FILTER_ESSAY_COUNT, Long.parseLong(obj.toString()) + RandomUtil.randomInt(1, 11)); + else redisUtils.set(FILTER_ESSAY_COUNT, RandomUtil.randomInt(1, 11)); + //8、返回 + return new CheckContentVO(content, length, filterSpeechList.size(), filterSpeechSet); + } + throw new BadRequestException("检测失败!!!"); + } + + @Override + public Object processArticle(EssayDTO essayDTO) { + String content = essayDTO.getContent(); + Integer type = essayDTO.getType(); + Set prohibitedWords = essayDTO.getProhibitedWords(); + + if (CollUtil.isNotEmpty(prohibitedWords)) { + for (String prohibitedWord : prohibitedWords) { + switch (type) { + case 1: + String firstLetter = PinyinUtil.getFirstLetter(prohibitedWord, ""); + content = content.replaceAll(prohibitedWord, firstLetter); + break; + case 2: + String pinyin = PinyinUtil.getPinyin(prohibitedWord, ""); + content = content.replaceAll(prohibitedWord, pinyin); + break; + case 3: + String convert = ZhConverterUtil.toTraditional(prohibitedWord); + content = content.replaceAll(prohibitedWord, convert); + break; + case 4: + String py = PinyinUtil.getPinyin(prohibitedWord, "_"); + JSONArray array = JSONUtil.parseArray(redisUtils.hget(PINYIN, py)); + if (CollUtil.isEmpty(array)) { + StringBuilder s = new StringBuilder(); + String[] split = py.split("_"); + for (String str : split) { + JSONArray arr = JSONUtil.parseArray(redisUtils.hget(PINYIN, str)); + if (CollUtil.isNotEmpty(arr)){ + String value = String.valueOf(arr.get(0)); + if (prohibitedWord.contains(value)){ + if (arr.get(1) != null) value = String.valueOf(arr.get(1)); + } + s.append(value); + } + } + array.set(s); + } + if (CollUtil.isNotEmpty(array)) { + int i = 0; + String dismantle = String.valueOf(array.get(i)); + if (dismantle.equals(prohibitedWord)) dismantle = (String) array.get(++i); + content = content.replaceAll(prohibitedWord, dismantle); + } else { + content = HtmlUtil.cleanHtmlTag(content); + } + break; + default: + throw new BadRequestException("暂不支持此方式!!!"); + } + } + } + return content; + } + + @Override + public void writeInRedisPinyinLexicon() { + Map map = new HashMap<>(); + List lexiconPinyinList = lexiconPinyinMapper.selectList(new QueryWrapper<>()); + for (LexiconPinyin lexiconPinyin : lexiconPinyinList) { + map.put(lexiconPinyin.getPhoneticTranscription(), lexiconPinyin.getWords()); + if (map.size() >= 100000) { + redisUtils.hmset(PINYIN, map); + map.clear(); + } + } + if (CollUtil.isNotEmpty(map)) redisUtils.hmset(PINYIN, map); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void writeInForbiddenWords(MultipartFile file, Long organizeId) { + if (file == null) throw new BadRequestException("文件不能为空"); + String extName = FileUtil.extName(file.getOriginalFilename()); + if (!extName.equals("txt")) throw new BadRequestException("只支持txt文件"); + try { + InputStreamReader reader = new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8); + // 读取到缓冲区 + BufferedReader br = new BufferedReader(reader); + String line; + int i = 0; + String organizeName = null; + // 一次读入一行数据 + while ((line = br.readLine()) != null) { + String[] split = line.split("\t"); + if (i == 0) { + organizeName = split[0]; + ++i; + } else { + LexiconFilter lexiconFilter = new LexiconFilter(); + lexiconFilter.setSpeech(split[0]); + lexiconFilter.setOrganizeId(organizeId); + lexiconFilter.setOrganizeName(organizeName); + lexiconFilterMapper.insert(lexiconFilter); + } + } + reader.close(); + br.close(); + } catch (IOException e) { + throw new BadRequestException("上传错误"); + } + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/impl/ProxyServiceImpl.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/impl/ProxyServiceImpl.java new file mode 100644 index 0000000..6f8685b --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/service/impl/ProxyServiceImpl.java @@ -0,0 +1,287 @@ +package com.baiye.service.impl; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.RandomUtil; +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.alibaba.excel.EasyExcelFactory; +import com.baiye.BadRequestException; +import com.baiye.config.JDKConfig; +import com.baiye.core.page.PageResult; +import com.baiye.core.util.StringUtils; +import com.baiye.dao.ProxyMapper; +import com.baiye.dao.ProxyAccountMapper; +import com.baiye.dto.ProxyDTO; +import com.baiye.easyexcel.dto.ProxyDto; +import com.baiye.easyexcel.listener.ProxyListener; +import com.baiye.entity.ProxyEntity; +import com.baiye.entity.ProxyAccountEntity; +import com.baiye.service.ProxyService; +import com.baiye.util.MobileUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @Author YQY + * @Date 2023/3/29 + */ +@Service +@Slf4j +@RequiredArgsConstructor +public class ProxyServiceImpl implements ProxyService { + + private final ProxyMapper proxyMapper; + private final ProxyAccountMapper proxyAccountMapper; + private final String QUILT_PROXY_URL = "https://tool.lu/ip"; + + @Override + @Transactional(rollbackFor = Exception.class) + public ProxyEntity findProxy(String phone) { + + // 1、校验手机号 + if (!MobileUtil.checkPhone(phone)) throw new BadRequestException("请输入正确手机号!!!"); + // 2、有绑定关系直接返回 + ProxyAccountEntity entity = proxyAccountMapper.selectOne(new LambdaQueryWrapper().eq(ProxyAccountEntity::getNid, phone)); + if (entity != null && entity.getId() != null) { + return proxyMapper.selectById(entity.getProxyId()); + } + + List proxyList = proxyMapper.selectList(new LambdaQueryWrapper().eq(ProxyEntity::getStatus, 0)); + if (CollUtil.isNotEmpty(proxyList)) { + //3、获取一个可用代理地址 + ProxyEntity proxyEntity = proxyList.get(RandomUtil.randomInt(0, proxyList.size())); + try { + String account = proxyEntity.getAccount(); + String password = proxyEntity.getPassword(); + /** + * 代理配置 以下配置要在项目启动之前{@link JDKConfig}配置 + * System.setProperty("jdk.http.auth.tunneling.disabledSchemes", ""); + */ + Authenticator.setDefault( + new Authenticator() { + @Override + public PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(account, password.toCharArray()); + } + } + ); + HttpResponse execute = HttpRequest.get(QUILT_PROXY_URL) + .setHttpProxy(proxyEntity.getIp(), proxyEntity.getPort()) + .basicProxyAuth(account, password) + .execute(); + if (execute.getStatus() != 200) { + log.info("代理请求失败: {}", proxyEntity.getIp()); + proxyEntity = askProxy(proxyList, proxyEntity); + } + } catch (Exception e) { + log.error("无效的代理地址: {}, 错误信息:{}", proxyEntity.getIp(), e.getMessage()); + proxyEntity = askProxy(proxyList, proxyEntity); + } + + //4、添加代理地址-手机号映射关系 + if (proxyEntity.getId() != null) { + ProxyAccountEntity proxyAccountEntity = new ProxyAccountEntity(); + proxyAccountEntity.setNid(phone); + proxyAccountEntity.setProxyId(proxyEntity.getId()); + proxyAccountMapper.insert(proxyAccountEntity); + return proxyEntity; + } else { + throw new BadRequestException("无可用代理,请联系管理员!!!"); + } + } + throw new BadRequestException("无可用代理,请联系管理员!!!"); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void add(ProxyEntity proxyEntity) { + // 添加IP + proxyEntity.setCreateTime(new Date()); + proxyMapper.insert(proxyEntity); + Long id = proxyEntity.getId(); + + // IP绑定账号 + List proxyAccountEntities = new ArrayList<>(); + List phoneList = proxyEntity.getPhoneList(); + if (CollUtil.isNotEmpty(phoneList)) { + for (String phone : phoneList) { + if (MobileUtil.checkPhone(phone)) { + ProxyAccountEntity proxyAccountEntity = new ProxyAccountEntity(phone, id); + proxyAccountEntities.add(proxyAccountEntity); + } + } + } + if (CollUtil.isNotEmpty(proxyAccountEntities)) proxyAccountMapper.insertAll(proxyAccountEntities); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void update(ProxyEntity proxyEntity) { + + // 修改IP的信息 + proxyMapper.updateById(proxyEntity); + List phoneList = proxyEntity.getPhoneList(); + List list = new ArrayList<>(); + + List proxyAccountEntities = proxyAccountMapper.selectList(new LambdaQueryWrapper() + .eq(ProxyAccountEntity::getProxyId, proxyEntity.getId())); + Map> map = proxyAccountEntities.stream().collect(Collectors.groupingBy(ProxyAccountEntity::getNid)); + + // 获取所有要绑定的账号-并添加绑定关系 + for (String phone : phoneList) { + if (!map.containsKey(phone)) { + ProxyAccountEntity proxyAccountEntity = new ProxyAccountEntity(phone, proxyEntity.getId()); + list.add(proxyAccountEntity); + } + } + if (CollUtil.isNotEmpty(list)) proxyAccountMapper.insertAll(list); + + // 删除的账号-删除绑定关系 + Set phones = map.keySet(); + List deleteList = phones.stream().filter(t -> !phoneList.contains(t)).collect(Collectors.toList()); + if (CollUtil.isNotEmpty(deleteList)) { + proxyAccountMapper.delete(new LambdaQueryWrapper().in(ProxyAccountEntity::getNid, deleteList)); + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void del(Long id, Integer type) { + proxyMapper.deleteById(id); + if (type == 0) { + // 删除绑定关系 + proxyAccountMapper.delete(new LambdaQueryWrapper().eq(ProxyAccountEntity::getProxyId, id)); + } else { + List entities = proxyAccountMapper.selectList(new LambdaQueryWrapper().eq(ProxyAccountEntity::getProxyId, id)); + proxyAccountMapper.delete(new LambdaQueryWrapper().eq(ProxyAccountEntity::getProxyId, id)); + // 账号重新绑定IP + List phoneList = entities.stream().map(ProxyAccountEntity::getNid).collect(Collectors.toList()); + phoneList.forEach(this::findProxy); + } + } + + @Override + public PageResult ipBindingList(ProxyDTO proxyDTO, Pageable pageable) { + String ip = proxyDTO.getIp(); + String phone = proxyDTO.getPhone(); + + Page page = new Page<>(pageable.getPageNumber(), pageable.getPageSize()); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + if (StringUtils.isNotBlank(ip)) wrapper.eq(ProxyEntity::getIp, ip); + + List entities; + if (StringUtils.isNotBlank(phone)) { + entities = proxyAccountMapper.selectList(new LambdaQueryWrapper().like(ProxyAccountEntity::getNid, phone)); + List proxyAccountIds = entities.stream().map(ProxyAccountEntity::getProxyId).collect(Collectors.toList()); + if (CollUtil.isNotEmpty(proxyAccountIds)) wrapper.in(ProxyEntity::getId, proxyAccountIds); + else return PageResult.success(0L, 0L, new ArrayList<>()); + } else { + entities = proxyAccountMapper.selectList(new LambdaQueryWrapper<>()); + } + Map> map = entities.stream().collect(Collectors.groupingBy(ProxyAccountEntity::getProxyId)); + + Page iPage = proxyMapper.selectPage(page, wrapper); + for (ProxyEntity record : iPage.getRecords()) { + if (map.containsKey(record.getId())) { + List phoneList = map.get(record.getId()).stream().map(ProxyAccountEntity::getNid).collect(Collectors.toList()); + record.setPhoneList(phoneList); + } + } + return PageResult.success(iPage.getTotal(), iPage.getPages(), iPage.getRecords()); + } + + @Override + public List list() { + return proxyMapper.selectList(new QueryWrapper<>()); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void delAccount(List phoneList) { + proxyAccountMapper.delete(new LambdaQueryWrapper().in(ProxyAccountEntity::getNid, phoneList)); + } + + @Override + public void replaceAccountIp(Long proxyId, String phone) { + ProxyAccountEntity proxyAccountEntity = proxyAccountMapper.selectOne(new LambdaQueryWrapper() + .eq(ProxyAccountEntity::getNid, phone)); + + proxyAccountEntity.setProxyId(proxyId); + proxyAccountMapper.updateById(proxyAccountEntity); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void insertAll(List proxyEntityList) { + proxyMapper.insertAll(proxyEntityList); + + List list = new ArrayList<>(); + for (ProxyEntity proxyEntity : proxyEntityList) { + List phoneList = proxyEntity.getPhoneList(); + for (String phone : phoneList) { + ProxyAccountEntity proxyAccountEntity = new ProxyAccountEntity(phone, proxyEntity.getId()); + list.add(proxyAccountEntity); + } + } + if (CollUtil.isNotEmpty(list)) proxyAccountMapper.insertAll(list); + } + + @Override + public void writeProxy(MultipartFile file) { + if (file == null) throw new BadRequestException("文件不能为空!!!"); + + String extName = FileUtil.extName(file.getOriginalFilename()); + if (extName != null && !(extName.equals("xlsx") || extName.equals("xls"))) { + throw new BadRequestException("仅支持Excel文件!!!"); + } + try { + EasyExcelFactory.read(file.getInputStream(), ProxyDto.class, new ProxyListener(this)).build().readAll(); + } catch (Exception e) { + log.error("错误信息:{}", e.getMessage()); + throw new BadRequestException("上传错误,请联系管理员!!!"); + } + } + + /** + * 随机代理失败后,循环获取可用代理 + */ + private ProxyEntity askProxy(List proxyList, ProxyEntity proxyEntity) { + proxyList.remove(proxyEntity); + if (CollUtil.isNotEmpty(proxyList)) { + for (ProxyEntity entity : proxyList) { + try { + Authenticator.setDefault( + new Authenticator() { + @Override + public PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(entity.getAccount(), entity.getPassword().toCharArray()); + } + } + ); + HttpResponse execute = HttpRequest.get(QUILT_PROXY_URL) + .setHttpProxy(entity.getIp(), entity.getPort()) + .basicProxyAuth(entity.getAccount(), entity.getPassword()) + .execute(); + if (execute.getStatus() != 200) log.info("无效的代理地址: {}", entity.getIp()); + else return entity; + } catch (Exception e) { + log.error("无效的代理地址: {}, 报错信息:{}", entity.getIp(), e.getMessage()); + } + } + } + return new ProxyEntity(); + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/task/ProxyExpireDateTask.java b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/task/ProxyExpireDateTask.java new file mode 100644 index 0000000..dc54b14 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/java/com/baiye/task/ProxyExpireDateTask.java @@ -0,0 +1,31 @@ +package com.baiye.task; + +import cn.hutool.core.date.DateUtil; +import com.baiye.dao.ProxyMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +/** + * @Author YQY + * @Date 2023/4/10 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ProxyExpireDateTask { + + private final ProxyMapper proxyMapper; + /** + * 代理地址到期时间减一天(每天0点) + */ + @Scheduled(cron = "0 0 0 * * ? ") + @Transactional(rollbackFor = Exception.class) + public void checkProxyExpireDate() { + log.info("--------------代理地址到期时间减一天,执行时间{}--------------", DateUtil.now()); + proxyMapper.updateExpireDateSubtractOne(); + proxyMapper.updateExpireDateStatus(); + } +} diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/LexiconFilterMapper.xml b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/LexiconFilterMapper.xml new file mode 100644 index 0000000..38bf4a9 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/LexiconFilterMapper.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/LexiconPinyinMapper.xml b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/LexiconPinyinMapper.xml new file mode 100644 index 0000000..89d4a4c --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/LexiconPinyinMapper.xml @@ -0,0 +1,13 @@ + + + + + + insert into tb_lexicon_pinyin + (phonetic_transcription, words) + values + + (#{item.phoneticTranscription}, #{item.words}) + + + \ No newline at end of file diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/ProxyAccountMapper.xml b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/ProxyAccountMapper.xml new file mode 100644 index 0000000..d5fd1c0 --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/ProxyAccountMapper.xml @@ -0,0 +1,13 @@ + + + + + + insert into tb_proxy_account + (nid, proxy_id) + values + + (#{item.nid}, #{item.proxyId}) + + + \ No newline at end of file diff --git a/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/ProxyMapper.xml b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/ProxyMapper.xml new file mode 100644 index 0000000..a33c55c --- /dev/null +++ b/cdp-tools/cdp-tool-xhs/cdp-tool-xhs-service/src/main/resources/mapper/ProxyMapper.xml @@ -0,0 +1,28 @@ + + + + + + + + insert into tb_proxy + (ip, port, account, password, address, agreement_type, surplus_day, is_domestic, agent, status, create_time) + values + + (#{item.ip}, #{item.port}, #{item.account}, #{item.password}, #{item.address}, + #{item.agreementType}, #{item.surplusDay}, #{item.isDomestic}, #{item.agent}, #{item.status}, #{item.createTime}) + + + + + update tb_proxy set surplus_day = surplus_day - 1 where surplus_day > 0 + + + + update tb_proxy set status = 1 where status = 0 AND surplus_day = 0 + + \ No newline at end of file