Skip to content

O
operate-customer-service
提交 7df1b8a2 作者: 沈振路
操作

刷公众号用户标签的测试类接口改为多线程

上级 f430e24b
正在显示 包含 79 行增加 和 27 行删除
src/main/java/com/yaoyaozw/customer/controller/TestController.java
......@@ -51,6 +51,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
......@@ -355,6 +356,11 @@ public class TestController {
private static final String TAG_NAME_ALL_USER = "全量用户";
private static final String TAG_NAME_MINI_PROGRAM_USER = "小程序用户";
private static final int BATCH_TAGGING_SIZE = 50;
private static final int ACCOUNT_BATCH_SIZE = 100;
private static final ThreadPoolExecutor THREAD_POOL = new ThreadPoolExecutor(
3, 3, 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
new ThreadPoolExecutor.CallerRunsPolicy());
@Resource
private WeChatTagCleanMapper weChatTagCleanMapper;
......@@ -365,18 +371,14 @@ public class TestController {
public void refreshFullUserTag() {
log.info("开始为公众号用户打全量用户标签...");
int startIdx = 0;
Object val = redisTemplate.opsForValue().get("operate:customer:refreshWechatTag:startIdx");
if (val != null) {
startIdx = Integer.parseInt(String.valueOf(val));
}
List<PendingAuthAccountVO> accounts = weChatTagCleanMapper.getPendingAuthAccounts();
accounts = accounts.subList(Math.min(startIdx, accounts.size()), accounts.size());
log.info("待处理公众号数量: {}", accounts.size());
Map<Long, Integer> fullUserWechatTagIdByAuthorizer = Collections.emptyMap();
Map<Long, Integer> miniProgramUserWechatTagIdByAuthorizer = Collections.emptyMap();
if (accounts.isEmpty()) {
log.info("没有待处理的公众号");
return;
}
List<Long> authorizerIdsToQuery = accounts.stream()
.map(PendingAuthAccountVO::getAuthId)
......@@ -384,7 +386,9 @@ public class TestController {
.distinct()
.collect(Collectors.toList());
if (!authorizerIdsToQuery.isEmpty()) {
if (authorizerIdsToQuery.isEmpty()) {
return;
}
List<AuthorizerFullUserTagVO> dbTagRows =
weChatTagCleanMapper.listFullUserWechatTagByAuthorizers(
authorizerIdsToQuery,
......@@ -401,9 +405,9 @@ public class TestController {
AuthorizerFullUserTagVO::getWechatTagId,
(left, right) -> left)));
fullUserWechatTagIdByAuthorizer =
Map<Long, Integer> fullUserWechatTagIdByAuthorizer =
tagMapByTagName.getOrDefault(TAG_NAME_ALL_USER, Collections.emptyMap());
miniProgramUserWechatTagIdByAuthorizer =
Map<Long, Integer> miniProgramUserWechatTagIdByAuthorizer =
tagMapByTagName.getOrDefault(TAG_NAME_MINI_PROGRAM_USER, Collections.emptyMap());
log.info("从库中载入标签映射: 「{}」{} 条, 「{}」{} 条 (授权方候选数: {})",
......@@ -412,32 +416,56 @@ public class TestController {
TAG_NAME_MINI_PROGRAM_USER,
miniProgramUserWechatTagIdByAuthorizer.size(),
authorizerIdsToQuery.size());
List<List<PendingAuthAccountVO>> accountBatches = Lists.partition(accounts, ACCOUNT_BATCH_SIZE);
int batchIdx = 0;
for (List<PendingAuthAccountVO> batch : accountBatches) {
batchIdx++;
log.info("开始处理第 {} / {} 组, 本组公众号数量: {}", batchIdx, accountBatches.size(), batch.size());
List<String> batchAppids = batch.stream()
.map(PendingAuthAccountVO::getAppid)
.filter(Objects::nonNull)
.collect(Collectors.toList());
Map<String, String> tokenMap;
if (!batchAppids.isEmpty()) {
List<PendingAuthAccountVO> tokenResults = weChatTagCleanMapper.getTokensByAppids(batchAppids);
tokenMap = tokenResults.stream()
.filter(t -> t.getAppid() != null && t.getAuthorizerAccessToken() != null)
.collect(Collectors.toMap(
PendingAuthAccountVO::getAppid,
PendingAuthAccountVO::getAuthorizerAccessToken,
(a, b) -> a));
} else {
tokenMap = Collections.emptyMap();
}
log.info("第 {} 组实时查询 token 结果: {} 条", batchIdx, tokenMap.size());
int idx = 1;
for (PendingAuthAccountVO account : accounts) {
List<Future<?>> futures = new ArrayList<>();
for (PendingAuthAccountVO account : batch) {
futures.add(THREAD_POOL.submit(() -> {
String appid = account.getAppid();
String nickName = account.getNickName();
String accessToken = account.getAuthorizerAccessToken();
String accessToken = tokenMap.get(appid);
Long authId = account.getAuthId();
if (StringUtils.isAnyBlank(appid, accessToken)) {
log.warn("公众号信息缺失, 跳过: appid={}, nickName={}", appid, nickName);
continue;
return;
}
if (authId == null) {
log.warn("公众号缺少 authId(authorizer_id), 跳过: appid={}, nickName={}", appid, nickName);
continue;
return;
}
log.info("刷新公众号用户标签进度：{} / {}，当前公众号：{}", idx++, accounts.size(), account.getNickName());
try {
Integer fullUserTagId = fullUserWechatTagIdByAuthorizer.get(authId);
if (fullUserTagId == null) {
log.warn("公众号 {}({}) 在库中未找到 [{}] 标签记录, 跳过", nickName, appid,
TAG_NAME_ALL_USER);
continue;
return;
}
Integer miniProgramUserTagId = miniProgramUserWechatTagIdByAuthorizer.get(authId);
......@@ -457,7 +485,7 @@ public class TestController {
.collect(Collectors.toList());
if (needTagOpenidList.isEmpty()) {
continue;
return;
}
List<List<String>> partitions = Lists.partition(needTagOpenidList, BATCH_TAGGING_SIZE);
......@@ -480,10 +508,22 @@ public class TestController {
log.info("公众号 appid={}, 名称={}, 此次更新标签的用户数={}, 目标标签id={}",
appid, nickName, updatedCount, fullUserTagId);
TimeUnit.MILLISECONDS.sleep(500);
} catch (Exception e) {
log.error("处理公众号 {}({}) 时发生错误: {}", nickName, appid, e.getMessage(), e);
}
}));
}
for (Future<?> future : futures) {
try {
future.get();
} catch (Exception e) {
log.error("等待线程执行完成时发生错误: {}", e.getMessage(), e);
}
}
log.info("第 {} / {} 组处理完成", batchIdx, accountBatches.size());
}
log.info("公众号用户全量用户标签清洗完成");
......
src/main/java/com/yaoyaozw/customer/mapper/WeChatTagCleanMapper.java
......@@ -32,5 +32,9 @@ public interface WeChatTagCleanMapper {
@Param("authorizerIds") List<Long> authorizerIds,
@Param("tagNames") List<String> tagNames);
/**
* 根据 appid 列表实时查询 access_token
*/
List<PendingAuthAccountVO> getTokensByAppids(@Param("appids") List<String> appids);
}
src/main/resources/mapper/WeChatTagCleanMapper.xml
......@@ -6,15 +6,23 @@
SELECT
ai.id AS authId,
ai.appid AS appid,
ai.nick_name AS nickName,
atk.authorizer_access_token AS authorizerAccessToken
ai.nick_name AS nickName
FROM authorizer_info ai
LEFT JOIN authorizer_token atk ON atk.authorizer_appid = ai.appid
WHERE ai.store_type IN (
'YANG_GUANG','TOMATO','YUE_WEN_1'
)
AND atk.create_time >= DATE_ADD(CURRENT_DATE,INTERVAL -1 DAY)
</select>
<select id="getTokensByAppids" resultType="com.yaoyaozw.customer.vo.wechat.PendingAuthAccountVO">
SELECT
ai.appid AS appid,
atk.authorizer_access_token AS authorizerAccessToken
FROM authorizer_info ai
LEFT JOIN authorizer_token atk ON atk.authorizer_appid = ai.appid
WHERE ai.appid IN
<foreach collection="appids" item="appid" open="(" separator="," close=")">
#{appid}
</foreach>
</select>
<select id="listFullUserWechatTagByAuthorizers"
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论