大纲
1.营销系统引入MQ实现异步化来进行性能优化
2.基于MQ释放优惠券提升系统扩展性
3.基于Redis实现重复促销活动去重
4.基于促销活动创建事件实现异步化
5.推送任务分片和分片消息batch合并发送实现
6.推送系统与用户群体查询逻辑解耦
7.查询用户数据以及批量发送推送消息
8.线程池封装以及推送系统多线程推送
9.推送系统的千万级消息多线程推送
10.千万级用户惰性发券代码实现
11.指定用户群体发券的代码实现
12.分片消息的batch合并算法重构实现
13.百万画像群体爆款商品推送代码实现
14.生产环境百万级用户PUSH全链路压测
接下来优化营销系统的四大促销场景的代码：全量用户推送促销活动、全量用户发放优惠券、特定用户推送领取优惠券消息、热门商品定时推送
1.营销系统引入MQ实现异步化来进行性能优化
查询全量用户、创建大量消息、发送大量消息到MQ，这三个操作都可能非常耗时。所以这三个操作都不应该在创建营销活动时同步处理，而最好利用MQ在消费时异步化处理。
促销活动创建接口举例：
初版：写库(10毫秒) + 全量用户拉取(几分钟) + 大量消息创建和发送到MQ(几分钟)，此时接口性能低下以及耗时。
优化：使用MQ实现异步化处理后，写库(10毫秒) + 发送一条消息给MQ(10毫秒)。
2.基于MQ释放优惠券提升系统扩展性
MQ主要有三大作用：削峰填谷、异步化提升性能、解耦提升扩展性。在初版主要用了MQ实现削峰填谷，解决面临的瞬时高并发写库或者调用接口的问题。而前面继续用MQ来剥离创建活动的接口时遇到的耗时问题，即通过实现异步化来提升性能。
下面介绍使用MQ提升系统扩展性：
例如创建订单时，订单里使用了一个优惠券。此时订单系统需要通知库存系统，对商品库存进行锁定。同时还需要通知营销系统，对该优惠券进行锁定，把其is_used字段进行设置。但后来用户发起取消订单操作，修改了订单状态，释放了库存和优惠券。库存恢复到原来的数量，优惠券也可以继续使用。那么在取消订单时，是否应该直接调用库存系统和营销系统的接口去释放库存和优惠券呢？
一般在取消订单时，会通过引入MQ，把一个订单取消事件消息OrderCanceledEvent发送到MQ。然后让库存系统、营销系统、积分系统等，关注这个订单取消事件消息，各自进行订单取消后的处理。以此实现订单系统对库存系统、营销系统、积分系统等系统的解耦，提升订单系统的扩展性。
3.基于Redis实现重复促销活动去重
(1)配置Redis
(2)创建促销活动时使用Redis进行去重
(1)配置Redis
@Data @Configuration @ConditionalOnClass(RedisConnectionFactory.class) public class RedisConfig { @Value("${spring.redis.host}") private String host; @Value("${spring.redis.port}") private String port; @Value("${spring.redis.password}") private String password; @Value("${spring.redis.timeout}") private int timeout; @Bean @ConditionalOnClass(RedisConnectionFactory.class) public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory) { RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>(); redisTemplate.setConnectionFactory(redisConnectionFactory); redisTemplate.setDefaultSerializer(new StringRedisSerializer()); redisTemplate.afterPropertiesSet(); return redisTemplate; } @Bean @ConditionalOnClass(RedissonClient.class) public RedissonClient redissonClient() { Config config = new Config(); config.useSingleServer() .setAddress("redis://" + host + ":" + port) .setPassword(password) .setConnectionMinimumIdleSize(10) .setConnectionPoolSize(100) .setIdleConnectionTimeout(600000) .setSubscriptionConnectionMinimumIdleSize(10) .setSubscriptionConnectionPoolSize(100) .setTimeout(timeout); config.setCodec(new StringCodec()); config.setThreads(5); config.setNettyThreads(5); RedissonClient client = Redisson.create(config); return client; } @Bean @ConditionalOnClass(RedisConnectionFactory.class) public RedisCache redisCache(RedisTemplate redisTemplate) { return new RedisCache(redisTemplate); } } public class RedisCache { private RedisTemplate redisTemplate; public RedisCache(RedisTemplate redisTemplate) { this.redisTemplate = redisTemplate; } //缓存存储 public void set(String key, String value, int seconds) { ValueOperations<String, String> vo = redisTemplate.opsForValue(); if (seconds > 0) { vo.set(key, value, seconds, TimeUnit.SECONDS); } else { vo.set(key, value); } } //缓存获取 public String get(String key) { ValueOperations<String, String> vo = redisTemplate.opsForValue(); return vo.get(key); } //缓存手动失效 public boolean delete(String key) { return redisTemplate.delete(key); } //判断hash key是否存在 public boolean hExists(String key) { return hGetAll(key).isEmpty(); } //获取hash变量中的键值对，对应redis hgetall 命令 public Map<String, String> hGetAll(String key) { return redisTemplate.opsForHash().entries(key); } //以map集合的形式添加hash键值对 public void hPutAll(String key, Map<String, String> map) { redisTemplate.opsForHash().putAll(key, map); } //执行lua脚本 public <T> T execute(RedisScript<T> script, List<String> keys, String... args) { return (T) redisTemplate.execute(script, keys, args); } public RedisTemplate getRedisTemplate() { return redisTemplate; } }(2)创建促销活动时使用Redis进行去重
@Service public class PromotionServiceImpl implements PromotionService { //开启促销活动DAO @Autowired private SalesPromotionDAO salesPromotionDAO; //redis缓存工具 @Resource private RedisCache redisCache; @Resource private PromotionConverter promotionConverter; //新增或修改一个促销活动 @Transactional(rollbackFor = Exception.class) @Override public SaveOrUpdatePromotionDTO saveOrUpdatePromotion(SaveOrUpdatePromotionRequest request) { //判断是否活动是否重复 String result = redisCache.get(PROMOTION_CONCURRENCY_KEY + request.getName() + request.getCreateUser() + request.getStartTime().getTime() + request.getEndTime().getTime()); if (StringUtils.isNotBlank(result)) { return null; } log.info("活动内容：{}", request); //活动规则 String rule = JsonUtil.object2Json(request.getRule()); //构造促销活动实体 SalesPromotionDO salesPromotionDO = promotionConverter.convertPromotionDO(request); salesPromotionDO.setRule(rule); //促销活动落库 salesPromotionDAO.saveOrUpdatePromotion(salesPromotionDO); //写Redis缓存用于下次创建去重 redisCache.set(PROMOTION_CONCURRENCY_KEY + request.getName() + request.getCreateUser() + request.getStartTime().getTime() + request.getEndTime().getTime(), UUID.randomUUID().toString(),30 * 60); //为所有用户推送促销活动，发MQ sendPlatformPromotionMessage(salesPromotionDO); //构造响应数据 SaveOrUpdatePromotionDTO dto = new SaveOrUpdatePromotionDTO(); dto.setName(request.getName()); dto.setType(request.getType()); dto.setRule(rule); dto.setCreateUser(request.getCreateUser()); dto.setSuccess(true); return dto; } ... }
4.基于促销活动创建事件实现异步化
(1)创建促销活动时发布创建活动事件消息到MQ
(2)营销系统需要消费创建活动事件消息
(1)创建促销活动时发布创建活动事件消息到MQ
//促销活动创建事件 @Data public class SalesPromotionCreatedEvent { private SalesPromotionDO salesPromotion; } @Service public class PromotionServiceImpl implements PromotionService { ... //新增或修改一个运营活动 @Transactional(rollbackFor = Exception.class) @Override public SaveOrUpdatePromotionDTO saveOrUpdatePromotion(SaveOrUpdatePromotionRequest request) { //判断是否活动是否重复 String result = redisCache.get(PROMOTION_CONCURRENCY_KEY + request.getName() + request.getCreateUser() + request.getStartTime().getTime() + request.getEndTime().getTime()); if (StringUtils.isNotBlank(result)) { return null; } log.info("活动内容：{}", request); //活动规则 String rule = JsonUtil.object2Json(request.getRule()); //构造促销活动实体 SalesPromotionDO salesPromotionDO = promotionConverter.convertPromotionDO(request); salesPromotionDO.setRule(rule); //促销活动落库 salesPromotionDAO.saveOrUpdatePromotion(salesPromotionDO); redisCache.set(PROMOTION_CONCURRENCY_KEY + request.getName() + request.getCreateUser() + request.getStartTime().getTime() + request.getEndTime().getTime(), UUID.randomUUID().toString(),30 * 60); //通过MQ为所有用户推送促销活动 //sendPlatformPromotionMessage(salesPromotionDO); //发布促销活动创建事件到MQ publishSalesPromotionCreatedEvent(salesPromotionDO); //构造响应数据 SaveOrUpdatePromotionDTO dto = new SaveOrUpdatePromotionDTO(); dto.setName(request.getName()); dto.setType(request.getType()); dto.setRule(rule); dto.setCreateUser(request.getCreateUser()); dto.setSuccess(true); return dto; } //发布促销活动创建事件 private void publishSalesPromotionCreatedEvent(SalesPromotionDO salesPromotion) { SalesPromotionCreatedEvent salesPromotionCreatedEvent = new SalesPromotionCreatedEvent(); salesPromotionCreatedEvent.setSalesPromotion(salesPromotion); String salesPromotionCreatedEventJSON = JsonUtil.object2Json(salesPromotionCreatedEvent); defaultProducer.sendMessage(RocketMqConstant.SALES_PROMOTION_CREATED_EVENT_TOPIC, salesPromotionCreatedEventJSON, "发布促销活动创建事件"); } ... }(2)营销系统需要消费创建活动事件消息
@Configuration public class ConsumerBeanConfig { ... @Bean("salesPromotionCreatedEventListener") public DefaultMQPushConsumer salesPromotionCreatedEventListener(SalesPromotionCreatedEventListener salesPromotionCreatedEventListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(SALES_PROMOTION_CREATED_EVENT_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(SALES_PROMOTION_CREATED_EVENT_TOPIC, "*"); consumer.registerMessageListener(salesPromotionCreatedEventListener); consumer.start(); return consumer; } ... } //促销活动创建事件监听器 @Component public class SalesPromotionCreatedEventListener implements MessageListenerConcurrently { ... }
5.推送任务分片和分片消息batch合并发送实现
营销系统在消费创建活动事件消息时，会进行千万级用户的推送任务分片和分片消息batch合并发送到MQ。
//发送到MQ的Topic是"PLATFORM_PROMOTION_SEND_USER_BUCKET_TOPIC"。 //促销活动创建事件监听器 @Component public class SalesPromotionCreatedEventListener implements MessageListenerConcurrently { @DubboReference(version = "1.0.0") private AccountApi accountApi; @Resource private DefaultProducer defaultProducer; @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext consumeConcurrentlyContext) { try { for (MessageExt messageExt : list) { //下面三行代码可以获取到一个刚刚创建成功的促销活动 String message = new String(messageExt.getBody()); SalesPromotionCreatedEvent salesPromotionCreatedEvent = JSON.parseObject(message, SalesPromotionCreatedEvent.class); SalesPromotionDO salesPromotion = salesPromotionCreatedEvent.getSalesPromotion(); //这个促销活动会针对全体用户发起Push //userBucketSize就是一个用户分片的大小，与一个startUserId ～ endUserId用户ID范围相对应 final int userBucketSize = 1000; //messageBatchSize就是合并多个任务消息成一个batch消息的大小，RocketMQ的每个batch消息包含了100个推送任务消息 //所以1w个推送任务消息会合并为100个batch消息 //发送1万个推送任务消息到MQ，只需要进行100次网络通信给RocketMQ即可，这样可以大幅降低发送消息的耗时 final int messageBatchSize = 100; //1.获取全体用户数量有两种做法： //第一种是进行count(效率不高)，第二种是获取max(userId)，通常会使用第二种做法 //select * from account order by id desc limit 1，类似于这样的sql语句去获取用户表里主键值最大的一个 JsonResult<Long> queryMaxUserIdResult = accountApi.queryMaxUserId(); if (!queryMaxUserIdResult.getSuccess()) { throw new BaseBizException(queryMaxUserIdResult.getErrorCode(), queryMaxUserIdResult.getErrorMessage()); } Long maxUserId = queryMaxUserIdResult.getData(); //2.获取到全体用户数量后，就可以根据一定算法结合自增ID，对千万级用户的推送任务进行分片，比如一个推送任务包含1000个用户或2000个用户 //userBuckets就是用户分片集合，其中有上万条key-value对，每个key-value对就是一个startUserId -> endUserId，代表一个推送任务分片 Map<Long, Long> userBuckets = new LinkedHashMap<>(); AtomicBoolean doSharding = new AtomicBoolean(true);// 当前是否需要执行分片 long startUserId = 1L;//起始用户ID，数据库自增主键是从1开始的 while (doSharding.get()) { if (startUserId > maxUserId) { doSharding.compareAndSet(true, false); break; } userBuckets.put(startUserId, startUserId + userBucketSize); startUserId += userBucketSize; } //3.完成分片后，就把可能成千上万的推送任务进行RocketMQ消息的batch合并 //通过batch模式一批一批的发送任务到MQ里去，从而减少和RocketMQ网络通信的耗时 int handledBucketCount = 0; List<String> promotionPushTaskBatch = new ArrayList<>(messageBatchSize); for (Map.Entry<Long, Long> userBucket : userBuckets.entrySet()) { handledBucketCount++; PlatformPromotionUserBucketMessage promotionPushTask = PlatformPromotionUserBucketMessage.builder() .startUserId(userBucket.getKey()) .endUserId(userBucket.getValue()) .promotionId(salesPromotion.getId()) .promotionType(salesPromotion.getType()) .mainMessage(salesPromotion.getName()) .message("您已获得活动资格，可以打开APP进入活动页面") .informType(salesPromotion.getInformType()) .build(); String promotionPushTaskJSON = JsonUtil.object2Json(promotionPushTask); promotionPushTaskBatch.add(promotionPushTaskJSON); //batch合并发送后，对promotionPushTaskBatch进行清空 if (promotionPushTaskBatch.size() == messageBatchSize || handledBucketCount == userBuckets.size()) { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_PROMOTION_SEND_USER_BUCKET_TOPIC, promotionPushTaskBatch, "平台发放促销活动用户桶消息"); promotionPushTaskBatch.clear(); } } } } catch(Exception e) { log.error("consume error, 促销活动创建事件处理异常", e); return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }
6.推送系统与用户群体查询逻辑解耦
营销系统将推送分片消息batch合并发送到MQ后，这些消息会由营销系统进行消费，而不是由推送系统进行消费。如果推送系统直接消费这些分片任务消息，那么推送系统需要自己去根据用户ID范围去查询用户群体信息。这样推送系统就会耦合会员系统(根据用户ID查)或大数据系统(根据用户画像查)，耦合具体的用户群体的查询逻辑。
所以，还是由营销系统来决定具体的用户群体查询逻辑，从而实现推送系统与会员系统或大数据系统的解耦。此时，营销系统会封装好每个用户的具体推送消息，再通过合并batch消息发送到MQ中，最后由推送系统消费。
因此，营销系统负责消费推送任务分片的消息。
7.查询用户数据以及批量发送推送消息
营销系统获取一个推送任务分片后，自己决定如何查询用户群体。营销系统会为查出来的每个用户进行推送消息封装，然后再将这些推送消息以batch模式发送到MQ由推送系统消费处理。
步骤1：获取到一个推送任务分片
步骤2：查询本次推送任务分片对应的用户群体
步骤3：为每个用户创建一条符合推送系统规定格式的用户推送消息，然后把每个用户的推送消息发送到MQ里，
步骤3的第一种实现(不推荐)：
用线程池并发地把一个任务分片里的1000条消息并发发送到MQ里，这种实现的问题是如果一个分片任务有1000个用户，那么此时虽然是多线程并发，但还是要发送1000次请求到MQ。
步骤3的第二种实现(推荐)：
用线程池并发地批量发送消息到MQ里。RocketMQ官网建议批量发送消息的一个batch不能超过1MB，在RocketMQ源码中实际上批量消息不能超过4MB。所以批量发送时，需要考虑发送消息的大小，然后根据网络压力和IO压力选择每批次发送多少条消息。
此处按照100条一批发送，1000条用户推送消息，会合并为10个batch进行发送。因此只要发起10次网络请求即可，每个推送任务分片的处理到写MQ的整个过程，速度是非常快的。一台营销系统单线程处理1万个推送分片任务，每个任务要写10次MQ，每次10ms，总共需要1000s=20分钟。多台营销系统，对每个分片任务的10个batch发送都可以看成是线程池并发处理的。假设2台机器，每台机器开50个线程，那么总共需要1000s / (2*50) = 10s，就可以把1万个分片任务处理完毕。
//下面是营销系统监听"PLATFORM_PROMOTION_SEND_USER_BUCKET_TOPIC"消费推送任务分片消息的实现。 //消费完成后会将消息发送到MQ的"PLATFORM_PROMOTION_SEND_TOPIC"： @Configuration public class ConsumerBeanConfig { ... //平台发放促销活动用户桶消费者 @Bean("platformPromotionUserBucketReceiveTopicConsumer") public DefaultMQPushConsumer receiveCouponUserBucketConsumer(PlatFormPromotionUserBucketListener platFormPromotionUserBucketListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_PROMOTION_SEND_USER_BUCKET_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_PROMOTION_SEND_USER_BUCKET_TOPIC, "*"); consumer.registerMessageListener(platFormPromotionUserBucketListener); consumer.start(); return consumer; } } @Component public class PlatFormPromotionUserBucketListener implements MessageListenerConcurrently { //会员服务 @DubboReference(version = "1.0.0") private AccountApi accountApi; //发送消息共用的线程池 @Autowired @Qualifier("sharedSendMsgThreadPool") private SafeThreadPool sharedSendMsgThreadPool; //RocketMQ生产者 @Autowired private DefaultProducer defaultProducer; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { for (MessageExt messageExt : msgList) { //1.获取到一个推送任务分片 String message = new String(messageExt.getBody()); log.debug("执行平台发送促销活动用户桶消息逻辑，消息内容：{}", message); PlatformPromotionUserBucketMessage promotionPushTask = JSON.parseObject(message, PlatformPromotionUserBucketMessage.class); //2.查询本次推送任务分片对应的用户群体 Long startUserId = promotionPushTask.getStartUserId(); Long endUserId = promotionPushTask.getEndUserId(); JsonResult<List<MembershipAccountDTO>> queryResult = accountApi.queryAccountByIdRange(startUserId, endUserId); if (!queryResult.getSuccess()) { throw new BaseBizException(queryResult.getErrorCode(), queryResult.getErrorMessage()); } List<MembershipAccountDTO> membershipAccounts = queryResult.getData(); if (CollectionUtils.isEmpty(membershipAccounts)) { log.info("根据用户桶内的id范围没有查询到用户, startUserId={}, endUserId{}", startUserId, endUserId); continue; } //3.为每个用户创建一条符合推送系统规定格式的用户推送消息，然后把每个用户的推送消息发送到MQ里； //第一种实现(不推荐)： //用线程池并发地把一个任务分片里的1000条消息并发发送到MQ里去； //这种实现的问题是如果一个分片任务有1000个用户，那么此时虽然是多线程并发，但还是要发送请求1000次到MQ； //第二种实现(推荐)： //用线程池并发地批量发送消息到MQ里； //RocketMQ官网对批量发送消息的说明是，一个batch不能超过1MB，在RocketMQ源码中实际上批量消息不能超过4MB； //所以批量发送的时候，需要综合考虑发送消息的大小，然后根据网络压力和IO压力综合对比评估后选择每批次发送多少条； //此处按照100条一批发送，1000条用户推送消息，会合并为10个batch进行发送； //因此只要发起10次网络请求即可，每个任务分片的处理到写MQ的整个过程，速度是非常快的； //一台营销系统单线程不停处理1万个分片任务，每个任务要写10次MQ，10万次，每次10ms，总共需要1000000ms=1000s=20分钟左右 //多台营销系统，对每个分片任务的10个batch都是线程池并发写的 //假设2台机器，每台机器开50个线程，那么总共需要：1000s / 50 = 20s，就可以把1万个分片任务在这里处理完毕； PlatformPromotionMessage promotionMessage = PlatformPromotionMessage.builder() .promotionId(promotionPushTask.getPromotionId()) .promotionType(promotionPushTask.getPromotionType()) .mainMessage(promotionPushTask.getMainMessage()) .message("您已获得活动资格，打开APP进入活动页面") .informType(promotionPushTask.getInformType()) .build(); List<String> batch = new ArrayList<>(100); for (MembershipAccountDTO account : membershipAccounts) { promotionMessage.setUserAccountId(account.getId()); batch.add(JSON.toJSONString(promotionMessage)); if (batch.size() == 100) { sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_PROMOTION_SEND_TOPIC, batch, "平台发送促销活动消息"); }); batch.clear(); } } //最后剩下的也批量发出 if (!CollectionUtils.isEmpty(batch)) { sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_PROMOTION_SEND_TOPIC, batch, "平台发送促销活动消息"); }); batch.clear(); } } } catch (Exception e) { log.error("consume error,促销活动消息消费失败", e); //本次消费失败，下次重新消费 return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }
8.线程池封装以及推送系统多线程推送
(1)通过@Configuration实例化一个线程池
(2)具体的线程池实例化
(1)通过@Configuration实例化一个线程池
@Configuration public class ThreadPoolConfig { //发送消息共用的线程池 //线程池名字、线程名字：sharedThreadPool //最多允许多少线程同时执行任务：100 @Bean("sharedSendMsgThreadPool") public SafeThreadPool sharedSendMsgThreadPool() { return new SafeThreadPool("sharedSendMsgThreadPool", 100); } }(2)具体的线程池实例化
发送消息的线程池的corePoolSize设置为0，可以在空闲时把线程都回收掉。
public class SafeThreadPool { private final Semaphore semaphore; private final ThreadPoolExecutor threadPoolExecutor; public SafeThreadPool(String name, int permits) { //如果超过了100个任务同时要运行，会通过semaphore信号量进行阻塞 semaphore = new Semaphore(permits); //为什么要设置corePoolSize是0？ //因为消息推送并不是一直要推送的，只有促销活动比如发优惠券时才需要进行消息推送，正常情况下是不会进行消息推送的 //所以发送消息的线程池的corePoolSize设置为0，可以在空闲时把线程都回收掉 threadPoolExecutor = new ThreadPoolExecutor( 0, permits * 2, 60, TimeUnit.SECONDS, new SynchronousQueue<>(), NamedDaemonThreadFactory.getInstance(name) ); } public void execute(Runnable task) { //超过了100个batch要并发推送，就会在这里阻塞住 //比如100个线程都在繁忙时，就不可能有超过100个batch要同时提交过来 //极端情况下，最多也就是100个batch可以拿到信号量，100 * 2的max容量 semaphore.acquireUninterruptibly(); threadPoolExecutor.submit(() -> { try { task.run(); } finally { semaphore.release(); } }); } }
9.推送系统的千万级消息多线程推送
根据前面可知：对一个千万级消息的推送任务，营销系统首先会对这个千万级消息的推送任务进行分片，分片成1万个推送任务然后batch发送到MQ。
营销系统会获取这个千万级消息的推送任务的推送任务分片，然后自己决定如何查询用户群体。
营销系统会为查出来的每个用户进行推送消息封装，然后再将这些推送消息以batch模式发送到MQ由推送系统消费处理。
所以营销系统会封装好千万级的推送消息，然后合并成10万个batch推送消息去发送到MQ。如果每个batch推送消息发送到MQ需要50ms，那么总共需要500万ms，即5000s。使用2台营销系统共200个线程并发去将这10万个batch推送消息发送到MQ，那么总共需要5000s / 200 = 25s，就可以把千万级推送消息发到MQ。
假设有5台4核8G的机器部署了推送系统，那么每个推送系统便会消费到200万条推送消息，接着使用多线程并发推送。由于部署5台机器，每台机器会拿到200w条消息，消费时会一批一批拿，放入msgList。如果每条消息调用第三方平台SDK发起推送耗时100ms~200ms，那么总共需要200w*200ms=40万s=几十个小时。这时必须用线程池采用多线程的方式并发去推送。
每台机器最多开启60个线程，那么5台机器总共300个线程。由于一次推送200ms，每个线程每秒钟可以推成5次，300个线程每秒1500次。那么经过6000s，5台机器300个线程就可以推1000万次，6000s / 60 = 100分钟，1个多小时。所以千万级用户全量推送，快则几十分钟，慢则两三个小时。
监听PLATFORM_PROMOTION_SEND_TOPIC的推送系统，对这200万条推送消息进行消费然后发起推送的代码如下：
@Configuration public class ConsumerBeanConfig { ... //平台活动推送消息消费者 completableFuture逻辑 @Bean("platformPromotionSendTopicConsumer") public DefaultMQPushConsumer platformPromotionSendConsumer(PlatFormPromotionListener platFormPromotionListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_PROMOTION_SEND_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_PROMOTION_SEND_TOPIC, "*"); consumer.registerMessageListener(platFormPromotionListener); consumer.start(); return consumer; } } @Component public class PlatFormPromotionListener implements MessageListenerConcurrently { //消息推送工厂提供者 @Autowired private FactoryProducer factoryProducer; @Autowired private RedisTemplate<String, String> redisTemplate; private static final int PERMITS = 30; private static final AtomicBoolean initializedRef = new AtomicBoolean(false); private static ThreadPoolExecutor THREAD_POOL_EXECUTOR = null; private static final Supplier<ThreadPoolExecutor> THREAD_POOL_EXECUTOR_SUPPLIER = () -> { if (initializedRef.compareAndSet(false, true)) { //corePoolSize是30个，maxPoolSize是60个 THREAD_POOL_EXECUTOR = new ThreadPoolExecutor(PERMITS, PERMITS * 2, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(1000), NamedDaemonThreadFactory.getInstance("consumePromotionMsg"), new ThreadPoolExecutor.CallerRunsPolicy()); } return THREAD_POOL_EXECUTOR; }; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { //使用自定义的业务线程池 List<CompletableFuture<AltResult>> futureList = msgList.stream() .map(e -> CompletableFuture.supplyAsync(() -> handleMessageExt(e), THREAD_POOL_EXECUTOR_SUPPLIER.get())) .collect(Collectors.toList()); List<Throwable> resultList = futureList.stream() .map(CompletableFuture::join) .filter(e -> e.ex != null) .map(e -> e.ex).collect(Collectors.toList()); if (!resultList.isEmpty()) { throw resultList.get(0); } } catch (Throwable e) { log.error("consume error,平台优惠券消费失败", e); //本次消费失败，下次重新消费 return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } private AltResult handleMessageExt(MessageExt messageExt) { try { log.debug("执行平台发送通知消息逻辑，消息内容：{}", messageExt.getBody()); String msg = new String(messageExt.getBody()); PlatformPromotionMessage message = JSON.parseObject(msg , PlatformPromotionMessage.class); //幂等控制 if (StringUtils.isNotBlank(redisTemplate.opsForValue().get(message.cacheKey()))) { return new AltResult(null); } //获取消息服务工厂 MessageSendServiceFactory messageSendServiceFactory = factoryProducer.getMessageSendServiceFactory(message.getInformType()); //消息发送服务组件 MessageSendService messageSendService = messageSendServiceFactory.createMessageSendService(); //构造消息 PlatformMessagePushMessage messagePushMessage = PlatformMessagePushMessage.builder() .informType(message.getInformType()) .mainMessage(message.getMainMessage()) .userAccountId(message.getUserAccountId()) .message(message.getMessage()) .build(); MessageSendDTO messageSendDTO = messageSendServiceFactory.createMessageSendDTO(messagePushMessage); messageSendService.send(messageSendDTO); //发送成功之后把已经发送成功记录到redis redisTemplate.opsForValue().set(message.cacheKey(), UUID.randomUUID().toString()); log.info("消息推送完成，messageSendDTO:{}", messageSendDTO); Thread.sleep(20); return new AltResult(null); } catch (Exception e) { return new AltResult(e); } } //completableFuture的返回结果，适用于无返回值的情况 //ex字段为null表示任务执行成功 //ex字段不为null表示任务执行失败，并把异常设置为ex字段 private static class AltResult { final Throwable ex; public AltResult(Throwable ex) { this.ex = ex; } } }
10.千万级用户惰性发券代码实现
(1)给全量用户发放优惠券的初版实现
(2)给全量用户惰性发放优惠券的优化实现
(1)给全量用户发放优惠券的初版实现
首先营销系统对全量用户发放优惠券的任务进行分片，然后将分片的消息发送到如下Topic。
PLATFORM_COUPON_SEND_USER_BUCKET_TOPIC@RestController @RequestMapping("/demo/promotion/coupon") public class PromotionCouponController { //优惠活动service @Autowired private CouponService couponService; //新增一个优惠券活动 @PostMapping public JsonResult<SaveOrUpdateCouponDTO> saveOrUpdateCoupon(@RequestBody SaveOrUpdateCouponRequest request) { try { log.info("新增一条优惠券:{}", JSON.toJSONString(request)); SaveOrUpdateCouponDTO dto = couponService.saveOrUpdateCoupon(request); return JsonResult.buildSuccess(dto); } catch (BaseBizException e) { log.error("biz error: request={}", JSON.toJSONString(request), e); return JsonResult.buildError(e.getErrorCode(), e.getErrorMsg()); } catch (Exception e) { log.error("system error: request={}", JSON.toJSONString(request), e); return JsonResult.buildError(e.getMessage()); } } ... } //优惠券接口实现 @Service public class CouponServiceImpl implements CouponService { ... //保存/修改优惠券活动方法 @Transactional(rollbackFor = Exception.class) @Override public SaveOrUpdateCouponDTO saveOrUpdateCoupon(SaveOrUpdateCouponRequest request) { SalesPromotionCouponDO couponDO = couponConverter.convertCouponDO(request); couponDO.setCouponReceivedCount(0); salesPromotionCouponDAO.saveOrUpdateCoupon(couponDO); //为所有用户发放优惠券 sendPlatformCouponMessage(couponDO); SaveOrUpdateCouponDTO dto = new SaveOrUpdateCouponDTO(); dto.setCouponName(request.getCouponName()); dto.setRule(request.getCouponRule()); dto.setSuccess(true); return dto; } ... //为所有用户发放优惠券 private void sendPlatformCouponMessage(SalesPromotionCouponDO promotionCouponDO) { //桶的大小 final int userBucketSize = 1000; final int messageBatchSize = 100; //1.查询出库里面最大的userId，作为用户的总数量 JsonResult<Long> maxUserIdJsonResult = accountApi.queryMaxUserId(); if (maxUserIdJsonResult.getSuccess()) { throw new BaseBizException(maxUserIdJsonResult.getErrorCode(), maxUserIdJsonResult.getErrorMessage()); } Long maxUserId = maxUserIdJsonResult.getData(); //2.分成m个桶，每个桶里面有n个用户，每个桶发送一条"批量发送优惠券用户桶消息" //例：maxUserId = 100w; userBucketSize=1000 //userBucket1 = [1, 1001) //userBucket2 = [1001, 2001) //userBucketCount = 1000 Map<Long, Long> userBuckets = new LinkedHashMap<>(); AtomicBoolean flagRef = new AtomicBoolean(true); long startUserId = 1L; while (flagRef.get()) { if (startUserId > maxUserId) { flagRef.compareAndSet(true, false); } userBuckets.put(startUserId, startUserId + userBucketSize); startUserId += userBucketSize; } //3.批量发送消息 //例：userBucketCount = 1000; messageBatchSize = 100 //批量发送次数 = 10次，经过两次分桶，这里发送消息的次数从100w次降到10次 int handledBucketCount = 0; List<String> jsonMessageBatch = new ArrayList<>(messageBatchSize); for (Map.Entry<Long, Long> userBucket : userBuckets.entrySet()) { handledBucketCount++; PlatformCouponUserBucketMessage message = PlatformCouponUserBucketMessage.builder() .startUserId(userBucket.getKey()) .endUserId(userBucket.getValue()) .informType(promotionCouponDO.getInformType()) .couponId(promotionCouponDO.getId()) .activityStartTime(promotionCouponDO.getActivityStartTime()) .activityEndTime(promotionCouponDO.getActivityEndTime()) .couponType(promotionCouponDO.getCouponType()) .build(); String jsonMessage = JsonUtil.object2Json(message); jsonMessageBatch.add(jsonMessage); if (jsonMessageBatch.size() == messageBatchSize || handledBucketCount == userBuckets.size()) { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_COUPON_SEND_USER_BUCKET_TOPIC, jsonMessageBatch, "平台发放优惠券用户桶消息"); jsonMessageBatch.clear(); } } } }接着营销系统监听如下Topic消费分片后的发放优惠券任务。
PLATFORM_COUPON_SEND_USER_BUCKET_TOPIC此时有两种处理方法：一.直接使用线程池进行发送发放优惠券消息到MQ。二.合并batch后再使用线程池发送MQ。
@Configuration public class ConsumerBeanConfig { ... //平台发放优惠券用户桶消费者 @Bean("platformCouponUserBucketReceiveTopicConsumer") public DefaultMQPushConsumer receiveCouponUserBucketConsumer(@Qualifier("platformCouponUserBucketReceiveTopicConsumer")PlatFormCouponUserBucketListener platFormCouponUserBucketListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_COUPON_SEND_USER_BUCKET_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_COUPON_SEND_USER_BUCKET_TOPIC, "*"); consumer.registerMessageListener(platFormCouponUserBucketListener); consumer.start(); return consumer; } } @Component public class PlatFormCouponUserBucketListener implements MessageListenerConcurrently { //账户服务 @DubboReference(version = "1.0.0") private AccountApi accountApi; //发送消息共用的线程池 @Autowired @Qualifier("sharedSendMsgThreadPool") private SafeThreadPool sharedSendMsgThreadPool; //RocketMQ生产者 @Autowired private DefaultProducer defaultProducer; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { for (MessageExt messageExt : msgList) { //1.反序列化消息 String messageString = new String(messageExt.getBody()); log.debug("执行平台发送优惠券用户桶消息逻辑，消息内容：{}", messageString); PlatformCouponUserBucketMessage message = JSON.parseObject(messageString, PlatformCouponUserBucketMessage.class); //2.查询桶内的用户信息 Long startUserId = message.getStartUserId(); Long endUserId = message.getEndUserId(); JsonResult<List<MembershipAccountDTO>> accountBucketResult = accountApi.queryAccountByIdRange(startUserId, endUserId); if (!accountBucketResult.getSuccess()) { throw new BaseBizException(accountBucketResult.getErrorCode(), accountBucketResult.getErrorMessage()); } List<MembershipAccountDTO> accountBucket = accountBucketResult.getData(); if (CollectionUtils.isEmpty(accountBucket)) { log.info("根据用户桶内的id范围没有查询到用户, startUserId={}, endUserId{}", startUserId, endUserId); continue; } //3.每个用户发送一条"平台发送优惠券消息" //方法一：直接使用线程池进行发送发放优惠券消息到MQ； //这里是并行消费的，以上逻辑已经是并行执行的了，而且有查库的操作 //accountBucket 默认是 1000 个用户，要为每一个用户都发送一条"平台发送优惠券消息"，也就是1000条消息 //下面我们使用线程池来并行发送这1000条消息（ps：另一种也可以像发送优惠券用户桶消息一样用批量发送） PlatformCouponMessage couponMessage = PlatformCouponMessage.builder() .couponId(message.getCouponId()) .activityStartTime(message.getActivityStartTime()) .activityEndTime(message.getActivityEndTime()) .couponType(message.getCouponType()) .build(); for (MembershipAccountDTO account : accountBucket) { sharedSendMsgThreadPool.execute(() -> { couponMessage.setUserAccountId(account.getId()); String jsonMessage = JSON.toJSONString(couponMessage); defaultProducer.sendMessage(RocketMqConstant.PLATFORM_COUPON_SEND_TOPIC, jsonMessage, "平台发送优惠券消息"); }); } //方法二：合并batch后再使用线程池发送MQ /*List<String> messages = new ArrayList<>(100); for (MembershipAccountDTO account : accountBucket) { couponMessage.setUserAccountId(account.getId()); messages.add(JSON.toJSONString(couponMessage)); if (messages.size() == 100) { sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_COUPON_SEND_TOPIC, messages, "平台发送优惠券消息"); }); messages.clear(); } } //最后剩下的也批量发出 if (!CollectionUtils.isEmpty(messages)) { sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_PROMOTION_SEND_TOPIC, messages, "平台发送促销活动消息"); }); messages.clear(); }*/ } } catch (Exception e){ log.error("consume error,平台优惠券消费失败", e); //本次消费失败，下次重新消费 return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }最后营销系统监听如下Topic，然后通过线程池对每个用户发放优惠券。
PLATFORM_PROMOTION_SEND_TOPIC@Configuration public class ConsumerBeanConfig { ... //平台发放优惠券领取消费者 @Bean("platformCouponReceiveTopicConsumer") public DefaultMQPushConsumer receiveCouponConsumer(PlatFormCouponListener platFormCouponListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_COUPON_SEND_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_COUPON_SEND_TOPIC, "*"); consumer.registerMessageListener(platFormCouponListener); consumer.start(); return consumer; } } @Component public class PlatFormCouponListener implements MessageListenerConcurrently { //优惠券服务service @Autowired private CouponItemService couponItemService; //测试completableFuture使用commonPool的时是不需要初始化业务ThreadPoolExecutor的 //这里用supplier懒加载让测试completableFuture使用commonPool时不要初始化线程池 //只有当使用completableFuture使用自定义的线程时才初始化线程池 private static final int PERMITS = 30; private static final AtomicBoolean initializedRef = new AtomicBoolean(false); private static ThreadPoolExecutor THREAD_POOL_EXECUTOR = null; private static final Supplier<ThreadPoolExecutor> THREAD_POOL_EXECUTOR_SUPPLIER = () -> { if (initializedRef.compareAndSet(false, true)) { THREAD_POOL_EXECUTOR = new ThreadPoolExecutor(PERMITS, PERMITS * 2, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(1000), NamedDaemonThreadFactory.getInstance("consumeCouponMsg"), new ThreadPoolExecutor.CallerRunsPolicy()); } return THREAD_POOL_EXECUTOR; }; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { //方式一：使用默认的commonPool来处理任务 //supplyAsync(Supplier<U> supplier) API //默认使用的是 ForkJoinPool.commonPool() 这个线程池 //该线程池在jvm内是唯一的，默认的线程数量是cpu的核数减1 //如果觉得线程数不够用可以通过jvm系统参数 java.util.concurrent.ForkJoinPool.common.parallelism 的值调整commonPool的并行度，或者采用方式二 List<CompletableFuture<SalesPromotionCouponItemDTO>> futureList = msgList.stream() .map(e -> CompletableFuture.supplyAsync(() -> handleMessageExt(e))) .collect(Collectors.toList()); //方式二：使用自定的业务线程池来处理任务 //List<CompletableFuture<SalesPromotionCouponItemDTO>> futureList = msgList.stream() // .map(e -> CompletableFuture.supplyAsync(() -> handleMessageExt(e), THREAD_POOL_EXECUTOR_SUPPLIER.get())) // .collect(Collectors.toList()); List<SalesPromotionCouponItemDTO> couponItemDTOList = futureList.stream() .map(CompletableFuture::join) .filter(Objects::nonNull) .collect(Collectors.toList()); //优惠券保存到数据库 couponItemService.saveCouponBatch(couponItemDTOList); } catch (Exception e) { log.error("consume error,平台优惠券消费失败", e); //本次消费失败，下次重新消费 return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } public SalesPromotionCouponItemDTO handleMessageExt(MessageExt messageExt) { log.debug("执行平台发放优惠券消费消息逻辑，消息内容：{}", messageExt.getBody()); String msg = new String(messageExt.getBody()); PlatformCouponMessage platformCouponMessage = JSON.parseObject(msg , PlatformCouponMessage.class); log.info("开始发放平台优惠券，couponId:{}", platformCouponMessage.getCouponId()); //幂等逻辑防止重复消费 JsonResult<Long> result = couponItemService.selectByAccountIdAndCouponId(platformCouponMessage.getUserAccountId(), platformCouponMessage.getCouponId()); //如果已经存在，直接跳过循环，不再执行优惠券保存操作 if (result.getSuccess()) { return null; } SalesPromotionCouponItemDTO itemDTO = new SalesPromotionCouponItemDTO(); itemDTO.setCouponId(platformCouponMessage.getCouponId()); itemDTO.setCouponType(platformCouponMessage.getCouponType()); itemDTO.setUserAccountId(platformCouponMessage.getUserAccountId()); itemDTO.setIsUsed(0); itemDTO.setActivityStartTime(platformCouponMessage.getActivityStartTime()); itemDTO.setActivityEndTime(platformCouponMessage.getActivityEndTime()); return itemDTO; } }(2)给全量用户惰性发放优惠券的优化实现
一.首先需要配置好要使用的Redis相关Bean
@Data @Configuration @ConditionalOnClass(RedisConnectionFactory.class) public class RedisConfig { @Value("${spring.redis.host}") private String host; @Value("${spring.redis.port}") private String port; @Value("${spring.redis.password}") private String password; @Value("${spring.redis.timeout}") private int timeout; @Bean @ConditionalOnClass(RedisConnectionFactory.class) public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory) { RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>(); redisTemplate.setConnectionFactory(redisConnectionFactory); redisTemplate.setDefaultSerializer(new StringRedisSerializer()); redisTemplate.afterPropertiesSet(); return redisTemplate; } @Bean @ConditionalOnClass(RedissonClient.class) public RedissonClient redissonClient() { Config config = new Config(); config.useSingleServer() .setAddress("redis://" + host + ":" + port) .setPassword(password) .setConnectionMinimumIdleSize(10) .setConnectionPoolSize(100) .setIdleConnectionTimeout(600000) .setSubscriptionConnectionMinimumIdleSize(10) .setSubscriptionConnectionPoolSize(100) .setTimeout(timeout); config.setCodec(new StringCodec()); config.setThreads(5); config.setNettyThreads(5); RedissonClient client = Redisson.create(config); return client; } @Bean @ConditionalOnClass(RedisConnectionFactory.class) public RedisCache redisCache(RedisTemplate redisTemplate) { return new RedisCache(redisTemplate); } }二.然后实现基于Redisson分布式锁维护优惠券缓存列表，以及惰性优惠券过期缓存清理
其实就是新增优惠券时，将优惠券信息写入Redis，并检查优惠券是否过期，如果过期就进行删除。
//优惠券接口实现 @Service public class CouponServiceImpl implements CouponService { //Redis客户端工具 @Autowired private RedisCache redisCache; @Autowired private RedissonClient redissonClient; ... //保存/修改优惠券活动方法 @Transactional(rollbackFor = Exception.class) @Override public SaveOrUpdateCouponDTO saveOrUpdateCoupon(SaveOrUpdateCouponRequest request) { SalesPromotionCouponDO couponDO = couponConverter.convertCouponDO(request); couponDO.setCouponReceivedCount(0); salesPromotionCouponDAO.saveOrUpdateCoupon(couponDO); //判断优惠券类型 if (CouponSendTypeEnum.PLATFORM_SEND.getCode().equals(request.getCouponReceiveType())) { //一.如果是系统发放类型，则针对所有用户，发送优惠券到MQ writeCouponToRedis(couponDO); } else { //二.如果是自己领取类型 //TODO } SaveOrUpdateCouponDTO dto = new SaveOrUpdateCouponDTO(); dto.setCouponName(request.getCouponName()); dto.setRule(request.getCouponRule()); dto.setSuccess(true); return dto; } private void writeCouponToRedis(SalesPromotionCouponDO coupon) { //首先需要用Redisson基于Redis做一个分布式锁的加锁PROMOTION_COUPON_ID_LIST_LOCK //再去维护一个"一共发出去了多少张券"的数据结构PROMOTION_COUPON_ID_LIST RLock lock = redissonClient.getLock(RedisKey.PROMOTION_COUPON_ID_LIST_LOCK); try { //进行加锁，超时时间为60s释放 lock.lock(60, TimeUnit.SECONDS); List<Long> couponIds = null; String couponIdsJSON = redisCache.get(RedisKey.PROMOTION_COUPON_ID_LIST); if (couponIdsJSON == null || couponIdsJSON.equals("")) { couponIds = new ArrayList<>(); } else { couponIds = JSON.parseObject(couponIdsJSON, List.class); } //检查每个优惠券时间是否过期了，如果过期或者已经发完了券，则把它从List里删除，以及从Redis里删除 //如果是全量发券，则不会发完，因此可以给所有人发，如果超过了时间才不能发券 if (couponIds.size() > 0) { Iterator<Long> couponIdIterator = couponIds.iterator(); while (couponIdIterator.hasNext()) { Long tempCouponId = couponIdIterator.next(); String tempCouponJSON = redisCache.get(RedisKey.PROMOTION_COUPON_KEY + "::" + tempCouponId); SalesPromotionCouponDO tempCoupon = JSON.parseObject(tempCouponJSON, SalesPromotionCouponDO.class); Date now = new Date(); if (now.after(tempCoupon.getActivityEndTime())) { couponIdIterator.remove(); redisCache.delete(RedisKey.PROMOTION_COUPON_KEY + "::" + tempCouponId); } } } couponIds.add(coupon.getId()); couponIdsJSON = JsonUtil.object2Json(couponIds); redisCache.set(RedisKey.PROMOTION_COUPON_ID_LIST, couponIdsJSON, -1); String couponJSON = JsonUtil.object2Json(coupon); redisCache.set(RedisKey.PROMOTION_COUPON_KEY + "::" + coupon.getId(), couponJSON, -1); } finally { lock.unlock(); } } }三.接着实现会员系统发布用户登录事件 + 营销系统在用户登录后的惰性发券
会员系统在用户登录时会发送消息到MQ的USER_LOGINED_EVENT_TOPIC：
@RestController @RequestMapping("/demo/membership") public class MembershipController { @Autowired private DefaultProducer defaultProducer; @Autowired private MembershipAccountService accountService; //触发用户登录 @PostMapping("/triggerUserLoginEvent") public JsonResult<Boolean> triggerUserLoginEvent(Long accountId) { try { List<MembershipAccountDTO> accounts = accountService.queryAccountByIdRange(accountId, accountId); if (accounts != null && accounts.size() > 0) { MembershipAccountDTO account = accounts.get(0); UserLoginedEvent userLoginedEvent = new UserLoginedEvent(); userLoginedEvent.setAccount(account); String userLoginedEventJSON = JsonUtil.object2Json(userLoginedEvent); defaultProducer.sendMessage(RocketMqConstant.USER_LOGINED_EVENT_TOPIC, userLoginedEventJSON, "用户登录事件发生了"); } return JsonResult.buildSuccess(true); } catch (BaseBizException e) { log.error("biz error: request={}", accountId, e); return JsonResult.buildError(e.getErrorCode(), e.getErrorMsg()); } catch (Exception e) { log.error("system error: request={}", accountId, e); return JsonResult.buildError(e.getMessage()); } } }营销系统监听USER_LOGINED_EVENT_TOPIC对用户登录时发送的登录事件消息进行惰性发券：
@Configuration public class ConsumerBeanConfig { ... @Bean("userLoginedEventListener") public DefaultMQPushConsumer userLoginedEventListener(UserLoginedEventListener userLoginedEventListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(USER_LOGINED_EVENT_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(USER_LOGINED_EVENT_TOPIC, "*"); consumer.registerMessageListener(userLoginedEventListener); consumer.start(); return consumer; } } //用户登录事件监听器 @Component public class UserLoginedEventListener implements MessageListenerConcurrently { @Autowired private RedisCache redisCache; @Autowired private SalesPromotionCouponItemDAO couponItemDAO; @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext consumeConcurrentlyContext) { try { //从Redis缓存里查询所有的优惠券 String couponIdsJSON = redisCache.get(RedisKey.PROMOTION_COUPON_ID_LIST); List<Long> couponIds = JSON.parseObject(couponIdsJSON, List.class); List<SalesPromotionCouponDO> coupons = new ArrayList<>(); for (Long couponId : couponIds) { String couponJSON = redisCache.get(RedisKey.PROMOTION_COUPON_KEY + "::" + couponId); SalesPromotionCouponDO coupon = JSON.parseObject(couponJSON, SalesPromotionCouponDO.class); Date now = new Date(); if (now.after(coupon.getActivityStartTime()) && now.before(coupon.getActivityEndTime())) { coupons.add(coupon); } } for (MessageExt messageExt : list) { //这个代码就可以拿到一个刚刚登陆成功的用户 String message = new String(messageExt.getBody()); UserLoginedEvent userLoginedEvent = JSON.parseObject(message, UserLoginedEvent.class); MembershipAccountDTO account = userLoginedEvent.getAccount(); //遍历每一个优惠券，检查这个优惠券是否有效，是否还可以继续进行发券，以及当前用户是否发过券，然后才给用户进行发券 for (SalesPromotionCouponDO coupon : coupons) { String receiveCouponFlag = redisCache.get(RedisKey.PROMOTION_USER_RECEIVE_COUPON + "::" + account.getId() + "::" + coupon.getId()); if (receiveCouponFlag == null || receiveCouponFlag.equals("")) { SalesPromotionCouponItemDO couponItem = new SalesPromotionCouponItemDO(); couponItem.setActivityEndTime(coupon.getActivityEndTime()); couponItem.setActivityStartTime(coupon.getActivityStartTime()); couponItem.setCouponId(coupon.getId()); couponItem.setCouponType(coupon.getCouponType()); couponItem.setCreateTime(new Date()); couponItem.setCreateUser(account.getId()); couponItem.setIsUsed(0); couponItem.setUpdateTime(new Date()); couponItem.setUpdateUser(account.getId()); couponItem.setUserAccountId(account.getId()); couponItemDAO.receiveCoupon(couponItem); redisCache.set(RedisKey.PROMOTION_USER_RECEIVE_COUPON + "::" + account.getId() + "::" + coupon.getId(), "true", -1); } } } } catch(Exception e) { log.error("consume error, 用户登录事件处理异常", e); return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }
11.指定用户群体发券的代码实现
(1)千万级用户推送和发放优惠券的方案总结
(2)指定用户群体发券实现
(1)千万级用户推送和发放优惠券的方案总结
创建促销活动和发放优惠券时，都不是直接进行推送和发放的，而是利用RocketMQ进行多次中转、异步化处理。
营销系统首先会查出用户总数，然后进行任务分片，接着把分片任务消息通过batch合并发送到MQ(异步化提升性能)。
营销系统会消费这些分片任务消息，并查询出用户和封装好每个用户的消息，然后发到MQ(解耦会员系统和推送系统)。
推送系统最后会消费每个用户的推送消息，并基于线程池采用多线程并发进行推送。
(2)指定用户群体发券实现
典型的例子就是为激活百万不活跃用户发放优惠券。
一.营销系统创建指定用户群体发券的入口代码
@RestController @RequestMapping("/demo/promotion/coupon") public class PromotionCouponController { ... @RequestMapping("/send") public JsonResult<SendCouponDTO> sendCouponByConditions(@RequestBody SendCouponRequest request) { try { log.info("发送优惠券给指定用户群体:{}", JSON.toJSONString(request)); SendCouponDTO dto = couponService.sendCouponByConditions(request); return JsonResult.buildSuccess(dto); } catch (BaseBizException e) { log.error("biz error: request={}", JSON.toJSONString(request), e); return JsonResult.buildError(e.getErrorCode(), e.getErrorMsg()); } catch (Exception e) { log.error("system error: request={}", JSON.toJSONString(request), e); return JsonResult.buildError(e.getMessage()); } } } @Service public class CouponServiceImpl implements CouponService { @DubboReference(version = "1.0.0") private MessagePushApi messagePushApi; ... @Transactional(rollbackFor = Exception.class) @Override public SendCouponDTO sendCouponByConditions(SendCouponRequest sendCouponRequest) { //保存优惠券信息 SalesPromotionCouponDO couponDO = couponConverter.convertCouponDO(sendCouponRequest); couponDO.setCouponReceivedCount(0); couponDO.setCouponStatus(CouponStatusEnum.NORMAL.getCode()); couponDO.setCouponReceiveType(CouponSendTypeEnum.SELF_RECEIVE.getCode()); salesPromotionCouponDAO.saveOrUpdateCoupon(couponDO); //分片和批量发送发放优惠券消息 shardBatchSendCouponMessage(sendCouponRequest); SendCouponDTO sendCouponDTO = new SendCouponDTO(); sendCouponDTO.setSuccess(Boolean.TRUE); sendCouponDTO.setCouponName(sendCouponRequest.getCouponName()); sendCouponDTO.setRule(sendCouponRequest.getCouponRule()); //TODO 发放数量 sendCouponDTO.setSendCount(0); return sendCouponDTO; } }二.营销系统分片和批量发送发券消息的代码
其中要去画像系统获取用户信息，然后根据用户数量进行分片，接着进行批量发送，发送到MQ的如下Topic。
PLATFORM_CONDITION_COUPON_SEND_USER_BUCKET_TOPIC@Service public class CouponServiceImpl implements CouponService { ... //分片和批量发送发放优惠券消息 private void shardBatchSendCouponMessage(SendCouponRequest sendCouponRequest) { //1.到画像系统获取当前条件下的count值 MembershipFilterDTO membershipFilterDTO = sendCouponRequest.getMembershipFilterDTO(); PersonaFilterConditionDTO conditionDTO = conditionConverter.convertFilterCondition(membershipFilterDTO); JsonResult<Integer> countResult = personaApi.countByCondition(conditionDTO); if (!countResult.getSuccess()) { throw new BaseBizException(countResult.getErrorCode(), countResult.getErrorMessage()); } //2.根据count值分片 //分成m个分片，每个分片中包含：(1)分片ID；(2)用户个数； //例：maxUserId = 100w; userBucketSize=1000 //userBucket1 = [1, 1000) //userBucket2 = [2, 1000) //userBucket2 = [n, 756)，最后一个分片可能数量不足1000 //userBucketCount = 1000 Integer count = countResult.getData(); Map<Integer, Integer> userBuckets = new LinkedHashMap<>(); AtomicBoolean flagRef = new AtomicBoolean(true); Integer shardId = 1; while (flagRef.get()) { if (USER_BUCKET_SIZE > count) { userBuckets.put(shardId, USER_BUCKET_SIZE); flagRef.compareAndSet(true, false); } userBuckets.put(shardId, USER_BUCKET_SIZE); shardId += 1; count -= USER_BUCKET_SIZE; } //3.批量发送消息 //例：userBucketCount = 1000; messageBatchSize = 100 List<String> messages = new ArrayList<>(); PlatformPromotionConditionUserBucketMessage message = PlatformPromotionConditionUserBucketMessage.builder().personaFilterCondition(JSON.toJSONString(conditionDTO)).build(); for (Map.Entry<Integer, Integer> userBucket : userBuckets.entrySet()) { message.setShardId(userBucket.getKey()); message.setBucketSize(userBucket.getValue()); String jsonMessage = JsonUtil.object2Json(message); messages.add(jsonMessage); } log.info("本次推送消息数量,{}",messages.size()); ListSplitter splitter = new ListSplitter(messages, MESSAGE_BATCH_SIZE); while (splitter.hasNext()) { List<String> sendBatch = splitter.next(); log.info("本次批次消息数量,{}",sendBatch.size()); sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_CONDITION_COUPON_SEND_USER_BUCKET_TOPIC, sendBatch, "部分用户优惠活动用户桶消息"); }); } } }三.营销系统对指定用户群体分片消息的处理和推送代码
首先，营销系统会消费MQ的如下Topic：
PLATFORM_CONDITION_COUPON_SEND_USER_BUCKET_TOPIC然后，营销系统会把领取优惠券的消息会发送到MQ的如下Topic：
PLATFORM_CONDITION_COUPON_SEND_TOPIC@Configuration public class ConsumerBeanConfig { ... @Bean("platFormConditionCouponUserBucketConsumer") public DefaultMQPushConsumer platFormConditionCouponUserBucketConsumer(PlatFormConditionCouponUserBucketListener platFormPromotionUserBucketListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_CONDITION_COUPON_SEND_USER_BUCKET_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_CONDITION_COUPON_SEND_USER_BUCKET_TOPIC, "*"); consumer.registerMessageListener(platFormPromotionUserBucketListener); consumer.start(); return consumer; } } @Component public class PlatFormConditionCouponUserBucketListener implements MessageListenerConcurrently { //用户画像服务 @DubboReference(version = "1.0.0") private PersonaApi personaApi; //发送消息共用的线程池 @Autowired @Qualifier("sharedSendMsgThreadPool") private SafeThreadPool sharedSendMsgThreadPool; //RocketMQ生产者 @Autowired private DefaultProducer defaultProducer; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { for (MessageExt messageExt : msgList) { //1.反序列化消息 String messageString = new String(messageExt.getBody()); log.debug("部分用户领取优惠券用户桶消息逻辑，消息内容：{}", messageString); PlatformPromotionConditionUserBucketMessage message = JSON.parseObject(messageString, PlatformPromotionConditionUserBucketMessage.class); //2.查询桶内的用户信息 Integer shardId = message.getShardId(); //根据分片id，和分片数量大小，计算出本次分片的起始userId Long startUserId = (shardId.longValue() - 1) * 1000; Integer bucketSize = message.getBucketSize(); String personaFilterCondition = message.getPersonaFilterCondition(); PersonaFilterConditionDTO personaFilterConditionDTO = JSON.parseObject(personaFilterCondition, PersonaFilterConditionDTO.class); //封装查询用户id的条件 PersonaConditionPage page = PersonaConditionPage.builder() .memberPoint(personaFilterConditionDTO.getMemberPoint()) .memberLevel(personaFilterConditionDTO.getMemberLevel()) .offset(startUserId) .limit(bucketSize) .build(); //从用户画像系统查询用户账号id JsonResult<List<Long>> accountIdsResult = personaApi.getAccountIdsByIdLimit(page); if (!accountIdsResult.getSuccess()) { throw new BaseBizException(accountIdsResult.getErrorCode(), accountIdsResult.getErrorMessage()); } List<Long> accountIds = accountIdsResult.getData(); if (CollectionUtils.isEmpty(accountIds)) { log.info("根据用户桶内的分片信息没有查询到用户, shardId={}", shardId); continue; } //3.每个用户发送一条领取优惠券的消息通知 PlatformMessagePushMessage pushMessage = PlatformMessagePushMessage.builder() .message("恭喜您获得优惠券领取资格，点击www.wjunt.com进入活动页面") .mainMessage("获得优惠券领取资格") .informType(InformTypeEnum.APP.getCode()) .build(); List<String> messages = new ArrayList<>(); for (Long accountId : accountIds) { pushMessage.setUserAccountId(accountId); messages.add(JSON.toJSONString(pushMessage)); } log.info("本次推送消息数量,{}",messages.size()); ListSplitter splitter = new ListSplitter(messages, MESSAGE_BATCH_SIZE); while (splitter.hasNext()) { List<String> sendBatch = splitter.next(); sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_CONDITION_COUPON_SEND_TOPIC, sendBatch, "平台发送优惠券消息"); }); } } } catch (Exception e){ log.error("consume error,消费失败", e); return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }四.推送系统会监听如下Topic消费消息推送领取优惠券的消息
PLATFORM_CONDITION_COUPON_SEND_TOPIC@Configuration public class ConsumerBeanConfig { ... @Bean("platFormConditionCouponConsumer") public DefaultMQPushConsumer platFormConditionCouponConsumer(PlatFormConditionCouponListener platFormPromotionListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_CONDITION_COUPON_SEND_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_CONDITION_COUPON_SEND_TOPIC, "*"); consumer.registerMessageListener(platFormPromotionListener); consumer.start(); return consumer; } } @Component public class PlatFormConditionCouponListener implements MessageListenerConcurrently { //消息推送工厂提供者 @Autowired private FactoryProducer factoryProducer; @Autowired private RedisTemplate<String, String> redisTemplate; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { //方式二：使用自定的业务线程池来处理任务 List<CompletableFuture<AltResult>> futureList = msgList.stream() .map(e -> CompletableFuture.supplyAsync(() -> handleMessageExt(e))) .collect(Collectors.toList()); List<Throwable> resultList = futureList.stream() .map(CompletableFuture::join) .filter(e -> e.ex != null) .map(e -> e.ex) .collect(Collectors.toList()); } catch (Exception e) { log.error("consume error,消费失败", e); return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } private AltResult handleMessageExt(MessageExt messageExt) { try { log.debug("执行平台发送通知消息逻辑，消息内容：{}", messageExt.getBody()); String msg = new String(messageExt.getBody()); PlatformMessagePushMessage message = JSON.parseObject(msg , PlatformMessagePushMessage.class); //幂等控制 if (StringUtils.isNotBlank(redisTemplate.opsForValue().get(PROMOTION_CONDITION_COUPON_KEY + message.getUserAccountId()))) { return new AltResult(null); } //获取消息服务工厂 MessageSendServiceFactory messageSendServiceFactory = factoryProducer.getMessageSendServiceFactory(message.getInformType()); //消息发送服务组件 MessageSendService messageSendService = messageSendServiceFactory.createMessageSendService(); //构造消息 PlatformMessagePushMessage messagePushMessage = PlatformMessagePushMessage.builder() .informType(message.getInformType()) .mainMessage(message.getMainMessage()) .userAccountId(message.getUserAccountId()) .message(message.getMessage()) .build(); MessageSendDTO messageSendDTO = messageSendServiceFactory.createMessageSendDTO(messagePushMessage); messageSendService.send(messageSendDTO); //发送成功之后把已经发送成功记录到redis redisTemplate.opsForValue().set(PROMOTION_CONDITION_COUPON_KEY + message.getUserAccountId(), UUID.randomUUID().toString()); log.info("消息推送完成，messageSendDTO:{}", messageSendDTO); return new AltResult(null); } catch (Exception e) { return new AltResult(e); } } //completableFuture的返回结果，适用于无返回值的情况 //ex字段为null表示任务执行成功 //ex字段不为null表示任务执行失败，并把异常设置为ex字段 private static class AltResult { final Throwable ex; public AltResult(Throwable ex) { this.ex = ex; } } }
12.分片消息的batch合并算法重构实现
之前的batch合并是按照条数进行合并的，现在重构为按照合并后的大小不超过800KB和不超过100条进行合并。
public class ListSplitter implements Iterator<List<String>> { //设置每一个batch最多不超过800k，因为RocketMQ官方推荐，一条消息不建议长度超过1MB //而封装一个RocketMQ的message，包括了MessageBody, Topic，Addr等数据，所以设置小一点 private int sizeLimit = 800 * 1024; private final List<String> messages; private int currIndex; private int batchSize = 100; public ListSplitter(List<String> messages, Integer batchSize) { this.messages = messages; this.batchSize = batchSize; } public ListSplitter(List<String> messages) { this.messages = messages; } @Override public boolean hasNext() { return currIndex < messages.size(); } //每次从list中取一部分 @Override public List<String> next() { int nextIndex = currIndex; int totalSize = 0; for (; nextIndex < messages.size(); nextIndex++) { String message = messages.get(nextIndex); //获取每条message的长度 int tmpSize = message.length(); if (tmpSize > sizeLimit) { if (nextIndex - currIndex == 0) { nextIndex++; } break; } if (tmpSize + totalSize > sizeLimit || (nextIndex - currIndex) == batchSize ) { break; } else { totalSize += tmpSize; } } List<String> subList = messages.subList(currIndex, nextIndex); currIndex = nextIndex; return subList; } @Override public void remove() { throw new UnsupportedOperationException("Not allowed to remove"); } }
13.百万画像群体爆款商品推送代码实现
(1)营销系统的定时推送任务
(2)推送系统消费分片推送任务消息，发送具体用户的推送消息到MQ
(3)推送系统消费具体用户的推送消息进行真正推送
(1)营销系统的定时推送任务
@Component public class ScheduleSendMessageJobHandler { ... //执行定时任务，筛选热门商品和用户发送给MQ @XxlJob("hotGoodsPushHandler") public void hotGoodsPushHandler() { log.info("hotGoodsPushHandler 开始执行"); //获取热门商品和用户画像，业务先简化为一对一关系 List<HotGoodsCrontabDO> crontabDOs = hotGoodsCrontabDAO.queryHotGoodsCrontabByCrontabDate(new Date()); log.info("获取热门商品和用户画像数据, crontabDOs:{}", JsonUtil.object2Json(crontabDOs)); //找出每个热门商品对应画像所匹配的用户 for (HotGoodsCrontabDO crontabDO : crontabDOs) { log.info("自动分片逻辑, 当前任务：crontabDO:{}", JsonUtil.object2Json(crontabDO)); if (StringUtils.isEmpty(crontabDO.getPortrayal())) { continue; } //热门商品对应的画像实体 MembershipPointDTO membershipPointDTO = JsonUtil.json2Object(crontabDO.getPortrayal(), MembershipPointDTO.class); if (Objects.isNull(membershipPointDTO)) { continue; } //获取匹配画像的用户实体 MembershipFilterConditionDTO conditionDTO = buildCondition(membershipPointDTO); PersonaFilterConditionDTO personaFilterConditionDTO = conditionConverter.convertFilterCondition(conditionDTO); log.info("用户查询条件：{}", personaFilterConditionDTO); //获取画像用户匹配的用户ID最大最小值 JsonResult<Long> accountMaxIdResult = personaApi.queryMaxIdByCondition(personaFilterConditionDTO); log.info("获取最大ID，result:{}", JsonUtil.object2Json(accountMaxIdResult)); if (!accountMaxIdResult.getSuccess()) { log.info("获取最大ID失败，condition:{}", JsonUtil.object2Json(personaFilterConditionDTO)); throw new BaseBizException(accountMaxIdResult.getErrorCode(), accountMaxIdResult.getErrorMessage()); } JsonResult<Long> accountMinIdResult = personaApi.queryMinIdByCondition(personaFilterConditionDTO); log.info("获取最小ID，result:{}", JsonUtil.object2Json(accountMinIdResult)); if (!accountMinIdResult.getSuccess()) { log.info("获取最小ID失败，condition:{}", JsonUtil.object2Json(personaFilterConditionDTO)); throw new BaseBizException(accountMinIdResult.getErrorCode(), accountMinIdResult.getErrorMessage()); } //需要执行推送的用户起始ID //注意：这是一个预估值，因为最小ID到最大ID中间会有很多不符合条件的用户 //针对这些用户，需要在下一层的业务逻辑中，用选人条件过滤掉 Long minUserId = accountMinIdResult.getData(); Long maxUserId = accountMaxIdResult.getData(); //bucket就是一个用户分片，对应的是一个startUserId -> endUserId，用户ID范围 //可以根据一定的算法，把千万级用户推送任务分片，比如一个分片后的推送任务包含1000个用户/2000个用户 //userBuckets就有上万条key-value对，每个key-value对就是一个startUserId -> endUserId的推送任务分片 final int userBucketSize = 1000; Map<Long, Long> userBuckets = new LinkedHashMap<>(); AtomicBoolean doSharding = new AtomicBoolean(true); long startUserId = minUserId; log.info("开始对任务人群进行分片，startId:{}",minUserId); while (doSharding.get()) { if ((maxUserId -minUserId) < userBucketSize) { userBuckets.put(startUserId, maxUserId); doSharding.compareAndSet(true, false); break; } userBuckets.put(startUserId, startUserId + userBucketSize); startUserId += userBucketSize; maxUserId -= userBucketSize; } //把可能成千上万的分片推送任务进行RocketMQ消息的batch合并，以batch模式的发送任务到MQ，减少跟RocketMQ网络通信的耗时 List<String> hotProductPushTasks = new ArrayList<>(); HotGoodsVO hotGoodsVO = buildHotGoodsVO(crontabDO); PlatformHotProductUserBucketMessage bucketMessage = PlatformHotProductUserBucketMessage.builder() .hotGoodsVO(JSON.toJSONString(hotGoodsVO)) .personaFilterConditionDTO(JSON.toJSONString(personaFilterConditionDTO)) .build(); for (Map.Entry<Long, Long> userBucket : userBuckets.entrySet()) { bucketMessage.setEndUserId(userBucket.getValue()); bucketMessage.setStartUserId(userBucket.getKey()); String promotionPushTaskJSON = JsonUtil.object2Json(bucketMessage); log.info("用户桶构建侧选人条件：{}",bucketMessage.getPersonaFilterConditionDTO()); hotProductPushTasks.add(promotionPushTaskJSON); } //MESSAGE_BATCH_SIZE指的是消息batch大小，RocketMQ的每个batch消息包含了100个推送任务 //这样可以将1万个推送任务消息合并为100个batch消息，进行100次网络通信发给RocketMQ，可以大幅度降低发送消息的耗时 ListSplitter splitter = new ListSplitter(hotProductPushTasks, MESSAGE_BATCH_SIZE); while (splitter.hasNext()) { List<String> sendBatch = splitter.next(); log.info("本次批次消息数量,{}",sendBatch.size()); sharedSendMsgThreadPool.execute(() -> { defaultProducer.sendMessages(RocketMqConstant.PLATFORM_HOT_PRODUCT_USER_BUCKET_SEND_TOPIC, sendBatch, "平台热门商品定时任务用户桶消息"); }); } } } }(2)推送系统消费分片推送任务消息，发送具体用户的推送消息到MQ
监听MQ的如下Topic：
PLATFORM_HOT_PRODUCT_USER_BUCKET_SEND_TOPIC@Configuration public class ConsumerBeanConfig { ... @Bean("PlatFormHotProductUserBucketConsumer") public DefaultMQPushConsumer PlatFormHotProductUserBucketConsumer(PlatFormHotProductUserBucketListener platFormHotProductUserBucketListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_HOT_PRODUCT_USER_BUCKET_SEND_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_HOT_PRODUCT_USER_BUCKET_SEND_TOPIC, "*"); consumer.registerMessageListener(platFormHotProductUserBucketListener); consumer.start(); return consumer; } } @Component public class PlatFormHotProductUserBucketListener implements MessageListenerConcurrently { @DubboReference(version = "1.0.0") private PersonaApi personaApi; @Autowired private DefaultProducer producer; //公用的消息推送线程池 @Autowired @Qualifier("sharedSendMsgThreadPool") private SafeThreadPool sharedSendMsgThreadPool; //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { for (MessageExt messageExt : msgList) { String msg = new String(messageExt.getBody()); PlatformHotProductUserBucketMessage hotProductMessagePushTask = JSON.parseObject(msg , PlatformHotProductUserBucketMessage.class); log.info("执行热门商品推送用户桶消息逻辑，消息内容：{}", hotProductMessagePushTask); //1.获取本次热门商品推送任务分片中的数据 String hotGoodString = hotProductMessagePushTask.getHotGoodsVO(); String personaFilterCondition = hotProductMessagePushTask.getPersonaFilterConditionDTO(); HotGoodsVO hotGoodsVO = JSON.parseObject(hotGoodString, HotGoodsVO.class); log.info("选人条件，内容：{}", personaFilterCondition); if (Objects.isNull(personaFilterCondition) || Objects.isNull(hotGoodsVO)) { continue; } PersonaFilterConditionDTO conditionDTO = JSON.parseObject(personaFilterCondition, PersonaFilterConditionDTO.class); Long startUserId = hotProductMessagePushTask.getStartUserId(); Long endUserId = hotProductMessagePushTask.getEndUserId(); //分页查询条件 PersonaConditionWithIdRange page = PersonaConditionWithIdRange.builder() .memberLevel(conditionDTO.getMemberLevel()) .memberPoint(conditionDTO.getMemberPoint()) .startId(startUserId) .endId(endUserId) .build(); //2.查询本次推送任务分片对应的用户群体 //注意：查询的时候，传入查询条件过滤掉不符合条件的用户id JsonResult<List<Long>> queryAccountIdsResult = personaApi.getAccountIdsByIdRange(page); List<Long> accountIds = queryAccountIdsResult.getData(); PlatformHotProductMessage hotMessage = PlatformHotProductMessage.builder() .goodsName(hotGoodsVO.getGoodsName()) .goodsDesc(hotGoodsVO.getGoodsDesc()) .keyWords(hotGoodsVO.getKeyWords()) .build(); int handledBucketCount = 0; List<String> messages = new ArrayList<>(); for (Long accountId : accountIds) { handledBucketCount++; hotMessage.setAccountId(accountId); log.info("构造热门商品MQ消息, hotMessage: {}", hotMessage); messages.add(JSON.toJSONString(hotMessage)); } ListSplitter splitter = new ListSplitter(messages, MESSAGE_BATCH_SIZE); while (splitter.hasNext()) { List<String> sendBatch = splitter.next(); log.info("本次批次消息数量,{}", sendBatch.size()); sharedSendMsgThreadPool.execute(() -> { producer.sendMessages(RocketMqConstant.PLATFORM_HOT_PRODUCT_SEND_TOPIC, sendBatch, "平台热门商品定时任务用户桶消息"); }); } } } catch (Exception e) { log.error("consume error,热门商品通知消费失败", e); //这边因为是推送任务，个别失败也可以直接丢弃 return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }(3)推送系统消费具体用户的推送消息进行真正推送
监听MQ的如下Topic：
PLATFORM_HOT_PRODUCT_SEND_TOPIC@Configuration public class ConsumerBeanConfig { ... @Bean("platformHotProductSendTopicConsumer") public DefaultMQPushConsumer platformHotProductConsumer(PlatFormHotProductListener platFormHotProductListener) throws MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(PLATFORM_HOT_PRODUCT_SEND_CONSUMER_GROUP); consumer.setNamesrvAddr(rocketMQProperties.getNameServer()); consumer.subscribe(PLATFORM_HOT_PRODUCT_SEND_TOPIC, "*"); consumer.registerMessageListener(platFormHotProductListener); consumer.start(); return consumer; } } @Component public class PlatFormHotProductListener implements MessageListenerConcurrently { //并发消费消息 @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgList, ConsumeConcurrentlyContext context) { try { for (MessageExt messageExt : msgList) { log.debug("执行平台发送通知消息逻辑，消息内容：{}", messageExt.getBody()); String msg = new String(messageExt.getBody()); HashMap hotProductMessage = JSON.parseObject(msg , HashMap.class); //推送通知 informByPush(hotProductMessage); } } catch (Exception e) { log.error("consume error,平台优惠券消费失败", e); //本次消费失败，下次重新消费 return ConsumeConcurrentlyStatus.RECONSUME_LATER; } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } //第三方平台推送消息到app private void informByPush(HashMap message){ String messageBody = "速戳!精致小物件，" + message.get("keywords")+"！" + message.get("goodsName") + message.get("goodsDesc"); log.info("消息推送中：消息内容：{}", messageBody); } }
14.生产环境百万级用户PUSH全链路压测
百万级用户Push生产环境的部署：营销系统部署了5台2核4G机器、推送系统部署了5台2核4G机器、会员系统1台、画像系统1台。
假设会员系统⼀共有150w的⽤户数据，现在开启⼀次全员推送通知的优惠活动，总计MQ的数据量⼤概就是150w。开启⼀个全员活动的时间，⼤概⽤时27分钟左右，即完成了全部消息的推送，整体效率还是算⽐较⾼的。所以如果是千万级别的推送，⼤概也就是需要27 * 5⼤概是3个⼩时左右，和我们的预期是⽐较相似的。
150万的用户推送任务，按1000用户进行分片，那么总共会有1500个分片任务，每个分片任务需处理1000条消息。这1500个人分片任务通过batch合并发送到MQ，也非常快，假设每100个分片合并成一个batch，才15个batch消息。5台推送机器，每台机器开启30个线程，那么总共有150个线程。假设一个线程完成一条消息的推送需要200ms，那么每个线程每秒能推送5条消息，5台机器每秒能推送750条消息。150万 / 750 = 2000s = 30分钟，这就是通过计算预估的结果，和实际的27分钟相差不大。
由于有些场景下，这种全员性质的消息推送，是不需要接收推送结果的。如果直接放弃推送结果的获取操作，效率还能稍微有所提升。
