问题发现
早上过来，饭都没来的及吃，运维就给我发来信息，说是某个接口调用大量超时。因为最近这个接口调用量是翻倍了，所以我就去检查了下慢SQL，发现确实是有较多的慢SQL，所以我就缩减了查询的时间范围，但是效果并不好。
过了一会发现，这个服务fullGC是有问题的，太频繁了，这个应该是导致接口超时的根本问题，因为时间也是对的上的。
这个是最近三个小时fullGC的监控图：
1、首先去服务器上面下载dump文件，分析是哪里造成了内存泄漏，频繁触发fullGC。首先找出服务器内java文件的PID，然后保存dump文件，我们公司java服务是固定端口号：1
2、根据dump文件，分析出堆内对象的分布情况
3、看看我的业务代码是哪里出现了问题
//异步执行某个查询方法，并且在规定时间内返回查询结果 public <T> T asyncWithTimeout(ScheduledThreadPoolExecutor executor, Callable<T> callable, long time, TimeUnit unit) { try { ListeningExecutorService listeningExecutorService = MoreExecutors.listeningDecorator(threadPoolExecutor); ListenableFuture<T> future = listeningExecutorService.submit(callable); //这里是创建一个定时任务线程，去定时检查是否在规定时间内查询完毕，应该就是这个去添加了钩子函数，进去看看 ScheduledExecutorService scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(executor); return Futures.withTimeout(future, time, unit, scheduledExecutorService).get(time, unit); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.warn("异步方法执行失败，error:{}", e.getMessage()); } return null; } //=======================guava并发包代码======================= @Beta @GwtIncompatible // TODO public static ScheduledExecutorService getExitingScheduledExecutorService( ScheduledThreadPoolExecutor executor) { //每次都去创建一个新的对象 return new Application().getExitingScheduledExecutorService(executor); } final ScheduledExecutorService getExitingScheduledExecutorService( ScheduledThreadPoolExecutor executor) { return getExitingScheduledExecutorService(executor, 120, TimeUnit.SECONDS); } final ScheduledExecutorService getExitingScheduledExecutorService( ScheduledThreadPoolExecutor executor, long terminationTimeout, TimeUnit timeUnit) { useDaemonThreadFactory(executor); ScheduledExecutorService service = Executors.unconfigurableScheduledExecutorService(executor); //添加构造函数的地方，进去看看 addDelayedShutdownHook(executor, terminationTimeout, timeUnit); return service; } final void addDelayedShutdownHook( final ExecutorService service, final long terminationTimeout, final TimeUnit timeUnit) { checkNotNull(service); checkNotNull(timeUnit); //继续点进去 addShutdownHook( MoreExecutors.newThread( //线程名字对上了，就在对象引用的截图里面出现过 "DelayedShutdownHook-for-" + service, new Runnable() { @Override public void run() { try { // We'd like to log progress and failures that may arise in the // following code, but unfortunately the behavior of logging // is undefined in shutdown hooks. // This is because the logging code installs a shutdown hook of its // own. See Cleaner class inside {@link LogManager}. service.shutdown(); service.awaitTermination(terminationTimeout, timeUnit); } catch (InterruptedException ignored) { // We're shutting down anyway, so just ignore. } } })); } @VisibleForTesting void addShutdownHook(Thread hook) { Runtime.getRuntime().addShutdownHook(hook); } //=======================guava并发包代码======================= public void addShutdownHook(Thread hook) { SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new RuntimePermission("shutdownHooks")); } //定位到问题了，就是这里添加的钩子函数 ApplicationShutdownHooks.add(hook); } static synchronized void add(Thread hook) { if(hooks == null) throw new IllegalStateException("Shutdown in progress"); if (hook.isAlive()) throw new IllegalArgumentException("Hook already running"); if (hooks.containsKey(hook)) throw new IllegalArgumentException("Hook previously registered"); //存在到 hooks 这个map对象里面，就是这个大对象 hooks.put(hook, hook); }private ListeningExecutorService listeningExecutorService; private ScheduledExecutorService scheduledExecutorService; public static AsyncUtils getInstance() { return ThreadHolder.INSTANCE.getAsyncWithCallback(); } @SuppressWarnings("UnstableApiUsage") private AsyncUtils() { listeningExecutorService = MoreExecutors.listeningDecorator(ThreadPoolConstant.THREAD_POOL_EXECUTOR); scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(ThreadPoolConstant.SCHEDULED_THREAD_POOL_EXECUTOR); } @SuppressWarnings("UnstableApiUsage") public <T> T asyncWithTimeout(Callable<T> callable, long time, TimeUnit unit) { try { ListenableFuture<T> future = listeningExecutorService.submit(callable); return Futures.withTimeout(future, time, unit, scheduledExecutorService).get(time, unit); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.warn("异步方法执行失败，error:{}", e.getMessage()); } return null; } private enum ThreadHolder { /** * 线程持有类 INSTANCE */ INSTANCE; private final AsyncUtils asyncUtils; ThreadHolder() { asyncUtils = new AsyncUtils(); } public AsyncUtils getAsyncWithCallback() { return asyncUtils; } }
