背景
某天下午，客服反馈线上服务出现问题，不能分配了。于是我登录到系统上，进行同样的操作发现也不行。当然同时我已经登录到服务器打开了日志，发现报错了，下面就是日志的错误信息：
java.sql.SQLException: No operations allowed after statement closed.
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:129)
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:97)
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:89)
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:63)
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:73)
at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:82)
at com.mysql.cj.jdbc.ClientPreparedStatement.clearBatch(ClientPreparedStatement.java:287)
at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchInternal(ClientPreparedStatement.java:439)
at com.mysql.cj.jdbc.StatementImpl.executeBatch(StatementImpl.java:794)
at com.alibaba.druid.filter.FilterChainImpl.statement_executeBatch(FilterChainImpl.java:3101)
at com.alibaba.druid.filter.FilterAdapter.statement_executeBatch(FilterAdapter.java:2506)
at com.alibaba.druid.filter.FilterEventAdapter.statement_executeBatch(FilterEventAdapter.java:273)
at com.alibaba.druid.filter.FilterChainImpl.statement_executeBatch(FilterChainImpl.java:3099)
at com.alibaba.druid.filter.FilterAdapter.statement_executeBatch(FilterAdapter.java:2506)
at com.alibaba.druid.filter.FilterEventAdapter.statement_executeBatch(FilterEventAdapter.java:273)
at com.alibaba.druid.filter.FilterChainImpl.statement_executeBatch(FilterChainImpl.java:3099)
at com.alibaba.druid.proxy.jdbc.StatementProxyImpl.executeBatch(StatementProxyImpl.java:196)
at com.alibaba.druid.pool.DruidPooledPreparedStatement.executeBatch(DruidPooledPreparedStatement.java:551)
at sun.reflect.GeneratedMethodAccessor2252.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.ibatis.logging.jdbc.PreparedStatementLogger.invoke(PreparedStatementLogger.java:78)
at com.sun.proxy.$Proxy716.executeBatch(Unknown Source)
at com.baomidou.mybatisplus.core.executor.MybatisBatchExecutor.doFlushStatements(MybatisBatchExecutor.java:135)
at org.apache.ibatis.executor.BaseExecutor.flushStatements(BaseExecutor.java:129)
at org.apache.ibatis.executor.BaseExecutor.flushStatements(BaseExecutor.java:122)
at com.baomidou.mybatisplus.core.executor.MybatisCachingExecutor.flushStatements(MybatisCachingExecutor.java:218)
at sun.reflect.GeneratedMethodAccessor3312.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.ibatis.plugin.Plugin.invoke(Plugin.java:63)
at com.sun.proxy.$Proxy713.flushStatements(Unknown Source)
at sun.reflect.GeneratedMethodAccessor3312.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.ibatis.plugin.Plugin.invoke(Plugin.java:63)
at com.sun.proxy.$Proxy713.flushStatements(Unknown Source)
at org.apache.ibatis.session.defaults.DefaultSqlSession.flushStatements(DefaultSqlSession.java:252)
at com.baomidou.mybatisplus.extension.toolkit.SqlHelper.lambda$executeBatch$0(SqlHelper.java:217)
at com.baomidou.mybatisplus.extension.toolkit.SqlHelper.executeBatch(SqlHelper.java:179)
at com.baomidou.mybatisplus.extension.toolkit.SqlHelper.executeBatch(SqlHelper.java:211)
at com.baomidou.mybatisplus.extension.service.impl.ServiceImpl.executeBatch(ServiceImpl.java:239)
at com.baomidou.mybatisplus.extension.service.impl.ServiceImpl.saveBatch(ServiceImpl.java:135)
at com.baomidou.mybatisplus.extension.service.IService.saveBatch(IService.java:73)
at com.baomidou.mybatisplus.extension.service.IService$$FastClassBySpringCGLIB$$f8525d18.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:779)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:750)
at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:123)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:388)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:750)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:692)
at com.abc.services.oa.oaHouseBrokerRel.service.impl.OaHouseBrokerRelServiceImpl$$EnhancerBySpringCGLIB$$e2d73fd6.saveBatch(<generated>)
at com.abc.services.oa.oaHouseBrokerRel.service.impl.OaHouseBrokerRelServiceImpl.saveRelBatch(OaHouseBrokerRelServiceImpl.java:132)
at com.abc.services.oa.oaHouseBrokerRel.service.impl.OaHouseBrokerRelServiceImpl.saveRel(OaHouseBrokerRelServiceImpl.java:58)
at com.abc.services.oa.oaHouseBrokerRel.service.impl.OaHouseBrokerRelServiceImpl$$FastClassBySpringCGLIB$$cc62fe90.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:688)
at com.abc.services.oa.oaHouseBrokerRel.service.impl.OaHouseBrokerRelServiceImpl$$EnhancerBySpringCGLIB$$e2d73fd6.saveRel(<generated>)
at com.abc.admin.controller.oa.oaHouseBrokerRel.OaHouseBrokerRelController.saveRel(OaHouseBrokerRelController.java:39)
at sun.reflect.GeneratedMethodAccessor3543.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:197)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:141)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:106)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:894)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1063)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:963)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:681)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:228)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:190)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
at com.abc.common.log.LogFilter.doFilterInternal(LogFilter.java:68)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:190)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:190)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:143)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:382)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1723)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:750)
Caused by: com.mysql.cj.exceptions.StatementIsClosedException: No operations allowed after statement closed.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.cj.exceptions.ExceptionFactory.createException(ExceptionFactory.java:61)
at com.mysql.cj.exceptions.ExceptionFactory.createException(ExceptionFactory.java:85)
at com.mysql.cj.jdbc.StatementImpl.checkClosed(StatementImpl.java:338)
at com.mysql.cj.jdbc.ClientPreparedStatement.clearBatch(ClientPreparedStatement.java:282)
... 101 common frames omitted解决思路
我能正常登录，正常查询，说明并不是所有的操作都有问题。我执行的业务操作是insert操作，发现有问题，于是查看一下MySQL的连接信息：
show full processlist;发现好多连接都被阻塞住了（下面的图片是问题解决后的结果,解决前的有很多State=Waiting for handler commit的）
查看MySQL的官方文档对该状态的解释：
-
waiting for handler commitThe thread is waiting for a transaction to commit versus other parts of query processing.（与查询处理的其他部分相比，线程正在等待事务提交。）
说明事务一直等待提交，为什么提交不了？我查看当时的info一列，该列会显示正在执行的 SQL 查询或命令的详细信息。这些SQL语句也没什么不一样的，也不会有长事务等等。等把这些可能原因都过一遍之后，突然想到，有可能服务器磁盘空间满了，于是我到服务器上执行：
df -h
发现已经没有剩余空间了（当时使用已经100%，已经没有可用空间了）
于是通过以下命令查看常用文件夹下的目录文件大小：
du -sh /abc发现日志文件过多，太大了，于是清空日志文件
> a.log随后再通过：show full processlist;命令去查看，发现事务都已经正常提交了，系统也已经恢复了正常。
总结
- 服务器使用情况监控（需要完善）
- 定期清理日志文件（需要完善）
- 预警机制（需要完善）
