Browse code

[perf] BM-15059 Fix: reduce BMExecutor log freq under heavy workload

Thomas Cataldo authored on 17/07/2019 09:30:02
Showing 1 changed files
... ...
@@ -52,6 +52,7 @@ public class BMExecutor {
52 52
 	private static final Vertx timerMgmt = VertxPlatform.getVertx();
53 53
 
54 54
 	private volatile long lastFullLog = 0;
55
+	private volatile long lastInfoLog = 0;
55 56
 
56 57
 	private final ThreadPoolExecutor executor;
57 58
 	private final ConcurrentHashMap<Runnable, Thread> runningThreads;
... ...
@@ -166,21 +167,24 @@ public class BMExecutor {
166 167
 		return asExecutorService;
167 168
 	}
168 169
 
170
+	private static final String logMsg = "Executor/{} running: {} on {}, queue size: {}, total timed-out: {}, rejected: {}";
171
+
169 172
 	protected void logExecutorState() {
170 173
 		int active = runningThreads.size();
171
-		String logMsg = "Executor/{} running: {} on {}, queue size: {}, total timed-out: {}, rejected: {}";
172
-		if (active > (threadCounts / 2)) {
173
-
174
-			logger.warn(logMsg, name, active, threadCounts, executor.getQueue().size(), timedoutCounter.longValue(),
175
-					rejectionCounter.longValue());
174
+		if (active > (threadCounts * 0.8)) {
175
+			long now = System.currentTimeMillis();
176
+			if (lastInfoLog < (now - 2000)) {
177
+				lastInfoLog = now;
178
+				logger.info(logMsg, name, active, threadCounts, executor.getQueue().size(), timedoutCounter.longValue(),
179
+						rejectionCounter.longValue());
180
+			}
176 181
 
177
-			if (lastFullLog < (System.currentTimeMillis() - 2000)) {
178
-				lastFullLog = System.currentTimeMillis();
179
-				logRunningTasks((msg, params) -> logger.warn(msg, params));
180
-				logQueuedTasks((msg, params) -> logger.warn(msg, params));
182
+			if (lastFullLog < (now - 60000)) {
183
+				lastFullLog = now;
184
+				logRunningTasks((msg, params) -> logger.info(msg, params));
181 185
 			}
182
-		} else if (active > (threadCounts / 4)) {
183
-			logger.info(logMsg, name, active, threadCounts, executor.getQueue().size(), timedoutCounter.longValue(),
186
+		} else if (logger.isDebugEnabled() && active > (threadCounts * 0.5)) {
187
+			logger.debug(logMsg, name, active, threadCounts, executor.getQueue().size(), timedoutCounter.longValue(),
184 188
 					rejectionCounter.longValue());
185 189
 		}
186 190
 	}
... ...
@@ -200,14 +204,6 @@ public class BMExecutor {
200 204
 
201 205
 	}
202 206
 
203
-	private void logQueuedTasks(LogFunction func) {
204
-		ImmutableSet<Runnable> queuedTasks = ImmutableSet.copyOf(executor.getQueue());
205
-
206
-		queuedTasks.forEach(queued -> {
207
-			func.log("queued task {}", queued);
208
-		});
209
-	}
210
-
211 207
 	public void execute(BMTask command) {
212 208
 		execute(command, GlobalConstants.DEFAULT_TIMEOUT);
213 209
 	}