Browse code

[check_and_repair] COM-47 Fix: on mail-api#resync(), also check cyrus filesystem to get rid of extra records

Thomas Cataldo authored on 06/07/2019 11:04:26
Showing 2 changed files
... ...
@@ -199,6 +199,38 @@ public class BaseMailboxRecordsService implements IChangelogSupport, ICountingSu
199 199
 
200 200
 	}
201 201
 
202
+	protected boolean checkExistOnBackend(long imapUid) {
203
+		SubtreeLocation recordsLocation = optRecordsLocation
204
+				.orElseThrow(() -> new ServerFault("Missing subtree location"));
205
+		String datalocation = DataSourceRouter.location(context, recordsLocation.subtreeContainer);
206
+		CyrusPartition partition = CyrusPartition.forServerAndDomain(datalocation, container.domainUid);
207
+
208
+		Iterator<String> mbox = Splitter.on("/").split(recordsLocation.contName).iterator();
209
+		String mboxType = mbox.next();
210
+
211
+		MailboxDescriptor md = new MailboxDescriptor();
212
+		md.type = "users".equals(mboxType) ? Type.user : Type.mailshare;
213
+		md.mailboxName = mbox.next();
214
+		md.utf7FolderPath = UTF7Converter.encode(recordsLocation.imapPath());
215
+
216
+		if (md.type == Type.mailshare) {
217
+			md.utf7FolderPath = md.utf7FolderPath.substring("Dossiers partag&AOk-s/".length(),
218
+					md.utf7FolderPath.length());
219
+		}
220
+		IServiceTopology topology = Topology.get();
221
+
222
+		boolean exist = false;
223
+		ItemValue<Server> backend = topology.datalocation(datalocation);
224
+
225
+		if (backend.uid.equals(topology.core().uid)) {
226
+			exist = directExist(md, partition, imapUid);
227
+		} else {
228
+			INodeClient nc = NodeActivator.get(backend.value.address());
229
+			exist = nodeExist(nc, md, partition, imapUid);
230
+		}
231
+		return exist;
232
+	}
233
+
202 234
 	private InputStream nodeRead(INodeClient nc, MailboxDescriptor md, CyrusPartition partition, long imapUid) {
203 235
 		String path = CyrusFileSystemPathHelper.getFileSystemPath(container.domainUid, md, partition, imapUid);
204 236
 		List<FileDescription> file = nc.listFiles(path);
... ...
@@ -214,6 +246,19 @@ public class BaseMailboxRecordsService implements IChangelogSupport, ICountingSu
214 246
 		return nc.openStream(path);
215 247
 	}
216 248
 
249
+	private boolean nodeExist(INodeClient nc, MailboxDescriptor md, CyrusPartition partition, long imapUid) {
250
+		String path = CyrusFileSystemPathHelper.getFileSystemPath(container.domainUid, md, partition, imapUid);
251
+		List<FileDescription> file = nc.listFiles(path);
252
+		if (file.isEmpty()) {
253
+			path = CyrusFileSystemPathHelper.getHSMFileSystemPath(container.domainUid, md, partition, imapUid);
254
+			file = nc.listFiles(path);
255
+			if (file.isEmpty()) {
256
+				return false;
257
+			}
258
+		}
259
+		return true;
260
+	}
261
+
217 262
 	private InputStream directRead(MailboxDescriptor md, CyrusPartition partition, long imapUid) {
218 263
 		String path = CyrusFileSystemPathHelper.getFileSystemPath(container.domainUid, md, partition, imapUid);
219 264
 		File pathFile = new File(path);
... ...
@@ -234,4 +279,17 @@ public class BaseMailboxRecordsService implements IChangelogSupport, ICountingSu
234 279
 		}
235 280
 	}
236 281
 
282
+	private boolean directExist(MailboxDescriptor md, CyrusPartition partition, long imapUid) {
283
+		String path = CyrusFileSystemPathHelper.getFileSystemPath(container.domainUid, md, partition, imapUid);
284
+		File pathFile = new File(path);
285
+		if (!pathFile.exists()) {
286
+			path = CyrusFileSystemPathHelper.getHSMFileSystemPath(container.domainUid, md, partition, imapUid);
287
+			pathFile = new File(path);
288
+			if (!pathFile.exists()) {
289
+				return false;
290
+			}
291
+		}
292
+		return true;
293
+	}
294
+
237 295
 }
... ...
@@ -213,15 +213,21 @@ public class ImapMailboxRecordsService extends BaseMailboxRecordsService impleme
213 213
 		Set<Long> knownUids = imapUids.stream().map(Long::new).collect(Collectors.toSet());
214 214
 		List<String> allUids = storeService.allUids();
215 215
 		List<ItemValue<MailboxRecord>> extraRecords = new ArrayList<>(allUids.size());
216
+		List<ItemValue<MailboxRecord>> unlinkedRecords = new ArrayList<>(allUids.size());
216 217
 		for (List<String> slice : Lists.partition(allUids, 50)) {
217 218
 			List<ItemValue<MailboxRecord>> records = storeService.getMultiple(slice);
218 219
 			for (ItemValue<MailboxRecord> iv : records) {
219
-				if (!knownUids.contains(iv.value.imapUid) && !iv.flags.contains(ItemFlag.Deleted)) {
220
-					extraRecords.add(iv);
220
+				if (!knownUids.contains(iv.value.imapUid)) {
221
+					if (!checkExistOnBackend(iv.value.imapUid)) {
222
+						unlinkedRecords.add(iv);
223
+					} else if (!iv.flags.contains(ItemFlag.Deleted)) {
224
+						extraRecords.add(iv);
225
+					}
221 226
 				}
222 227
 			}
223 228
 		}
224
-		logger.info("Found {} extra record(s) before resync of {}", extraRecords.size(), imapFolder);
229
+		logger.info("Found {} extra record(s), {} unlinked record(s) before resync of {}", extraRecords.size(),
230
+				unlinkedRecords.size(), imapFolder);
225 231
 		if (!extraRecords.isEmpty()) {
226 232
 			IDbMailboxRecords recsApi = context.provider().instance(IDbMailboxRecords.class,
227 233
 					IMailReplicaUids.uniqueId(container.uid));
... ...
@@ -232,6 +238,11 @@ public class ImapMailboxRecordsService extends BaseMailboxRecordsService impleme
232 238
 			}).collect(Collectors.toList());
233 239
 			recsApi.updates(batch);
234 240
 		}
241
+		if (!unlinkedRecords.isEmpty()) {
242
+			IDbMailboxRecords recsApi = context.provider().instance(IDbMailboxRecords.class,
243
+					IMailReplicaUids.uniqueId(container.uid));
244
+			recsApi.deleteImapUids(unlinkedRecords.stream().map(iv -> iv.value.imapUid).collect(Collectors.toList()));
245
+		}
235 246
 		time = System.currentTimeMillis() - time;
236 247
 		logger.info("{} re-sync completed in {}ms.", imapFolder, time);
237 248
 	}