Browse code

[mail-api] MAINTBL-26 Fix: make EmlBuilder consider encoding, charset & dispositionType from Part objects when crafting messages

Thomas Cataldo authored on 27/11/2019 14:53:31
Showing 10 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,11 @@
1
+<?xml version="1.0" encoding="UTF-8"?>
2
+<classpath>
3
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
4
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
5
+	<classpathentry kind="src" path="src">
6
+		<attributes>
7
+			<attribute name="test" value="true"/>
8
+		</attributes>
9
+	</classpathentry>
10
+	<classpathentry kind="output" path="bin"/>
11
+</classpath>
0 12
new file mode 100644
... ...
@@ -0,0 +1,28 @@
1
+<?xml version="1.0" encoding="UTF-8"?>
2
+<projectDescription>
3
+	<name>net.bluemind.backend.mail.parsing.tests</name>
4
+	<comment></comment>
5
+	<projects>
6
+	</projects>
7
+	<buildSpec>
8
+		<buildCommand>
9
+			<name>org.eclipse.jdt.core.javabuilder</name>
10
+			<arguments>
11
+			</arguments>
12
+		</buildCommand>
13
+		<buildCommand>
14
+			<name>org.eclipse.pde.ManifestBuilder</name>
15
+			<arguments>
16
+			</arguments>
17
+		</buildCommand>
18
+		<buildCommand>
19
+			<name>org.eclipse.pde.SchemaBuilder</name>
20
+			<arguments>
21
+			</arguments>
22
+		</buildCommand>
23
+	</buildSpec>
24
+	<natures>
25
+		<nature>org.eclipse.pde.PluginNature</nature>
26
+		<nature>org.eclipse.jdt.core.javanature</nature>
27
+	</natures>
28
+</projectDescription>
0 29
new file mode 100644
... ...
@@ -0,0 +1,13 @@
1
+Manifest-Version: 1.0
2
+Bundle-ManifestVersion: 2
3
+Bundle-Name: net.bluemind.backend.mail.parsing.tests
4
+Bundle-SymbolicName: net.bluemind.backend.mail.parsing.tests
5
+Bundle-Version: 4.1.0.qualifier
6
+Bundle-Vendor: bluemind.net
7
+Require-Bundle: org.eclipse.core.runtime,
8
+ org.junit,
9
+ net.bluemind.backend.mail.parsing,
10
+ net.bluemind.mime4j.common,
11
+ com.google.guava
12
+Bundle-RequiredExecutionEnvironment: JavaSE-1.8
13
+Automatic-Module-Name: net.bluemind.backend.parsing.tests
0 14
new file mode 100644
... ...
@@ -0,0 +1,4 @@
1
+source.. = src/
2
+output.. = bin/
3
+bin.includes = META-INF/,\
4
+               .
0 5
new file mode 100644
... ...
@@ -0,0 +1,31 @@
1
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3
+	<modelVersion>4.0.0</modelVersion>
4
+	<parent>
5
+		<groupId>net.bluemind</groupId>
6
+		<version>4.1.0-SNAPSHOT</version>
7
+		<artifactId>net.bluemind.mailbackend.plugins</artifactId>
8
+	</parent>
9
+	<artifactId>net.bluemind.backend.mail.parsing.tests</artifactId>
10
+	<packaging>eclipse-test-plugin</packaging>
11
+
12
+	<build>
13
+		<plugins>
14
+			<plugin>
15
+				<groupId>org.eclipse.tycho</groupId>
16
+				<artifactId>target-platform-configuration</artifactId>
17
+				<configuration>
18
+					<dependency-resolution>
19
+						<extraRequirements>
20
+							<requirement>
21
+								<type>eclipse-feature</type>
22
+								<id>net.bluemind.tests.feature</id>
23
+								<versionRange>0.0.0</versionRange>
24
+							</requirement>
25
+						</extraRequirements>
26
+					</dependency-resolution>
27
+				</configuration>
28
+			</plugin>
29
+		</plugins>
30
+	</build>
31
+</project>
0 32
new file mode 100644
... ...
@@ -0,0 +1,134 @@
1
+/* BEGIN LICENSE
2
+ * Copyright © Blue Mind SAS, 2012-2019
3
+ *
4
+ * This file is part of BlueMind. BlueMind is a messaging and collaborative
5
+ * solution.
6
+ *
7
+ * This program is free software; you can redistribute it and/or modify
8
+ * it under the terms of either the GNU Affero General Public License as
9
+ * published by the Free Software Foundation (version 3 of the License).
10
+ *
11
+ * This program is distributed in the hope that it will be useful,
12
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
14
+ *
15
+ * See LICENSE.txt
16
+ * END LICENSE
17
+ */
18
+package net.bluemind.backend.mail.parsing.tests;
19
+
20
+import static org.junit.Assert.assertFalse;
21
+import static org.junit.Assert.assertNotNull;
22
+import static org.junit.Assert.assertTrue;
23
+
24
+import java.io.File;
25
+import java.io.IOException;
26
+import java.io.InputStream;
27
+import java.util.LinkedList;
28
+import java.util.List;
29
+import java.util.UUID;
30
+
31
+import org.apache.james.mime4j.dom.Message;
32
+import org.junit.After;
33
+import org.junit.Test;
34
+
35
+import com.google.common.io.ByteStreams;
36
+import com.google.common.io.Files;
37
+
38
+import net.bluemind.backend.mail.api.DispositionType;
39
+import net.bluemind.backend.mail.api.MessageBody;
40
+import net.bluemind.backend.mail.api.MessageBody.Part;
41
+import net.bluemind.backend.mail.parsing.Bodies;
42
+import net.bluemind.backend.mail.parsing.EmlBuilder;
43
+import net.bluemind.mime4j.common.Mime4JHelper;
44
+
45
+public class EmlBuilderTests {
46
+
47
+	private List<File> clearQueue = new LinkedList<>();
48
+
49
+	@After
50
+	public void after() {
51
+		clearQueue.forEach(File::delete);
52
+		clearQueue.clear();
53
+	}
54
+
55
+	@Test
56
+	public void testBuilderTextPlainUtf8() throws IOException {
57
+		MessageBody mb = new MessageBody();
58
+		mb.subject = "un text plain avec de l'unicod€";
59
+		mb.structure = text("Des caractèr€s accentués", TextStyle.plain);
60
+		Message built = EmlBuilder.of(mb, "john.doe@devenv.blue");
61
+		String eml = toEML(built);
62
+		assertNotNull(eml);
63
+		assertFalse("Euro unicode character must be encoded", eml.contains("€"));
64
+		assertTrue("UTF-8 charset must be explicit in eml", eml.contains("charset=utf-8"));
65
+	}
66
+
67
+	@Test
68
+	public void testBuilderAlternativeAndRelated() throws IOException {
69
+		MessageBody mb = new MessageBody();
70
+		mb.subject = "un mail réaliste";
71
+		Part text = text("Coucou, tu veux...", TextStyle.plain);
72
+
73
+		Part html = text("<b>Coucou, tu veux...</b>\r\n<img src=\"cid:toto_id\"/>", TextStyle.html);
74
+		Part img = new Part();
75
+		img.mime = "image/png";
76
+		img.contentId = "toto_id";
77
+		img.fileName = "zizi coptère.png";
78
+		img.dispositionType = DispositionType.INLINE;
79
+		img.address = genPart(new byte[0]);
80
+		Part related = multipart(MultipartStyle.related, html, img);
81
+		Part alternative = multipart(MultipartStyle.alternative, text, related);
82
+		mb.structure = alternative;
83
+		String eml = toEML(EmlBuilder.of(mb, "john.doe@devenv.blue"));
84
+		assertNotNull(eml);
85
+		assertTrue(eml.contains("inline"));
86
+		assertTrue(eml.contains("<toto_id>"));
87
+		// attach filename encoding
88
+		assertFalse(eml.contains("coptère"));
89
+		assertTrue(eml.contains("zizi"));
90
+	}
91
+
92
+	public enum TextStyle {
93
+		plain, html;
94
+	}
95
+
96
+	public enum MultipartStyle {
97
+		mixed, related, alternative;
98
+	}
99
+
100
+	Part multipart(MultipartStyle multipartStyle, Part... children) {
101
+		Part mpart = new Part();
102
+		mpart.mime = "multipart/" + multipartStyle.name();
103
+		for (Part p : children) {
104
+			mpart.children.add(p);
105
+		}
106
+		return mpart;
107
+	}
108
+
109
+	private Part text(String content, TextStyle markup) throws IOException {
110
+		Part p = new Part();
111
+		p.mime = "text/" + markup.name();
112
+		p.charset = "utf-8";
113
+		p.encoding = "quoted-printable";
114
+		p.address = genPart(content.getBytes());
115
+		return p;
116
+	}
117
+
118
+	private String genPart(byte[] content) throws IOException {
119
+		String addr = UUID.randomUUID().toString();
120
+		File staging = new File(Bodies.STAGING, addr + ".part");
121
+		Files.write(content, staging);
122
+		clearQueue.add(staging);
123
+		return addr;
124
+	}
125
+
126
+	private String toEML(Message built) throws IOException {
127
+		try (InputStream in = Mime4JHelper.asStream(built)) {
128
+			String s = new String(ByteStreams.toByteArray(in));
129
+			System.err.println(s);
130
+			return s;
131
+		}
132
+	}
133
+
134
+}
... ...
@@ -9,13 +9,16 @@ Require-Bundle: org.eclipse.core.runtime,
9 9
  com.google.guava,
10 10
  net.bluemind.core.commons;bundle-version="4.1.0",
11 11
  net.bluemind.backend.mail.api,
12
- net.bluemind.mime4j.common,
13 12
  net.bluemind.config,
14
- net.bluemind.backend.mail.replica.api,
13
+ net.bluemind.backend.mail.replica.api;visibility:=reexport,
15 14
  io.vertx.core,
16 15
  net.bluemind.core.rest,
17 16
  net.bluemind.content.analysis,
18
- org.jsoup
17
+ org.jsoup,
18
+ org.apache.james.apache-mime4j-dom;bundle-version="0.7.3";visibility:=reexport,
19
+ org.apache.james.apache-mime4j-core;bundle-version="0.7.3";visibility:=reexport,
20
+ net.bluemind.mime4j.common;bundle-version="4.1.0",
21
+ javax.mail-api
19 22
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
20 23
 Bundle-ActivationPolicy: lazy
21 24
 Export-Package: net.bluemind.backend.mail.parsing
... ...
@@ -27,4 +27,7 @@ public class Bodies {
27 27
 		STAGING.mkdirs();
28 28
 	}
29 29
 
30
+	private Bodies() {
31
+	}
32
+
30 33
 }
... ...
@@ -21,13 +21,19 @@ import java.io.File;
21 21
 import java.io.FileInputStream;
22 22
 import java.io.IOException;
23 23
 import java.io.InputStream;
24
+import java.io.UnsupportedEncodingException;
24 25
 import java.nio.charset.StandardCharsets;
25 26
 import java.nio.file.Files;
26 27
 import java.nio.file.StandardOpenOption;
27 28
 import java.util.Arrays;
28 29
 import java.util.Date;
30
+import java.util.HashMap;
29 31
 import java.util.LinkedList;
30 32
 import java.util.List;
33
+import java.util.Map;
34
+import java.util.Objects;
35
+
36
+import javax.mail.internet.MimeUtility;
31 37
 
32 38
 import org.apache.james.mime4j.MimeException;
33 39
 import org.apache.james.mime4j.dom.Body;
... ...
@@ -35,8 +41,10 @@ import org.apache.james.mime4j.dom.Header;
35 41
 import org.apache.james.mime4j.dom.Message;
36 42
 import org.apache.james.mime4j.dom.address.Address;
37 43
 import org.apache.james.mime4j.dom.address.Mailbox;
44
+import org.apache.james.mime4j.dom.field.ContentDispositionField;
38 45
 import org.apache.james.mime4j.dom.field.ParsedField;
39 46
 import org.apache.james.mime4j.field.LenientFieldParser;
47
+import org.apache.james.mime4j.message.AbstractEntity;
40 48
 import org.apache.james.mime4j.message.BasicBodyFactory;
41 49
 import org.apache.james.mime4j.message.BodyPart;
42 50
 import org.apache.james.mime4j.message.MessageImpl;
... ...
@@ -46,6 +54,7 @@ import org.slf4j.LoggerFactory;
46 54
 
47 55
 import com.google.common.base.Throwables;
48 56
 
57
+import net.bluemind.backend.mail.api.DispositionType;
49 58
 import net.bluemind.backend.mail.api.MessageBody;
50 59
 import net.bluemind.backend.mail.api.MessageBody.Part;
51 60
 import net.bluemind.backend.mail.api.MessageBody.Recipient;
... ...
@@ -59,12 +68,15 @@ public class EmlBuilder {
59 68
 
60 69
 	private static final Logger logger = LoggerFactory.getLogger(EmlBuilder.class);
61 70
 
71
+	private EmlBuilder() {
72
+	}
73
+
62 74
 	public static Message of(MessageBody mb, String owner) {
63 75
 
64 76
 		MessageImpl msg = new MessageImpl();
65 77
 		msg.setDate(mb.date);
66 78
 		BasicBodyFactory bbf = new BasicBodyFactory();
67
-		logger.info("******************* Building with subject {}", mb.subject);
79
+		logger.info("Subject is '{}'", mb.subject);
68 80
 		msg.setSubject(mb.subject);
69 81
 		try {
70 82
 			fillHeader(msg.getHeader(), mb.headers);
... ...
@@ -79,7 +91,7 @@ public class EmlBuilder {
79 91
 			if (body instanceof MultipartImpl) {
80 92
 				msg.setMultipart((MultipartImpl) body);
81 93
 			} else {
82
-				msg.setBody(body, structure.mime);
94
+				setBody(msg, body, structure);
83 95
 			}
84 96
 
85 97
 		} catch (IOException e) {
... ...
@@ -133,6 +145,40 @@ public class EmlBuilder {
133 145
 		}
134 146
 	}
135 147
 
148
+	private static void setBody(AbstractEntity ae, Body b, Part p) {
149
+		Map<String, String> bodyParams = new HashMap<>();
150
+		if (p.charset != null) {
151
+			bodyParams.put("charset", p.charset);
152
+		}
153
+		ae.setBody(b, p.mime, bodyParams);
154
+		if (p.encoding != null) {
155
+			ae.setContentTransferEncoding(p.encoding);
156
+		}
157
+		if (p.fileName != null && p.dispositionType == DispositionType.ATTACHMENT) {
158
+			ae.setContentDisposition(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT, safeEncode(p.fileName));
159
+		} else if (p.dispositionType == DispositionType.INLINE && p.contentId != null) {
160
+			if (p.fileName == null) {
161
+				ae.setContentDisposition(ContentDispositionField.DISPOSITION_TYPE_INLINE);
162
+			} else {
163
+				ae.setContentDisposition(ContentDispositionField.DISPOSITION_TYPE_INLINE, safeEncode(p.fileName));
164
+			}
165
+			try {
166
+				ae.getHeader().addField(LenientFieldParser.parse("Content-ID: <" + p.contentId + ">"));
167
+			} catch (MimeException e) {
168
+				logger.warn("Failed to set content-id to {}: {}", p.contentId, e.getMessage());
169
+			}
170
+		}
171
+	}
172
+
173
+	private static String safeEncode(String s) {
174
+		try {
175
+			return MimeUtility.encodeWord(s, "utf-8", "Q");
176
+		} catch (UnsupportedEncodingException e) {
177
+			// should not happen as utf-8 is always available
178
+			return s;
179
+		}
180
+	}
181
+
136 182
 	private static Body createBody(BasicBodyFactory bbf, Part structure, String owner) throws IOException {
137 183
 		Body body = null;
138 184
 		if (structure.children.isEmpty()) {
... ...
@@ -154,7 +200,7 @@ public class EmlBuilder {
154 200
 				if (childBody instanceof MultipartImpl) {
155 201
 					bp.setMultipart((MultipartImpl) childBody);
156 202
 				} else {
157
-					bp.setBody(childBody, p.mime);
203
+					setBody(bp, childBody, p);
158 204
 				}
159 205
 				Header partHeader = bp.getHeader();
160 206
 				try {
... ...
@@ -172,8 +218,8 @@ public class EmlBuilder {
172 218
 	private static void fillHeader(Header partHeader, List<net.bluemind.backend.mail.api.MessageBody.Header> headers)
173 219
 			throws MimeException {
174 220
 		for (net.bluemind.backend.mail.api.MessageBody.Header h : headers) {
175
-			if (h.name.equals("Content-Type")) {
176
-				// remove previous one, only one "Content-Type" allowed
221
+			if (h.name.equals("Content-Type") || h.name.equals("Content-Transfer-Encoding")) {
222
+				// remove previous one, only one "Content-Type" or "Content-Transfer-Encoding" allowed
177 223
 				partHeader.removeFields(h.name);
178 224
 			}
179 225
 			if (h.values.size() == 1) {
... ...
@@ -186,6 +232,7 @@ public class EmlBuilder {
186 232
 	}
187 233
 
188 234
 	public static SizedStream inputStream(Long id, String previousBody, Date date, Part structure, String owner) {
235
+		Objects.requireNonNull(structure.address, "Part address must not be null");
189 236
 		File emlInput = new File(Bodies.STAGING, structure.address + ".part");
190 237
 		if (!emlInput.exists()) {
191 238
 			throw ServerFault.notFound("Missing staging file " + emlInput.getAbsolutePath());
... ...
@@ -35,6 +35,7 @@
35 35
 		<module>net.bluemind.backend.mail.replica.service</module>
36 36
 		<module>net.bluemind.backend.mail.replica.indexing</module>
37 37
 		<module>net.bluemind.backend.mail.parsing</module>
38
+		<module>net.bluemind.backend.mail.parsing.tests</module>
38 39
 		<module>net.bluemind.backend.mail.replica.service.tests</module>
39 40
 		<module>net.bluemind.backend.mail.replica.sqlschema</module>
40 41
 		<module>net.bluemind.backend.cyrus.replication.testhelper</module>