add emoji auto completion (suggestion)

- typing `<:` will start the emoji auto complition inside the plantuml editor
- for the sake of simplicity the emoji preview of the completion documentation will fetch the image from the original github repository (not plantuml). The reason is that the images (SVGs) inside plantuml have sometimes removed their svg tag, hence it's difficult to set the correct rendering size.
- expand auto completion (suggestion) documentation by default
- add emoji example GIF and documentation
- set charset to utf-8 for each website
- refactor JSON creation inside UI Helper

docs/WebUI/language-features.md

@@ -6,13 +6,23 @@
 
 - type `<&` to get a list of PlantUML available icons
 - see a preview of the suggested icon in its description
+- [PlantUML documentation](https://plantuml.com/openiconic)
 
 ![icons](https://raw.githubusercontent.com/plantuml/plantuml-server/master/docs/WebUI/gifs/auto-completion-icons.gif)
 
+### Emojis
+
+- type `<:` to get a list of PlantUML available icons
+- see a preview of the suggested icon in its description
+- [PlantUML documentation](https://plantuml.com/creole#68305e25f5788db0)
+
+![emojis](https://raw.githubusercontent.com/plantuml/plantuml-server/master/docs/WebUI/gifs/auto-completion-emojis.gif)
+
 ### Themes
 
 - type `!t` to get the suggestion `theme`
 - type `!theme ` to get a list of (local) available PlantUML themes.
+- [PlantUML documentation](https://plantuml.com/theme)
 
 ![themes](https://raw.githubusercontent.com/plantuml/plantuml-server/master/docs/WebUI/gifs/auto-completion-themes.gif)
 

src/main/java/net/sourceforge/plantuml/servlet/PlantUmlUIHelperServlet.java

@@ -30,12 +30,11 @@ import java.io.InputStreamReader;
 import java.io.StringWriter;
 import java.io.Writer;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
+import javax.xml.XMLConstants;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
@@ -50,7 +49,10 @@ import jakarta.servlet.ServletException;
 import jakarta.servlet.http.HttpServlet;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
-
+import net.sourceforge.plantuml.FileFormat;
+import net.sourceforge.plantuml.emoji.data.Dummy;
+import net.sourceforge.plantuml.json.Json;
+import net.sourceforge.plantuml.json.JsonArray;
 import net.sourceforge.plantuml.theme.ThemeUtils;
 import net.sourceforge.plantuml.openiconic.data.DummyIcon;
 
@@ -73,6 +75,7 @@ public class PlantUmlUIHelperServlet extends HttpServlet {
 
     public PlantUmlUIHelperServlet() {
         // add all supported request items/helper methods
+        helpers.put("emojis", this::sendEmojis);
         helpers.put("icons.svg", this::sendIconsSprite);
         helpers.put("icons", this::sendIcons);
         helpers.put("themes", this::sendThemes);
@@ -91,8 +94,7 @@ public class PlantUmlUIHelperServlet extends HttpServlet {
             errorMsg = "Unknown requested item: " + requestItem;
         }
         if (errorMsg != null) {
-            response.addHeader("Access-Control-Allow-Origin", "*");
+            setDefaultHeader(response, FileFormat.UTXT);
-            response.setContentType("text/plain;charset=UTF-8");
             response.getWriter().write(errorMsg);
             response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
             return;
@@ -101,29 +103,36 @@ public class PlantUmlUIHelperServlet extends HttpServlet {
         requestHelper.accept(request, response);
     }
 
-    private String toJson(List<String> list) {
+    private void setDefaultHeader(HttpServletResponse response, FileFormat fileFormat) {
-        return "[" + list.stream()
+        setDefaultHeader(response, fileFormat.getMimeType());
-            .map(item -> "\"" + item.replace("\"", "\\\"") + "\"")
+    }
-            .collect(Collectors.joining(",")) + "]";
+
+    private HttpServletResponse setDefaultHeader(HttpServletResponse response, String contentType) {
+        response.addHeader("Access-Control-Allow-Origin", "*");
+        response.setContentType(contentType);
+        return response;
     }
 
     private void sendJson(HttpServletResponse response, String json) throws IOException {
-        response.addHeader("Access-Control-Allow-Origin", "*");
+        setDefaultHeader(response, "application/json;charset=UTF-8");
-        response.setContentType("application/json;charset=UTF-8");
         response.getWriter().write(json);
     }
 
-    private List<String> getIcons() throws IOException {
+    private String[] getIcons() throws IOException {
         InputStream in = DummyIcon.class.getResourceAsStream("all.txt");
         try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) {
-            return br.lines().collect(Collectors.toList());
+            return br.lines().toArray(String[]::new);
         }
     }
 
+    private void sendIcons(HttpServletRequest request, HttpServletResponse response) throws IOException {
+        sendJson(response, Json.array(getIcons()).toString());
+    }
+
     private void sendIconsSprite(HttpServletRequest request, HttpServletResponse response) throws IOException {
         if (svgIconsSpriteCache == null) {
             // NOTE: all icons has the following svg tag attributes: width="8" height="8" viewBox="0 0 8 8"
-            List<String> iconNames = getIcons();
+            String[] iconNames = getIcons();
             StringBuilder sprite = new StringBuilder();
             sprite.append("<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"8\" height=\"8\" viewBox=\"0 0 8 8\">\n");
             sprite.append("<defs>\n");
@@ -134,12 +143,19 @@ public class PlantUmlUIHelperServlet extends HttpServlet {
             sprite.append("</defs>\n");
             for (String name : iconNames) {
                 try (InputStream in = DummyIcon.class.getResourceAsStream(name + ".svg")) {
-                    DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+                    DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+                    docFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+                    docFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+                    DocumentBuilder db = docFactory.newDocumentBuilder();
                     Document doc = db.parse(in);
+
                     Writer out = new StringWriter();
                     out.write("<g class=\"sprite\" id=\"" + name + "\">");
 
-                    Transformer tf = TransformerFactory.newInstance().newTransformer();
+                    TransformerFactory tfFactory = TransformerFactory.newInstance();
+                    tfFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+                    tfFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+                    Transformer tf = tfFactory.newTransformer();
                     tf.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
                     tf.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
                     tf.setOutputProperty(OutputKeys.INDENT, "no");
@@ -153,22 +169,34 @@ public class PlantUmlUIHelperServlet extends HttpServlet {
                 } catch (ParserConfigurationException | SAXException | TransformerException ex) {
                     // skip icons which can not be parsed/read
                     Logger logger = Logger.getLogger("com.plantuml");
-                    logger.log(Level.WARNING, "SVG icon '{0}' could not be parsed. Skip!", name);
+                    logger.log(Level.WARNING, "SVG icon \"{0}\" could not be parsed. Skip!", name);
                 }
             }
             sprite.append("</svg>\n");
             svgIconsSpriteCache = sprite.toString();
         }
-        response.addHeader("Access-Control-Allow-Origin", "*");
+        setDefaultHeader(response, FileFormat.SVG);
-        response.setContentType("image/svg+xml;charset=UTF-8");
         response.getWriter().write(svgIconsSpriteCache);
     }
 
-    private void sendIcons(HttpServletRequest request, HttpServletResponse response) throws IOException {
+    private String[][] getEmojis() throws IOException {
-        sendJson(response, toJson(getIcons()));
+        InputStream in = Dummy.class.getResourceAsStream("emoji.txt");
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) {
+            return br.lines().map(line -> line.split(";")).toArray(String[][]::new);
+        }
+    }
+
+    private void sendEmojis(HttpServletRequest request, HttpServletResponse response) throws IOException {
+        String[][] emojis = getEmojis();
+        JsonArray json = new JsonArray();
+        for (String[] emojiUnicodeNamePair : emojis) {
+            json.add(Json.array(emojiUnicodeNamePair));
+        }
+        sendJson(response, json.toString());
     }
 
     private void sendThemes(HttpServletRequest request, HttpServletResponse response) throws IOException {
-        sendJson(response, toJson(ThemeUtils.getAllThemeNames()));
+        String[] themes = ThemeUtils.getAllThemeNames().toArray(new String[0]);
+        sendJson(response, Json.array(themes).toString());
     }
 }

src/main/webapp/plantuml.css

@@ -153,9 +153,10 @@ select {
 #monaco-editor {
   height: 100%;
 }
-/* Hack to display the icons in the icon auto completion documentation in a visible size.
+/* Hack to display the icons and emojis in the auto completion documentation in a visible size.
- * (see PlantUmlLanguageFeatures.registerIconCompletion) */
+ * (see PlantUmlLanguageFeatures.register{Icon,Emoji}Completion) */
-#monaco-editor .overlayWidgets .suggest-details p img[alt="icon"] {
+#monaco-editor .overlayWidgets .suggest-details p img[alt="icon"],
+#monaco-editor .overlayWidgets .suggest-details p img[alt="emoji"] {
   height: 1.2rem;
 }
 

src/main/webapp/plantuml.js

@@ -549,10 +549,21 @@ function initializeCodeEditor() {
       }, document.appConfig.editorWatcherTimeout);
     }
   });
+  // create storage service to expand suggestion documentation by default
+  const storageService = {
+    get() {},
+    getBoolean(key) { return key === 'expandSuggestionDocs'; },
+    getNumber() { return 0; },
+    remove() {},
+    store() {},
+    onWillSaveState() {},
+    onDidChangeStorage() {},
+    onDidChangeValue() {},
+  };
   // create editor
   document.editor = monaco.editor.create(document.getElementById("monaco-editor"), {
     model, ...document.appConfig.editorCreateOptions
-  });
+  }, { storageService });
   // sometimes the monaco editor has resize problems
   document.addEventListener("resize", () => document.editor.layout());
 }

src/main/webapp/plantumllanguage.js

@@ -284,6 +284,48 @@ const PlantUmlLanguageFeatures = (function() {
       });
     };
 
+    this.registerEmojiCompletion = () => {
+      const createEmojiProposals = async (range, filter = undefined) => {
+        const emojis = await this.getEmojis();
+        return emojis?.filter(([unicode, name]) => filter ? unicode.includes(filter) || name?.includes(filter) : true)
+          .map(([unicode, name]) => {
+            // NOTE: load images direct from GitHub source: https://github.com/twitter/twemoji#download
+            const emojiUrl = "https://raw.githubusercontent.com/twitter/twemoji/gh-pages/v/13.1.0/svg/" + unicode + ".svg";
+            const docHint = (name) ? name + " (" + unicode + ")" : unicode;
+            const isUnicode = !name || (filter && unicode.includes(filter));
+            const label = isUnicode ? unicode : name;
+            return {
+              label: label,
+              kind: monaco.languages.CompletionItemKind.Constant,
+              documentation: {
+                //supportHtml: true,  // also a possibility but quite limited html
+                value: "![emoji](" + emojiUrl + ")   " + docHint
+              },
+              insertText: label + ":>",
+              range: range
+            };
+          }) || [];
+      };
+
+      monaco.languages.registerCompletionItemProvider(languageSelector, {
+        triggerCharacters: [":"],
+        provideCompletionItems: async (model, position) => {
+          const textUntilPosition = model.getValueInRange({
+            startLineNumber: position.lineNumber,
+            startColumn: 1,
+            endLineNumber: position.lineNumber,
+            endColumn: position.column,
+          });
+          const match = textUntilPosition.match(/<:(?<emoji>[^\s>]*)$/);
+          if (match) {
+            const suggestions = await createEmojiProposals(getWordRange(model, position), match.groups.emoji);
+            return { suggestions };
+          }
+          return { suggestions: [] };
+        }
+      });
+    };
+
 
     // ==========================================================================================================
     // == helper methods ==
@@ -308,6 +350,16 @@ const PlantUmlLanguageFeatures = (function() {
       }
     })();
 
+    this.getEmojis = (function(){
+      let emojis = undefined;
+      return async () => {
+        if (emojis === undefined) {
+          emojis = await makeRequest("GET", "ui-helper?request=emojis", { responseType: "json" });
+        }
+        return emojis;
+      }
+    })();
+
     this.getThemes = (function(){
       let themes = undefined;
       return async () => {
@@ -377,6 +429,7 @@ const PlantUmlLanguageFeatures = (function() {
       this.addStartEndValidationListeners();
       this.registerThemeCompletion();
       this.registerIconCompletion();
+      this.registerEmojiCompletion();
     }
   }
 

src/main/webapp/resource/htmlheadbase.jsp

@@ -1,4 +1,5 @@
 <base href="<%= request.getContextPath() %>/" />
+<meta charset="utf-8">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta http-equiv="expires" content="0" />
 <meta http-equiv="pragma" content="no-cache" />