View Javadoc
1   /*
2    * This file is part of ***  M y C o R e  ***
3    * See http://www.mycore.de/ for details.
4    *
5    * MyCoRe is free software: you can redistribute it and/or modify
6    * it under the terms of the GNU General Public License as published by
7    * the Free Software Foundation, either version 3 of the License, or
8    * (at your option) any later version.
9    *
10   * MyCoRe is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU General Public License for more details.
14   *
15   * You should have received a copy of the GNU General Public License
16   * along with MyCoRe.  If not, see <http://www.gnu.org/licenses/>.
17   */
18  
19  package org.mycore.solr.schema;
20  
21  import static java.util.Map.entry;
22  
23  import java.io.IOException;
24  import java.io.UnsupportedEncodingException;
25  import java.nio.charset.StandardCharsets;
26  import java.util.List;
27  import java.util.Locale;
28  import java.util.Map;
29  import java.util.Map.Entry;
30  import java.util.Set;
31  import java.util.stream.Collectors;
32  import java.util.stream.Stream;
33  
34  import org.apache.commons.lang3.StringUtils;
35  import org.apache.http.HttpResponse;
36  import org.apache.http.HttpStatus;
37  import org.apache.http.StatusLine;
38  import org.apache.http.client.methods.CloseableHttpResponse;
39  import org.apache.http.client.methods.HttpGet;
40  import org.apache.http.client.methods.HttpPost;
41  import org.apache.http.entity.StringEntity;
42  import org.apache.http.impl.client.CloseableHttpClient;
43  import org.apache.http.impl.client.HttpClients;
44  import org.apache.http.util.EntityUtils;
45  import org.apache.logging.log4j.LogManager;
46  import org.apache.logging.log4j.Logger;
47  import org.mycore.common.config.MCRConfiguration2;
48  import org.mycore.common.config.MCRConfigurationInputStream;
49  import org.mycore.solr.MCRSolrClientFactory;
50  import org.mycore.solr.MCRSolrCore;
51  import org.mycore.solr.MCRSolrUtils;
52  
53  import com.google.common.io.ByteStreams;
54  import com.google.gson.Gson;
55  import com.google.gson.JsonArray;
56  import com.google.gson.JsonElement;
57  import com.google.gson.JsonObject;
58  import com.google.gson.JsonParser;
59  import com.google.gson.JsonSyntaxException;
60  
61  /**
62   * This class provides methods to reload a SOLR configuration using the SOLR configuration API
63   * see https://lucene.apache.org/solr/guide/8_6/config-api.html
64   *
65   * @author Robert Stephan
66   * @author Jens Kupferschmidt
67   */
68  public class MCRSolrConfigReloader {
69  
70      /**
71      from https://lucene.apache.org/solr/guide/8_6/config-api.html
72      key = lowercase object name from config api command (add-*, update-* delete-*)
73      value = keyName in SOLR config json (retrieved via URL ../core-name/config)
74      */
75      private static final Map<String, String> SOLR_CONFIG_OBJECT_NAMES = Map.ofEntries(
76          entry("requesthandler", "requestHandler"), //checked -> id in subfield "name"
77          entry("searchcomponent", "searchComponent"), //checked  -> id in subfield "name"
78          entry("initparams", "initParams"), //checked - id in key (TODO special handling)
79          entry("queryresponsewriter", "queryResponseWriter"), //checked  -> id in subfield "name"
80          entry("queryparser", "queryParser"),
81          entry("valuesourceparser", "valueSourceParser"),
82          entry("transformer", "transformer"),
83          entry("updateprocessor", "updateProcessor"), //checked  -> id in subfield "name"
84          entry("queryconverter", "queryConverter"),
85          entry("listener", "listener"), //checked -> id in subfield "event" -> special handling
86          entry("runtimelib", "runtimeLib"));
87  
88      private static final List<String> SOLR_CONFIG_PROPERTY_COMMANDS = List.of("set-property", "unset-property");
89  
90      private static final Logger LOGGER = LogManager.getLogger();
91  
92      private static final String SOLR_CONFIG_UPDATE_FILE_NAME = "solr-config.json";
93  
94      /**
95       * Removed items from SOLR configuration overlay. This removal works over all in the property
96       * MCR.Solr.ObserverConfigTypes defined SOLR configuration parts. For each entry the
97       * method will process a SOLR delete command via API.
98       *
99       * @param configType the name of the configuration directory containing the SOLR core configuration
100      * @param coreID the ID of the core, which the configuration should be applied to
101      */
102     public static void reset(String configType, String coreID) {
103         LOGGER.info(() -> "Resetting config definitions for core " + coreID + " using configuration " + configType);
104         String coreURL = MCRSolrClientFactory.get(coreID)
105             .map(MCRSolrCore::getV1CoreURL)
106             .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID));
107         JsonObject currentSolrConfig = retrieveCurrentSolrConfigOverlay(coreURL);
108         JsonObject configPart = currentSolrConfig.getAsJsonObject("overlay");
109 
110         for (String observedType : getObserverConfigTypes()) {
111             JsonObject overlaySection = configPart.getAsJsonObject(observedType);
112             if (overlaySection == null) {
113                 continue;
114             }
115             Set<Map.Entry<String, JsonElement>> configuredComponents = overlaySection.entrySet();
116             final String deleteSectionCommand = "delete-" + observedType.toLowerCase(Locale.ROOT);
117             if (configuredComponents.isEmpty() || !isKnownSolrConfigCommmand(deleteSectionCommand)) {
118                 continue;
119             }
120             for (Map.Entry<String, JsonElement> configuredComponent : configuredComponents) {
121                 final JsonObject deleteCommand = new JsonObject();
122                 deleteCommand.addProperty(deleteSectionCommand, configuredComponent.getKey());
123                 LOGGER.debug(deleteCommand);
124                 try {
125                     executeSolrCommand(coreURL, deleteCommand);
126                 } catch (IOException e) {
127                     LOGGER.error(() -> "Exception while executing '" + deleteCommand + "'.", e);
128                 }
129             }
130         }
131     }
132 
133     /**
134      * This method modified the SOLR configuration definition based on all solr/{coreType}/solr-config.json 
135      * in the MyCoRe-Maven modules resource path.
136      *
137      * @param configType the name of the configuration directory containing the SOLR core configuration
138      * @param coreID the ID of the core, which the configuration should be applied to
139      */
140     public static void processConfigFiles(String configType, String coreID) {
141         LOGGER.info(() -> "Load config definitions for core " + coreID + " using configuration " + configType);
142         try {
143             String coreURL = MCRSolrClientFactory.get(coreID)
144                 .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID)).getV1CoreURL();
145             List<String> observedTypes = getObserverConfigTypes();
146             JsonObject currentSolrConfig = retrieveCurrentSolrConfig(coreURL);
147 
148             List<byte[]> configFileContents = MCRConfigurationInputStream.getConfigFileContents(
149                 "solr/" + configType + "/" + SOLR_CONFIG_UPDATE_FILE_NAME);
150             for (byte[] configFileData : configFileContents) {
151                 String content = new String(configFileData, StandardCharsets.UTF_8);
152                 JsonElement json = JsonParser.parseString(content);
153                 if (!json.isJsonArray()) {
154                     JsonElement e = json;
155                     json = new JsonArray();
156                     json.getAsJsonArray().add(e);
157                 }
158 
159                 for (JsonElement command : json.getAsJsonArray()) {
160                     LOGGER.debug(command);
161                     processConfigCommand(coreURL, command, currentSolrConfig, observedTypes);
162                 }
163             }
164         } catch (IOException e) {
165             LOGGER.error(e);
166         }
167     }
168 
169     /**
170      * get the content of property MCR.Solr.ObserverConfigTypes as List
171      * @return the list of observed SOLR configuration types, a.k.a. top-level sections of config API
172      */
173     private static List<String> getObserverConfigTypes() {
174         return MCRConfiguration2
175             .getString("MCR.Solr.ObserverConfigTypes")
176             .map(MCRConfiguration2::splitValue)
177             .orElseGet(Stream::empty)
178             .collect(Collectors.toList());
179     }
180 
181     /**
182      * processes a single SOLR configuration command
183      * @param coreURL - the URL of the core
184      * @param command - the command in JSON syntax
185      */
186     private static void processConfigCommand(String coreURL, JsonElement command, JsonObject currentSolrConfig,
187         List<String> observedTypes) {
188         if (command.isJsonObject()) {
189             try {
190                 //get first and only? property of the command object
191                 final JsonObject commandJsonObject = command.getAsJsonObject();
192                 Entry<String, JsonElement> commandObject = commandJsonObject.entrySet().iterator().next();
193                 final String configCommand = commandObject.getKey();
194                 final String configType = StringUtils.substringAfter(configCommand, "add-");
195 
196                 if (isKnownSolrConfigCommmand(configCommand)) {
197 
198                     if (observedTypes.contains(configType) && configCommand.startsWith("add-") &&
199                         commandObject.getValue() instanceof JsonObject) {
200                         final JsonElement configCommandName = commandObject.getValue().getAsJsonObject().get("name");
201                         if (isConfigTypeAlreadyAdded(configType, configCommandName, currentSolrConfig)) {
202                             LOGGER.info(() -> "Current configuration has already an " + configCommand
203                                 + " with name " + configCommandName.getAsString()
204                                 + ". Rewrite config command as update-" + configType);
205                             commandJsonObject.add("update-" + configType, commandJsonObject.get(configCommand));
206                             commandJsonObject.remove(configCommand);
207                         }
208                     }
209                     executeSolrCommand(coreURL, commandJsonObject);
210                 }
211             } catch (IOException e) {
212                 LOGGER.error(e);
213             }
214         }
215 
216     }
217 
218     /**
219      * Sends a command to SOLR server
220      * @param coreURL to which the command will be send
221      * @param command the command
222      * @throws UnsupportedEncodingException if command encoding is not supported
223      */
224     private static void executeSolrCommand(String coreURL, JsonObject command) throws UnsupportedEncodingException {
225         HttpPost post = new HttpPost(coreURL + "/config");
226         post.setHeader("Content-type", "application/json");
227         post.setEntity(new StringEntity(command.toString()));
228         String commandprefix = command.keySet().stream().findFirst().orElse("unknown command");
229         HttpResponse response;
230         try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
231             response = httpClient.execute(post);
232             String respContent = new String(ByteStreams.toByteArray(response.getEntity().getContent()),
233                 StandardCharsets.UTF_8);
234             if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
235                 LOGGER.debug(() -> "SOLR config " + commandprefix + " command was successful \n" + respContent);
236             } else {
237                 LOGGER
238                     .error(() -> "SOLR config " + commandprefix + " error: " + response.getStatusLine().getStatusCode()
239                         + " " + response.getStatusLine().getReasonPhrase() + "\n" + respContent);
240             }
241 
242         } catch (IOException e) {
243             LOGGER.error(() -> "Could not execute the following Solr config command:\n" + command, e);
244         }
245     }
246 
247     /**
248      * Checks if a given configType with passed name already was added to current SORL configuration
249      * @param configType - Type of localized SOLR component e.g. request handlers, search components
250      * @param name - identification of configuration type
251      * @param solrConfig - current SOLR configuration
252      * @return - Is there already an entry in current SOLR configuration
253      */
254     private static boolean isConfigTypeAlreadyAdded(String configType, JsonElement name, JsonObject solrConfig) {
255 
256         JsonObject configPart = solrConfig.getAsJsonObject("config");
257         JsonObject observedConfig = configPart.getAsJsonObject(configType);
258 
259         return observedConfig.has(name.getAsString());
260     }
261 
262     /**
263      * retrieves the current SOLR configuration for the given core 
264      * @param coreURL from which the current SOLR configuration will be load
265      * @return the configuration as JSON object
266      */
267     private static JsonObject retrieveCurrentSolrConfig(String coreURL) {
268         HttpGet getConfig = new HttpGet(coreURL + "/config");
269         return getJSON(getConfig);
270     }
271 
272     /**
273      * retrieves the current SOLR configuration overlay for the given core
274      * @param coreURL from which the current SOLR configuration will be load
275      * @return the configuration as JSON object
276      */
277     private static JsonObject retrieveCurrentSolrConfigOverlay(String coreURL) {
278         HttpGet getConfig = new HttpGet(coreURL + "/config/overlay");
279         return getJSON(getConfig);
280     }
281 
282     private static JsonObject getJSON(HttpGet getConfig) {
283         JsonObject convertedObject = null;
284         try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
285             CloseableHttpResponse response = httpClient.execute(getConfig);
286             StatusLine statusLine = response.getStatusLine();
287             if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
288                 String configAsString = EntityUtils.toString(response.getEntity(), "UTF-8");
289                 convertedObject = new Gson().fromJson(configAsString, JsonObject.class);
290             } else {
291                 LOGGER.error(() -> "Could not retrieve current Solr configuration from solr server. Http Status: "
292                     + statusLine.getStatusCode() + " " + statusLine.getReasonPhrase());
293             }
294 
295         } catch (IOException e) {
296             LOGGER.error("Could not read current Solr configuration", e);
297         } catch (JsonSyntaxException e) {
298             LOGGER.error("Current json configuration is not a valid json", e);
299         }
300         return convertedObject;
301     }
302 
303     /**
304      *
305      * @param cmd the SOLR API command
306      * @return true, if the command is in the list of known SOLR commands.
307      */
308 
309     private static boolean isKnownSolrConfigCommmand(String cmd) {
310         String cfgObjName = cmd.substring(cmd.indexOf("-") + 1).toLowerCase(Locale.ROOT);
311         return ((cmd.startsWith("add-") || cmd.startsWith("update-") || cmd.startsWith("delete-"))
312             && (SOLR_CONFIG_OBJECT_NAMES.containsKey(cfgObjName)))
313             || SOLR_CONFIG_PROPERTY_COMMANDS.contains(cmd);
314     }
315 
316 }