View Javadoc
1   /*
2    * This file is part of ***  M y C o R e  ***
3    * See http://www.mycore.de/ for details.
4    *
5    * MyCoRe is free software: you can redistribute it and/or modify
6    * it under the terms of the GNU General Public License as published by
7    * the Free Software Foundation, either version 3 of the License, or
8    * (at your option) any later version.
9    *
10   * MyCoRe is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU General Public License for more details.
14   *
15   * You should have received a copy of the GNU General Public License
16   * along with MyCoRe.  If not, see <http://www.gnu.org/licenses/>.
17   */
18  
19  package org.mycore.migration.cli;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.net.URISyntaxException;
25  import java.nio.file.Files;
26  import java.nio.file.StandardOpenOption;
27  import java.util.ArrayList;
28  import java.util.Collection;
29  import java.util.Date;
30  import java.util.List;
31  import java.util.TreeSet;
32  import java.util.stream.Collectors;
33  
34  import org.apache.logging.log4j.LogManager;
35  import org.apache.logging.log4j.Logger;
36  import org.jdom2.Document;
37  import org.jdom2.Element;
38  import org.jdom2.JDOMException;
39  import org.jdom2.filter.Filters;
40  import org.jdom2.output.Format;
41  import org.jdom2.output.XMLOutputter;
42  import org.jdom2.xpath.XPathExpression;
43  import org.jdom2.xpath.XPathFactory;
44  import org.mycore.access.MCRAccessException;
45  import org.mycore.backend.jpa.MCREntityManagerProvider;
46  import org.mycore.backend.jpa.links.MCRLINKHREF;
47  import org.mycore.backend.jpa.links.MCRLINKHREFPK_;
48  import org.mycore.backend.jpa.links.MCRLINKHREF_;
49  import org.mycore.common.MCRConstants;
50  import org.mycore.common.MCRException;
51  import org.mycore.common.MCRPersistenceException;
52  import org.mycore.common.MCRSessionMgr;
53  import org.mycore.common.content.MCRContent;
54  import org.mycore.common.content.MCRStreamContent;
55  import org.mycore.common.content.transformer.MCRXSLTransformer;
56  import org.mycore.common.xml.MCRXMLFunctions;
57  import org.mycore.datamodel.common.MCRAbstractMetadataVersion;
58  import org.mycore.datamodel.common.MCRActiveLinkException;
59  import org.mycore.datamodel.common.MCRLinkTableManager;
60  import org.mycore.datamodel.common.MCRXMLMetadataManager;
61  import org.mycore.datamodel.metadata.MCRBase;
62  import org.mycore.datamodel.metadata.MCRDerivate;
63  import org.mycore.datamodel.metadata.MCRMetaDerivateLink;
64  import org.mycore.datamodel.metadata.MCRMetaEnrichedLinkID;
65  import org.mycore.datamodel.metadata.MCRMetaLangText;
66  import org.mycore.datamodel.metadata.MCRMetaLinkID;
67  import org.mycore.datamodel.metadata.MCRMetadataManager;
68  import org.mycore.datamodel.metadata.MCRObject;
69  import org.mycore.datamodel.metadata.MCRObjectID;
70  import org.mycore.datamodel.metadata.MCRObjectService;
71  import org.mycore.datamodel.metadata.MCRObjectStructure;
72  import org.mycore.datamodel.niofs.MCRPath;
73  import org.mycore.frontend.cli.annotation.MCRCommand;
74  import org.mycore.frontend.cli.annotation.MCRCommandGroup;
75  import org.mycore.iview2.services.MCRTileJob;
76  import org.xml.sax.SAXException;
77  
78  import jakarta.persistence.EntityManager;
79  import jakarta.persistence.TypedQuery;
80  import jakarta.persistence.criteria.CriteriaBuilder;
81  import jakarta.persistence.criteria.CriteriaQuery;
82  import jakarta.persistence.criteria.Root;
83  
84  /**
85   * @author Thomas Scheffler (yagee)
86   */
87  @MCRCommandGroup(name = "MyCoRe migration")
88  public class MCRMigrationCommands {
89  
90      private static final Logger LOGGER = LogManager.getLogger();
91  
92      @MCRCommand(syntax = "migrate author servflags",
93          help = "Create missing servflags for createdby and modifiedby. (MCR-786)",
94          order = 20)
95      public static List<String> addServFlags() {
96          TreeSet<String> ids = new TreeSet<>(MCRXMLMetadataManager.instance().listIDs());
97          ArrayList<String> cmds = new ArrayList<>(ids.size());
98          for (String id : ids) {
99              cmds.add("migrate author servflags for " + id);
100         }
101         return cmds;
102     }
103 
104     @MCRCommand(syntax = "migrate author servflags for {0}",
105         help = "Create missing servflags for createdby and modifiedby for object {0}. (MCR-786)",
106         order = 10)
107     public static void addServFlags(String id)
108         throws IOException, MCRPersistenceException, MCRActiveLinkException, MCRAccessException {
109         MCRObjectID objectID = MCRObjectID.getInstance(id);
110         MCRBase obj = MCRMetadataManager.retrieve(objectID);
111         MCRObjectService service = obj.getService();
112         if (!service.isFlagTypeSet(MCRObjectService.FLAG_TYPE_CREATEDBY)) { //the egg
113             List<? extends MCRAbstractMetadataVersion<?>> versions = MCRXMLMetadataManager.instance()
114                 .listRevisions(objectID);
115             String createUser = null, modifyUser = null;
116             if (versions == null) {
117                 LOGGER.warn(
118                     "Cannot restore author servflags as there are no versions available. Setting to current user.");
119                 createUser = MCRSessionMgr.getCurrentSession().getUserInformation().getUserID();
120                 modifyUser = createUser;
121             } else {
122                 MCRAbstractMetadataVersion<?> firstVersion = versions.get(0);
123                 for (MCRAbstractMetadataVersion<?> version : versions) {
124                     if (version.getType() == 'A') {
125                         firstVersion = version; // get last 'added'
126                     }
127                 }
128                 MCRAbstractMetadataVersion<?> lastVersion = versions.get(versions.size() - 1);
129                 createUser = firstVersion.getUser();
130                 modifyUser = lastVersion.getUser();
131             }
132             service.addFlag(MCRObjectService.FLAG_TYPE_CREATEDBY, createUser);
133             LOGGER.info("{}, created by: {}", objectID, createUser);
134             if (!service.isFlagTypeSet(MCRObjectService.FLAG_TYPE_MODIFIEDBY)) { //the chicken
135                 //have to restore also modifiedby from version history.
136                 LOGGER.info("{}, modified by: {}", objectID, modifyUser);
137                 service.addFlag(MCRObjectService.FLAG_TYPE_MODIFIEDBY, modifyUser);
138             }
139             obj.setImportMode(true);
140             MCRMetadataManager.update(obj);
141         }
142     }
143 
144     @MCRCommand(syntax = "fix MCR-1717", help = "Fixes wrong entries in tile job table (see MCR-1717 comments)")
145     public static void fixMCR1717() {
146         EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
147         TypedQuery<MCRTileJob> allTileJobQuery = em.createNamedQuery("MCRTileJob.all", MCRTileJob.class);
148         List<MCRTileJob> tiles = allTileJobQuery.getResultList();
149         tiles.stream()
150             .filter(tj -> !tj.getPath().startsWith("/"))
151             .peek(tj -> LOGGER.info("Fixing TileJob {}:{}", tj.getDerivate(), tj.getPath()))
152             .forEach(tj -> {
153                 String newPath = "/" + tj.getPath();
154                 tj.setPath(newPath);
155             });
156     }
157 
158     @MCRCommand(syntax = "fix invalid derivate links {0} for {1}",
159         help = "Fixes the paths of all derivate links "
160             + "({0} -> xpath -> e.g. /mycoreobject/metadata/derivateLinks/derivateLink) for object {1}. (MCR-1267)",
161         order = 15)
162     public static void fixDerivateLinks(String xpath, String id) throws IOException, JDOMException, SAXException {
163         // get mcr object
164         MCRObjectID objectID = MCRObjectID.getInstance(id);
165 
166         // find derivate links
167         Document xml = MCRXMLMetadataManager.instance().retrieveXML(objectID);
168         Element mcrObjectXML = xml.getRootElement();
169         XPathExpression<Element> expression = XPathFactory.instance().compile(xpath, Filters.element());
170         List<Element> derivateLinkElements = expression.evaluate(mcrObjectXML);
171 
172         // check them
173         boolean changedObject = false;
174         for (Element derivateLinkElement : derivateLinkElements) {
175             String href = derivateLinkElement.getAttributeValue("href", MCRConstants.XLINK_NAMESPACE);
176             MCRMetaDerivateLink link = new MCRMetaDerivateLink();
177             link.setReference(href, null, null);
178             String owner = link.getOwner();
179             try {
180                 String path = link.getPath();
181                 MCRPath mcrPath = MCRPath.getPath(owner, path);
182                 if (!Files.exists(mcrPath)) {
183                     // path is correct URI encoded but not found.
184                     // this could have two reasons
185                     // 1. the file does not exist on the file system
186                     // 2. maybe the path isn't correct URI decoded
187                     //    -> e.g. a?c.tif -> path (a), query (c.tif) which is obvious wrong
188                     if (tryRawPath(objectID, derivateLinkElement, href, link, owner)) {
189                         changedObject = true;
190                     } else {
191                         LOGGER.warn("{} of {}cannot be found on file system. This is most likly a dead link.", href,
192                             objectID);
193                     }
194                 }
195             } catch (URISyntaxException uriExc) {
196                 // path could not be decoded, so maybe its already decoded
197                 // check if the file with href exists, if so, the path is
198                 // not encoded properly
199                 if (tryRawPath(objectID, derivateLinkElement, href, link, owner)) {
200                     changedObject = true;
201                 } else {
202                     LOGGER.warn(
203                         "{} of {} isn't URI encoded and cannot be found on file system."
204                             + " This is most likly a dead link.",
205                         href, objectID);
206                 }
207             }
208         }
209 
210         // store the mcr object if its changed
211         if (changedObject) {
212             // we use MCRXMLMetadataMananger because we don't want to validate the old mcr object
213             MCRXMLMetadataManager.instance().update(objectID, xml, new Date());
214             // manually fire update event
215             MCRObject newObject = MCRMetadataManager.retrieveMCRObject(objectID);
216             newObject.setImportMode(true);
217             MCRMetadataManager.fireUpdateEvent(newObject);
218         }
219     }
220 
221     private static boolean tryRawPath(MCRObjectID objectID, Element derivateLinkElement, String href,
222         MCRMetaDerivateLink link, String owner) {
223         String rawPath = link.getRawPath();
224         MCRPath mcrPath = MCRPath.getPath(owner, rawPath);
225         if (Files.exists(mcrPath)) {
226             // path exists -> do URI encoding for href
227             try {
228                 String encodedHref = MCRXMLFunctions.encodeURIPath(rawPath);
229                 derivateLinkElement.setAttribute("href", owner + encodedHref, MCRConstants.XLINK_NAMESPACE);
230                 return true;
231             } catch (URISyntaxException uriEncodeException) {
232                 LOGGER.error("Unable to encode {} for object {}", rawPath, objectID, uriEncodeException);
233                 return false;
234             }
235         }
236         return false;
237     }
238 
239     @MCRCommand(syntax = "add missing children to {0}",
240         help = "Adds missing children to structure of parent {0}. (MCR-1480)",
241         order = 15)
242     public static void fixMissingChildren(String id) throws IOException, JDOMException, SAXException {
243         MCRObjectID parentId = MCRObjectID.getInstance(id);
244         Collection<String> children = MCRLinkTableManager.instance().getSourceOf(parentId,
245             MCRLinkTableManager.ENTRY_TYPE_PARENT);
246         if (children.isEmpty()) {
247             return;
248         }
249         MCRObject parent = MCRMetadataManager.retrieveMCRObject(parentId);
250         MCRObjectStructure parentStructure = parent.getStructure();
251         int sizeBefore = parentStructure.getChildren().size();
252         children.stream().map(MCRObjectID::getInstance)
253             .filter(cid -> !parentStructure.getChildren().stream()
254                 .anyMatch(candidate -> candidate.getXLinkHrefID().equals(cid)))
255             .sorted().map(MCRMigrationCommands::toLinkId).sequential()
256             .peek(lid -> LOGGER.info("Adding {} to {}", lid, parentId)).forEach(parentStructure::addChild);
257         if (parentStructure.getChildren().size() != sizeBefore) {
258             MCRMetadataManager.fireUpdateEvent(parent);
259         }
260     }
261 
262     private static MCRMetaLinkID toLinkId(MCRObjectID mcrObjectID) {
263         return new MCRMetaLinkID("child", mcrObjectID, null, null);
264     }
265 
266     @MCRCommand(syntax = "add missing children",
267         help = "Adds missing children to structure of parent objects using MCRLinkTableManager. (MCR-1480)",
268         order = 20)
269     public static List<String> fixMissingChildren() throws IOException, JDOMException, SAXException {
270         EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
271         CriteriaBuilder cb = em.getCriteriaBuilder();
272         CriteriaQuery<String> query = cb.createQuery(String.class);
273         Root<MCRLINKHREF> ac = query.from(MCRLINKHREF.class);
274         return em
275             .createQuery(query
276                 .select(cb.concat(cb.literal("add missing children to "),
277                     ac.get(MCRLINKHREF_.key).get(MCRLINKHREFPK_.mcrto)))
278                 .where(cb.equal(ac.get(MCRLINKHREF_.key).get(MCRLINKHREFPK_.mcrtype),
279                     MCRLinkTableManager.ENTRY_TYPE_PARENT))
280                 .distinct(true).orderBy(cb.asc(cb.literal(1))))
281             .getResultList();
282     }
283 
284     // 2017-> 2018
285     @MCRCommand(syntax = "migrate tei entries in mets file of derivate {0}")
286     public static void migrateTEIEntrysOfMetsFileOfDerivate(String derivateIdStr)
287         throws IOException, JDOMException, SAXException {
288         final MCRObjectID derivateID = MCRObjectID.getInstance(derivateIdStr);
289         if (!MCRMetadataManager.exists(derivateID)) {
290             LOGGER.info("Derivate " + derivateIdStr + " does not exist!");
291             return;
292         }
293 
294         final MCRPath metsPath = MCRPath.getPath(derivateIdStr, "mets.xml");
295         if (!Files.exists(metsPath)) {
296             LOGGER.info("Derivate " + derivateIdStr + " has not mets.xml!");
297             return;
298         }
299 
300         final MCRXSLTransformer transformer = MCRXSLTransformer.getInstance("xsl/mets-translation-migration.xsl");
301 
302         final Document xml;
303 
304         try (InputStream is = Files.newInputStream(metsPath)) {
305             final MCRContent content = transformer.transform(new MCRStreamContent(is));
306             xml = content.asXML();
307         }
308 
309         try (OutputStream os = Files.newOutputStream(metsPath, StandardOpenOption.TRUNCATE_EXISTING)) {
310             final XMLOutputter writer = new XMLOutputter(Format.getPrettyFormat());
311             writer.output(xml, os);
312 
313         }
314 
315         LOGGER.info("Migrated mets of " + derivateIdStr);
316     }
317 
318     // 2018 -> 2019
319     @MCRCommand(syntax = "migrate all derivates",
320         help = "Migrates the order and label of all derivates (MCR-2003, MCR-2099)")
321     public static List<String> migrateAllDerivates() {
322         List<String> objectTypes = MCRObjectID.listTypes();
323         objectTypes.remove("derivate");
324         objectTypes.remove("class");
325 
326         ArrayList<String> commands = new ArrayList<>();
327         for (String t : objectTypes) {
328             for (String objID : MCRXMLMetadataManager.instance().listIDsOfType(t)) {
329                 commands.add("migrate derivatelinks for object " + objID);
330             }
331         }
332         return commands;
333     }
334 
335     @MCRCommand(syntax = "migrate derivatelinks for object {0}",
336         help = "Migrates the Order of derivates from object {0} to derivate "
337             + "(MCR-2003, MCR-2099)")
338     public static List<String> migrateDerivateLink(String objectIDStr) {
339         final MCRObjectID objectID = MCRObjectID.getInstance(objectIDStr);
340 
341         if (!MCRMetadataManager.exists(objectID)) {
342             throw new MCRException("The object " + objectIDStr + "does not exist!");
343         }
344 
345         final MCRObject mcrObject = MCRMetadataManager.retrieveMCRObject(objectID);
346         final List<MCRMetaEnrichedLinkID> derivates = mcrObject.getStructure().getDerivates();
347 
348         return derivates.stream().map(
349             (der) -> "migrate derivate " + der.getXLinkHrefID() + " using order " + (derivates.indexOf(der) + 1))
350             .collect(Collectors.toList());
351     }
352 
353     @MCRCommand(syntax = "migrate derivate {0} using order {1}",
354         help = "Sets the order of derivate {0} to the number {1}")
355     public static void setOrderOfDerivate(String derivateIDStr, String orderStr) throws MCRAccessException {
356         final int order = Integer.parseInt(orderStr);
357 
358         final MCRObjectID derivateID = MCRObjectID.getInstance(derivateIDStr);
359 
360         if (!MCRMetadataManager.exists(derivateID)) {
361             throw new MCRException("The object " + derivateIDStr + "does not exist!");
362         }
363 
364         final MCRDerivate derivate = MCRMetadataManager.retrieveMCRDerivate(derivateID);
365         derivate.setOrder(order);
366 
367         //migrate title:
368         //in professorenkatalog we used a service flag to store the title -> should be moved to titles/tile
369         if (derivate.getService().getFlags("title").size() > 0) {
370             String title = derivate.getService().getFlags("title").get(0);
371             derivate.getDerivate().getTitles().add(new MCRMetaLangText("title", "de", null, 0, "main", title));
372             derivate.getService().removeFlags("title");
373         }
374 
375         //update derivate
376         MCRMetadataManager.update(derivate);
377     }
378 }