View Javadoc
1   /*
2    * This file is part of ***  M y C o R e  ***
3    * See http://www.mycore.de/ for details.
4    *
5    * MyCoRe is free software: you can redistribute it and/or modify
6    * it under the terms of the GNU General Public License as published by
7    * the Free Software Foundation, either version 3 of the License, or
8    * (at your option) any later version.
9    *
10   * MyCoRe is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU General Public License for more details.
14   *
15   * You should have received a copy of the GNU General Public License
16   * along with MyCoRe.  If not, see <http://www.gnu.org/licenses/>.
17   */
18  
19  package org.mycore.iview2.frontend;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.nio.file.DirectoryStream;
24  import java.nio.file.FileSystem;
25  import java.nio.file.FileVisitResult;
26  import java.nio.file.Files;
27  import java.nio.file.Path;
28  import java.nio.file.SimpleFileVisitor;
29  import java.nio.file.attribute.BasicFileAttributes;
30  import java.text.MessageFormat;
31  import java.util.ArrayList;
32  import java.util.Enumeration;
33  import java.util.List;
34  import java.util.Locale;
35  import java.util.Objects;
36  import java.util.concurrent.TimeUnit;
37  import java.util.stream.Collectors;
38  import java.util.zip.CRC32;
39  import java.util.zip.ZipEntry;
40  import java.util.zip.ZipFile;
41  
42  import javax.imageio.ImageReader;
43  
44  import org.apache.logging.log4j.LogManager;
45  import org.apache.logging.log4j.Logger;
46  import org.jdom2.JDOMException;
47  import org.mycore.backend.jpa.MCREntityManagerProvider;
48  import org.mycore.common.MCRException;
49  import org.mycore.datamodel.metadata.MCRMetadataManager;
50  import org.mycore.datamodel.metadata.MCRObjectID;
51  import org.mycore.datamodel.niofs.MCRPath;
52  import org.mycore.datamodel.niofs.utils.MCRRecursiveDeleter;
53  import org.mycore.frontend.cli.MCRAbstractCommands;
54  import org.mycore.frontend.cli.MCRCommandUtils;
55  import org.mycore.frontend.cli.annotation.MCRCommand;
56  import org.mycore.frontend.cli.annotation.MCRCommandGroup;
57  import org.mycore.imagetiler.MCRImage;
58  import org.mycore.imagetiler.MCRTiledPictureProps;
59  import org.mycore.iview2.services.MCRIView2Tools;
60  import org.mycore.iview2.services.MCRImageTiler;
61  import org.mycore.iview2.services.MCRTileJob;
62  import org.mycore.iview2.services.MCRTilingQueue;
63  
64  import jakarta.persistence.EntityManager;
65  import jakarta.persistence.TypedQuery;
66  
67  /**
68   * Provides commands for Image Viewer.
69   * @author Thomas Scheffler (yagee)
70   *
71   */
72  
73  @MCRCommandGroup(name = "IView2 Tile Commands")
74  public class MCRIView2Commands extends MCRAbstractCommands {
75      private static final Logger LOGGER = LogManager.getLogger(MCRIView2Commands.class);
76  
77      private static final String TILE_DERIVATE_TILES_COMMAND_SYNTAX = "tile images of derivate {0}";
78  
79      private static final String CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX = "check tiles of derivate {0}";
80  
81      private static final String CHECK_TILES_OF_IMAGE_COMMAND_SYNTAX = "check tiles of image {0} {1}";
82  
83      private static final String TILE_IMAGE_COMMAND_SYNTAX = "tile image {0} {1}";
84  
85      private static final String DEL_DERIVATE_TILES_COMMAND_SYNTAX = "delete tiles of derivate {0}";
86  
87      /**
88       * meta command to tile all images of all derivates.
89       * @return list of commands to execute.
90       */
91      // tile images
92      @MCRCommand(syntax = "tile images of all derivates",
93          help = "tiles all images of all derivates with a supported image type as main document",
94          order = 40)
95      public static List<String> tileAll() {
96          MessageFormat syntaxMF = new MessageFormat(TILE_DERIVATE_TILES_COMMAND_SYNTAX, Locale.ROOT);
97          return MCRCommandUtils.getIdsForType("derivate")
98              .map(id -> syntaxMF.format(new Object[] { id }))
99              .collect(Collectors.toList());
100     }
101 
102     /**
103      * meta command to check (and repair) tiles of all images of all derivates.
104      * @return list of commands to execute.
105      */
106     @MCRCommand(syntax = "check tiles of all derivates",
107         help = "checks if all images have valid iview2 files and start tiling if not",
108         order = 10)
109     public static List<String> checkAll() {
110         MessageFormat syntaxMF = new MessageFormat(CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX, Locale.ROOT);
111         return MCRCommandUtils.getIdsForType("derivate")
112             .map(id -> syntaxMF.format(new Object[] { id }))
113             .collect(Collectors.toList());
114     }
115 
116     /**
117      * meta command to tile all images of derivates of a project.
118      * @return list of commands to execute.
119      */
120     @MCRCommand(syntax = "tile images of derivates of project {0}",
121         help = "tiles all images of derivates of a project with a supported image type as main document",
122         order = 41)
123     public static List<String> tileAllOfProject(String project) {
124         MessageFormat syntaxMF = new MessageFormat(TILE_DERIVATE_TILES_COMMAND_SYNTAX, Locale.ROOT);
125         return MCRCommandUtils.getIdsForProjectAndType(project, "derivate")
126             .map(id -> syntaxMF.format(new Object[] { id }))
127             .collect(Collectors.toList());
128     }
129 
130     /**
131      * meta command to check (and repair) tiles of all images of derivates of a project.
132      * @return list of commands to execute.
133      */
134     @MCRCommand(syntax = "check tiles of derivates of project {0}",
135         help = "checks if all images have valid iview2 files and start tiling if not",
136         order = 11)
137     public static List<String> checkAllOfProject(String project) {
138         MessageFormat syntaxMF = new MessageFormat(CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX, Locale.ROOT);
139         return MCRCommandUtils.getIdsForProjectAndType(project, "derivate")
140             .map(id -> syntaxMF.format(new Object[] { id }))
141             .collect(Collectors.toList());
142     }
143 
144     /**
145      * meta command to tile all images of derivates of an object .
146      * @param objectID a object ID
147      * @return list of commands to execute.
148      */
149     @MCRCommand(syntax = "tile images of object {0}",
150         help = "tiles all images of derivates of object {0} with a supported image type as main document",
151         order = 50)
152     public static List<String> tileDerivatesOfObject(String objectID) {
153         return forAllDerivatesOfObject(objectID, TILE_DERIVATE_TILES_COMMAND_SYNTAX);
154     }
155 
156     /**
157      * meta command to tile all images of this derivate.
158      * @param derivateID a derivate ID
159      * @return list of commands to execute.
160      */
161     @MCRCommand(syntax = TILE_DERIVATE_TILES_COMMAND_SYNTAX,
162         help = "tiles all images of derivate {0} with a supported image type as main document",
163         order = 60)
164     public static List<String> tileDerivate(String derivateID) throws IOException {
165         return forAllImages(derivateID, TILE_IMAGE_COMMAND_SYNTAX);
166     }
167 
168     /**
169      * meta command to check (and repair) all tiles of all images of this derivate.
170      * @param derivateID a derivate ID
171      * @return list of commands to execute.
172      */
173     @MCRCommand(syntax = CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX,
174         help = "checks if all images of derivate {0} with a supported image type as main document have valid iview2" +
175             " files and start tiling if not ",
176         order = 20)
177     public static List<String> checkTilesOfDerivate(String derivateID) throws IOException {
178         return forAllImages(derivateID, CHECK_TILES_OF_IMAGE_COMMAND_SYNTAX);
179     }
180 
181     private static List<String> forAllImages(String derivateID, String batchCommandSyntax) throws IOException {
182         if (!MCRIView2Tools.isDerivateSupported(derivateID)) {
183             LOGGER.info("Skipping tiling of derivate {} as it's main file is not supported by IView2.", derivateID);
184             return null;
185         }
186         MCRPath derivateRoot = MCRPath.getPath(derivateID, "/");
187 
188         if (!Files.exists(derivateRoot)) {
189             throw new MCRException("Derivate " + derivateID + " does not exist or is not a directory!");
190         }
191 
192         List<MCRPath> supportedFiles = getSupportedFiles(derivateRoot);
193         return supportedFiles.stream()
194             .map(
195                 image -> new MessageFormat(batchCommandSyntax, Locale.ROOT).format(
196                     new Object[] { derivateID, image.getOwnerRelativePath() }))
197             .collect(Collectors.toList());
198     }
199 
200     @MCRCommand(syntax = "fix dead tile jobs", help = "Deletes entries for files which dont exist anymore!")
201     public static void fixDeadEntries() {
202         EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
203         TypedQuery<MCRTileJob> allTileJobQuery = em.createNamedQuery("MCRTileJob.all", MCRTileJob.class);
204         List<MCRTileJob> tiles = allTileJobQuery.getResultList();
205         tiles.stream()
206             .filter(tj -> {
207                 MCRPath path = MCRPath.getPath(tj.getDerivate(), tj.getPath());
208                 return !Files.exists(path);
209             })
210             .peek(tj -> LOGGER.info("Delete TileJob {}:{}", tj.getDerivate(), tj.getPath()))
211             .forEach(em::remove);
212     }
213 
214     /**
215      * checks and repairs tile of this derivate.
216      * @param derivate derivate ID
217      * @param absoluteImagePath absolute path to image file
218      */
219     @MCRCommand(syntax = CHECK_TILES_OF_IMAGE_COMMAND_SYNTAX,
220         help = "checks if tiles a specific file identified by its derivate {0} and absolute path {1} are valid or" +
221             " generates new one",
222         order = 30)
223     public static void checkImage(String derivate, String absoluteImagePath) throws IOException {
224         Path iviewFile = MCRImage.getTiledFile(MCRIView2Tools.getTileDir(), derivate, absoluteImagePath);
225         //file checks
226         if (!Files.exists(iviewFile)) {
227             LOGGER.warn("IView2 file does not exist: {}", iviewFile);
228             tileImage(derivate, absoluteImagePath);
229             return;
230         }
231         MCRTiledPictureProps props;
232         try {
233             props = MCRTiledPictureProps.getInstanceFromFile(iviewFile);
234         } catch (Exception e) {
235             LOGGER.warn("Error while reading image metadata. Recreating tiles.", e);
236             tileImage(derivate, absoluteImagePath);
237             return;
238         }
239         if (props == null) {
240             LOGGER.warn("Could not get tile metadata");
241             tileImage(derivate, absoluteImagePath);
242             return;
243         }
244         ZipFile iviewImage;
245         try {
246             iviewImage = new ZipFile(iviewFile.toFile());
247             validateZipFile(iviewImage);
248         } catch (Exception e) {
249             LOGGER.warn("Error while reading Iview2 file: {}", iviewFile, e);
250             tileImage(derivate, absoluteImagePath);
251             return;
252         }
253         try (FileSystem fs = MCRIView2Tools.getFileSystem(iviewFile)) {
254             Path iviewFileRoot = fs.getRootDirectories().iterator().next();
255             //structure and metadata checks
256             int tilesCount = iviewImage.size() - 1; //one for metadata
257             if (props.getTilesCount() != tilesCount) {
258                 LOGGER.warn("Metadata tile count does not match stored tile count: {}", iviewFile);
259                 tileImage(derivate, absoluteImagePath);
260                 return;
261             }
262             int x = props.getWidth();
263             int y = props.getHeight();
264             if (MCRImage.getTileCount(x, y) != tilesCount) {
265                 LOGGER.warn("Calculated tile count does not match stored tile count: {}", iviewFile);
266                 tileImage(derivate, absoluteImagePath);
267                 return;
268             }
269             try {
270                 ImageReader imageReader = MCRIView2Tools.getTileImageReader();
271                 MCRIView2Tools.getZoomLevel(iviewFileRoot, props, imageReader, 0);
272                 int maxX = (int) Math.ceil((double) props.getWidth() / MCRImage.getTileSize());
273                 int maxY = (int) Math.ceil((double) props.getHeight() / MCRImage.getTileSize());
274                 LOGGER.debug("Image size:{}x{}, tiles:{}x{}", props.getWidth(), props.getHeight(), maxX, maxY);
275                 try {
276                     MCRIView2Tools.readTile(iviewFileRoot, imageReader,
277                         props.getZoomlevel(), maxX - 1, 0);
278                 } finally {
279                     imageReader.dispose();
280                 }
281             } catch (IOException | JDOMException e) {
282                 LOGGER.warn("Could not read thumbnail of {}", iviewFile, e);
283                 tileImage(derivate, absoluteImagePath);
284             }
285         }
286     }
287 
288     private static void validateZipFile(ZipFile iviewImage) throws IOException {
289         Enumeration<? extends ZipEntry> entries = iviewImage.entries();
290         CRC32 crc = new CRC32();
291         byte[] data = new byte[4096];
292         int read;
293         while (entries.hasMoreElements()) {
294             ZipEntry entry = entries.nextElement();
295             try (InputStream is = iviewImage.getInputStream(entry)) {
296                 while ((read = is.read(data, 0, data.length)) != -1) {
297                     crc.update(data, 0, read);
298                 }
299             }
300             if (entry.getCrc() != crc.getValue()) {
301                 throw new IOException("CRC32 does not match for entry: " + entry.getName());
302             }
303             crc.reset();
304         }
305     }
306 
307     /**
308      * Tiles this image.
309      * @param derivate derivate ID
310      * @param absoluteImagePath absolute path to image file
311      */
312     @MCRCommand(syntax = TILE_IMAGE_COMMAND_SYNTAX,
313         help = "tiles a specific file identified by its derivate {0} and absolute path {1}",
314         order = 70)
315     public static void tileImage(String derivate, String absoluteImagePath) {
316         MCRTileJob job = new MCRTileJob();
317         job.setDerivate(derivate);
318         job.setPath(absoluteImagePath);
319         MCRTilingQueue.getInstance().offer(job);
320         startMasterTilingThread();
321     }
322 
323     /**
324      * Tiles this {@link MCRPath}
325      */
326     public static void tileImage(MCRPath file) throws IOException {
327         if (MCRIView2Tools.isFileSupported(file)) {
328             MCRTileJob job = new MCRTileJob();
329             job.setDerivate(file.getOwner());
330             job.setPath(file.getOwnerRelativePath());
331             MCRTilingQueue.getInstance().offer(job);
332             LOGGER.info("Added to TilingQueue: {}", file);
333             startMasterTilingThread();
334         }
335     }
336 
337     private static void startMasterTilingThread() {
338         if (!MCRImageTiler.isRunning()) {
339             LOGGER.info("Starting Tiling thread.");
340             final Thread tiling = new Thread(MCRImageTiler.getInstance());
341             tiling.start();
342         }
343     }
344 
345     /**
346      * Deletes all image tiles.
347      */
348     @MCRCommand(syntax = "delete all tiles", help = "removes all tiles of all derivates", order = 80)
349     public static void deleteAllTiles() throws IOException {
350         Path storeDir = MCRIView2Tools.getTileDir();
351         Files.walkFileTree(storeDir, MCRRecursiveDeleter.instance());
352         MCRTilingQueue.getInstance().clear();
353     }
354 
355     /**
356      * Deletes all image tiles of derivates of this object.
357      * @param objectID a object ID
358      */
359     @MCRCommand(syntax = "delete tiles of object {0}",
360         help = "removes tiles of a specific file identified by its object ID {0}",
361         order = 90)
362     public static List<String> deleteDerivateTilesOfObject(String objectID) {
363         return forAllDerivatesOfObject(objectID, DEL_DERIVATE_TILES_COMMAND_SYNTAX);
364     }
365 
366     private static List<String> forAllDerivatesOfObject(String objectID, String batchCommandSyntax) {
367         MCRObjectID mcrobjid;
368         try {
369             mcrobjid = MCRObjectID.getInstance(objectID);
370         } catch (Exception e) {
371             LOGGER.error("The object ID {} is wrong", objectID);
372             return null;
373         }
374         List<MCRObjectID> derivateIds = MCRMetadataManager.getDerivateIds(mcrobjid, 0, TimeUnit.MILLISECONDS);
375         if (derivateIds == null) {
376             LOGGER.error("Object does not exist: {}", mcrobjid);
377         }
378         ArrayList<String> cmds = new ArrayList<>(derivateIds.size());
379         for (MCRObjectID derId : derivateIds) {
380             cmds.add(new MessageFormat(batchCommandSyntax, Locale.ROOT).format(new String[] { derId.toString() }));
381         }
382         return cmds;
383     }
384 
385     /**
386      * Deletes all image tiles of this derivate.
387      * @param derivateID a derivate ID
388      */
389     @MCRCommand(syntax = DEL_DERIVATE_TILES_COMMAND_SYNTAX,
390         help = "removes tiles of a specific file identified by its derivate ID {0}",
391         order = 100)
392     public static void deleteDerivateTiles(String derivateID) throws IOException {
393         Path derivateDir = MCRImage.getTiledFile(MCRIView2Tools.getTileDir(), derivateID, null);
394         Files.walkFileTree(derivateDir, MCRRecursiveDeleter.instance());
395         MCRTilingQueue.getInstance().remove(derivateID);
396     }
397 
398     /**
399      * Deletes all image tiles of this derivate.
400      * @param derivate derivate ID
401      * @param absoluteImagePath absolute path to image file
402      */
403     @MCRCommand(syntax = "delete tiles of image {0} {1}",
404         help = "removes tiles of a specific file identified by its derivate ID {0} and absolute path {1}",
405         order = 110)
406     public static void deleteImageTiles(String derivate, String absoluteImagePath) throws IOException {
407         Path tileFile = MCRImage.getTiledFile(MCRIView2Tools.getTileDir(), derivate, absoluteImagePath);
408         deleteFileAndEmptyDirectories(tileFile);
409         int removed = MCRTilingQueue.getInstance().remove(derivate, absoluteImagePath);
410         LOGGER.info("removed tiles from {} images", removed);
411     }
412 
413     private static void deleteFileAndEmptyDirectories(Path file) throws IOException {
414         if (Files.isRegularFile(file)) {
415             Files.delete(file);
416         }
417         if (Files.isDirectory(file)) {
418             try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(file)) {
419                 for (@SuppressWarnings("unused")
420                 Path entry : directoryStream) {
421                     return;
422                 }
423                 Files.delete(file);
424             }
425         }
426         Path parent = file.getParent();
427         if (parent != null && parent.getNameCount() > 0) {
428             deleteFileAndEmptyDirectories(parent);
429         }
430     }
431 
432     private static List<MCRPath> getSupportedFiles(MCRPath rootNode) throws IOException {
433         final ArrayList<MCRPath> files = new ArrayList<>();
434         SimpleFileVisitor<Path> test = new SimpleFileVisitor<>() {
435             @Override
436             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
437                 Objects.requireNonNull(file);
438                 Objects.requireNonNull(attrs);
439                 if (MCRIView2Tools.isFileSupported(file)) {
440                     files.add(MCRPath.toMCRPath(file));
441                 }
442                 return FileVisitResult.CONTINUE;
443             }
444         };
445         Files.walkFileTree(rootNode, test);
446         return files;
447     }
448 
449 }