1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.mycore.iview2.frontend;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.nio.file.DirectoryStream;
24 import java.nio.file.FileSystem;
25 import java.nio.file.FileVisitResult;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.SimpleFileVisitor;
29 import java.nio.file.attribute.BasicFileAttributes;
30 import java.text.MessageFormat;
31 import java.util.ArrayList;
32 import java.util.Enumeration;
33 import java.util.List;
34 import java.util.Locale;
35 import java.util.Objects;
36 import java.util.concurrent.TimeUnit;
37 import java.util.stream.Collectors;
38 import java.util.zip.CRC32;
39 import java.util.zip.ZipEntry;
40 import java.util.zip.ZipFile;
41
42 import javax.imageio.ImageReader;
43
44 import org.apache.logging.log4j.LogManager;
45 import org.apache.logging.log4j.Logger;
46 import org.jdom2.JDOMException;
47 import org.mycore.backend.jpa.MCREntityManagerProvider;
48 import org.mycore.common.MCRException;
49 import org.mycore.datamodel.metadata.MCRMetadataManager;
50 import org.mycore.datamodel.metadata.MCRObjectID;
51 import org.mycore.datamodel.niofs.MCRPath;
52 import org.mycore.datamodel.niofs.utils.MCRRecursiveDeleter;
53 import org.mycore.frontend.cli.MCRAbstractCommands;
54 import org.mycore.frontend.cli.MCRCommandUtils;
55 import org.mycore.frontend.cli.annotation.MCRCommand;
56 import org.mycore.frontend.cli.annotation.MCRCommandGroup;
57 import org.mycore.imagetiler.MCRImage;
58 import org.mycore.imagetiler.MCRTiledPictureProps;
59 import org.mycore.iview2.services.MCRIView2Tools;
60 import org.mycore.iview2.services.MCRImageTiler;
61 import org.mycore.iview2.services.MCRTileJob;
62 import org.mycore.iview2.services.MCRTilingQueue;
63
64 import jakarta.persistence.EntityManager;
65 import jakarta.persistence.TypedQuery;
66
67
68
69
70
71
72
73 @MCRCommandGroup(name = "IView2 Tile Commands")
74 public class MCRIView2Commands extends MCRAbstractCommands {
75 private static final Logger LOGGER = LogManager.getLogger(MCRIView2Commands.class);
76
77 private static final String TILE_DERIVATE_TILES_COMMAND_SYNTAX = "tile images of derivate {0}";
78
79 private static final String CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX = "check tiles of derivate {0}";
80
81 private static final String CHECK_TILES_OF_IMAGE_COMMAND_SYNTAX = "check tiles of image {0} {1}";
82
83 private static final String TILE_IMAGE_COMMAND_SYNTAX = "tile image {0} {1}";
84
85 private static final String DEL_DERIVATE_TILES_COMMAND_SYNTAX = "delete tiles of derivate {0}";
86
87
88
89
90
91
92 @MCRCommand(syntax = "tile images of all derivates",
93 help = "tiles all images of all derivates with a supported image type as main document",
94 order = 40)
95 public static List<String> tileAll() {
96 MessageFormat syntaxMF = new MessageFormat(TILE_DERIVATE_TILES_COMMAND_SYNTAX, Locale.ROOT);
97 return MCRCommandUtils.getIdsForType("derivate")
98 .map(id -> syntaxMF.format(new Object[] { id }))
99 .collect(Collectors.toList());
100 }
101
102
103
104
105
106 @MCRCommand(syntax = "check tiles of all derivates",
107 help = "checks if all images have valid iview2 files and start tiling if not",
108 order = 10)
109 public static List<String> checkAll() {
110 MessageFormat syntaxMF = new MessageFormat(CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX, Locale.ROOT);
111 return MCRCommandUtils.getIdsForType("derivate")
112 .map(id -> syntaxMF.format(new Object[] { id }))
113 .collect(Collectors.toList());
114 }
115
116
117
118
119
120 @MCRCommand(syntax = "tile images of derivates of project {0}",
121 help = "tiles all images of derivates of a project with a supported image type as main document",
122 order = 41)
123 public static List<String> tileAllOfProject(String project) {
124 MessageFormat syntaxMF = new MessageFormat(TILE_DERIVATE_TILES_COMMAND_SYNTAX, Locale.ROOT);
125 return MCRCommandUtils.getIdsForProjectAndType(project, "derivate")
126 .map(id -> syntaxMF.format(new Object[] { id }))
127 .collect(Collectors.toList());
128 }
129
130
131
132
133
134 @MCRCommand(syntax = "check tiles of derivates of project {0}",
135 help = "checks if all images have valid iview2 files and start tiling if not",
136 order = 11)
137 public static List<String> checkAllOfProject(String project) {
138 MessageFormat syntaxMF = new MessageFormat(CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX, Locale.ROOT);
139 return MCRCommandUtils.getIdsForProjectAndType(project, "derivate")
140 .map(id -> syntaxMF.format(new Object[] { id }))
141 .collect(Collectors.toList());
142 }
143
144
145
146
147
148
149 @MCRCommand(syntax = "tile images of object {0}",
150 help = "tiles all images of derivates of object {0} with a supported image type as main document",
151 order = 50)
152 public static List<String> tileDerivatesOfObject(String objectID) {
153 return forAllDerivatesOfObject(objectID, TILE_DERIVATE_TILES_COMMAND_SYNTAX);
154 }
155
156
157
158
159
160
161 @MCRCommand(syntax = TILE_DERIVATE_TILES_COMMAND_SYNTAX,
162 help = "tiles all images of derivate {0} with a supported image type as main document",
163 order = 60)
164 public static List<String> tileDerivate(String derivateID) throws IOException {
165 return forAllImages(derivateID, TILE_IMAGE_COMMAND_SYNTAX);
166 }
167
168
169
170
171
172
173 @MCRCommand(syntax = CHECK_TILES_OF_DERIVATE_COMMAND_SYNTAX,
174 help = "checks if all images of derivate {0} with a supported image type as main document have valid iview2" +
175 " files and start tiling if not ",
176 order = 20)
177 public static List<String> checkTilesOfDerivate(String derivateID) throws IOException {
178 return forAllImages(derivateID, CHECK_TILES_OF_IMAGE_COMMAND_SYNTAX);
179 }
180
181 private static List<String> forAllImages(String derivateID, String batchCommandSyntax) throws IOException {
182 if (!MCRIView2Tools.isDerivateSupported(derivateID)) {
183 LOGGER.info("Skipping tiling of derivate {} as it's main file is not supported by IView2.", derivateID);
184 return null;
185 }
186 MCRPath derivateRoot = MCRPath.getPath(derivateID, "/");
187
188 if (!Files.exists(derivateRoot)) {
189 throw new MCRException("Derivate " + derivateID + " does not exist or is not a directory!");
190 }
191
192 List<MCRPath> supportedFiles = getSupportedFiles(derivateRoot);
193 return supportedFiles.stream()
194 .map(
195 image -> new MessageFormat(batchCommandSyntax, Locale.ROOT).format(
196 new Object[] { derivateID, image.getOwnerRelativePath() }))
197 .collect(Collectors.toList());
198 }
199
200 @MCRCommand(syntax = "fix dead tile jobs", help = "Deletes entries for files which dont exist anymore!")
201 public static void fixDeadEntries() {
202 EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
203 TypedQuery<MCRTileJob> allTileJobQuery = em.createNamedQuery("MCRTileJob.all", MCRTileJob.class);
204 List<MCRTileJob> tiles = allTileJobQuery.getResultList();
205 tiles.stream()
206 .filter(tj -> {
207 MCRPath path = MCRPath.getPath(tj.getDerivate(), tj.getPath());
208 return !Files.exists(path);
209 })
210 .peek(tj -> LOGGER.info("Delete TileJob {}:{}", tj.getDerivate(), tj.getPath()))
211 .forEach(em::remove);
212 }
213
214
215
216
217
218
219 @MCRCommand(syntax = CHECK_TILES_OF_IMAGE_COMMAND_SYNTAX,
220 help = "checks if tiles a specific file identified by its derivate {0} and absolute path {1} are valid or" +
221 " generates new one",
222 order = 30)
223 public static void checkImage(String derivate, String absoluteImagePath) throws IOException {
224 Path iviewFile = MCRImage.getTiledFile(MCRIView2Tools.getTileDir(), derivate, absoluteImagePath);
225
226 if (!Files.exists(iviewFile)) {
227 LOGGER.warn("IView2 file does not exist: {}", iviewFile);
228 tileImage(derivate, absoluteImagePath);
229 return;
230 }
231 MCRTiledPictureProps props;
232 try {
233 props = MCRTiledPictureProps.getInstanceFromFile(iviewFile);
234 } catch (Exception e) {
235 LOGGER.warn("Error while reading image metadata. Recreating tiles.", e);
236 tileImage(derivate, absoluteImagePath);
237 return;
238 }
239 if (props == null) {
240 LOGGER.warn("Could not get tile metadata");
241 tileImage(derivate, absoluteImagePath);
242 return;
243 }
244 ZipFile iviewImage;
245 try {
246 iviewImage = new ZipFile(iviewFile.toFile());
247 validateZipFile(iviewImage);
248 } catch (Exception e) {
249 LOGGER.warn("Error while reading Iview2 file: {}", iviewFile, e);
250 tileImage(derivate, absoluteImagePath);
251 return;
252 }
253 try (FileSystem fs = MCRIView2Tools.getFileSystem(iviewFile)) {
254 Path iviewFileRoot = fs.getRootDirectories().iterator().next();
255
256 int tilesCount = iviewImage.size() - 1;
257 if (props.getTilesCount() != tilesCount) {
258 LOGGER.warn("Metadata tile count does not match stored tile count: {}", iviewFile);
259 tileImage(derivate, absoluteImagePath);
260 return;
261 }
262 int x = props.getWidth();
263 int y = props.getHeight();
264 if (MCRImage.getTileCount(x, y) != tilesCount) {
265 LOGGER.warn("Calculated tile count does not match stored tile count: {}", iviewFile);
266 tileImage(derivate, absoluteImagePath);
267 return;
268 }
269 try {
270 ImageReader imageReader = MCRIView2Tools.getTileImageReader();
271 MCRIView2Tools.getZoomLevel(iviewFileRoot, props, imageReader, 0);
272 int maxX = (int) Math.ceil((double) props.getWidth() / MCRImage.getTileSize());
273 int maxY = (int) Math.ceil((double) props.getHeight() / MCRImage.getTileSize());
274 LOGGER.debug("Image size:{}x{}, tiles:{}x{}", props.getWidth(), props.getHeight(), maxX, maxY);
275 try {
276 MCRIView2Tools.readTile(iviewFileRoot, imageReader,
277 props.getZoomlevel(), maxX - 1, 0);
278 } finally {
279 imageReader.dispose();
280 }
281 } catch (IOException | JDOMException e) {
282 LOGGER.warn("Could not read thumbnail of {}", iviewFile, e);
283 tileImage(derivate, absoluteImagePath);
284 }
285 }
286 }
287
288 private static void validateZipFile(ZipFile iviewImage) throws IOException {
289 Enumeration<? extends ZipEntry> entries = iviewImage.entries();
290 CRC32 crc = new CRC32();
291 byte[] data = new byte[4096];
292 int read;
293 while (entries.hasMoreElements()) {
294 ZipEntry entry = entries.nextElement();
295 try (InputStream is = iviewImage.getInputStream(entry)) {
296 while ((read = is.read(data, 0, data.length)) != -1) {
297 crc.update(data, 0, read);
298 }
299 }
300 if (entry.getCrc() != crc.getValue()) {
301 throw new IOException("CRC32 does not match for entry: " + entry.getName());
302 }
303 crc.reset();
304 }
305 }
306
307
308
309
310
311
312 @MCRCommand(syntax = TILE_IMAGE_COMMAND_SYNTAX,
313 help = "tiles a specific file identified by its derivate {0} and absolute path {1}",
314 order = 70)
315 public static void tileImage(String derivate, String absoluteImagePath) {
316 MCRTileJob job = new MCRTileJob();
317 job.setDerivate(derivate);
318 job.setPath(absoluteImagePath);
319 MCRTilingQueue.getInstance().offer(job);
320 startMasterTilingThread();
321 }
322
323
324
325
326 public static void tileImage(MCRPath file) throws IOException {
327 if (MCRIView2Tools.isFileSupported(file)) {
328 MCRTileJob job = new MCRTileJob();
329 job.setDerivate(file.getOwner());
330 job.setPath(file.getOwnerRelativePath());
331 MCRTilingQueue.getInstance().offer(job);
332 LOGGER.info("Added to TilingQueue: {}", file);
333 startMasterTilingThread();
334 }
335 }
336
337 private static void startMasterTilingThread() {
338 if (!MCRImageTiler.isRunning()) {
339 LOGGER.info("Starting Tiling thread.");
340 final Thread tiling = new Thread(MCRImageTiler.getInstance());
341 tiling.start();
342 }
343 }
344
345
346
347
348 @MCRCommand(syntax = "delete all tiles", help = "removes all tiles of all derivates", order = 80)
349 public static void deleteAllTiles() throws IOException {
350 Path storeDir = MCRIView2Tools.getTileDir();
351 Files.walkFileTree(storeDir, MCRRecursiveDeleter.instance());
352 MCRTilingQueue.getInstance().clear();
353 }
354
355
356
357
358
359 @MCRCommand(syntax = "delete tiles of object {0}",
360 help = "removes tiles of a specific file identified by its object ID {0}",
361 order = 90)
362 public static List<String> deleteDerivateTilesOfObject(String objectID) {
363 return forAllDerivatesOfObject(objectID, DEL_DERIVATE_TILES_COMMAND_SYNTAX);
364 }
365
366 private static List<String> forAllDerivatesOfObject(String objectID, String batchCommandSyntax) {
367 MCRObjectID mcrobjid;
368 try {
369 mcrobjid = MCRObjectID.getInstance(objectID);
370 } catch (Exception e) {
371 LOGGER.error("The object ID {} is wrong", objectID);
372 return null;
373 }
374 List<MCRObjectID> derivateIds = MCRMetadataManager.getDerivateIds(mcrobjid, 0, TimeUnit.MILLISECONDS);
375 if (derivateIds == null) {
376 LOGGER.error("Object does not exist: {}", mcrobjid);
377 }
378 ArrayList<String> cmds = new ArrayList<>(derivateIds.size());
379 for (MCRObjectID derId : derivateIds) {
380 cmds.add(new MessageFormat(batchCommandSyntax, Locale.ROOT).format(new String[] { derId.toString() }));
381 }
382 return cmds;
383 }
384
385
386
387
388
389 @MCRCommand(syntax = DEL_DERIVATE_TILES_COMMAND_SYNTAX,
390 help = "removes tiles of a specific file identified by its derivate ID {0}",
391 order = 100)
392 public static void deleteDerivateTiles(String derivateID) throws IOException {
393 Path derivateDir = MCRImage.getTiledFile(MCRIView2Tools.getTileDir(), derivateID, null);
394 Files.walkFileTree(derivateDir, MCRRecursiveDeleter.instance());
395 MCRTilingQueue.getInstance().remove(derivateID);
396 }
397
398
399
400
401
402
403 @MCRCommand(syntax = "delete tiles of image {0} {1}",
404 help = "removes tiles of a specific file identified by its derivate ID {0} and absolute path {1}",
405 order = 110)
406 public static void deleteImageTiles(String derivate, String absoluteImagePath) throws IOException {
407 Path tileFile = MCRImage.getTiledFile(MCRIView2Tools.getTileDir(), derivate, absoluteImagePath);
408 deleteFileAndEmptyDirectories(tileFile);
409 int removed = MCRTilingQueue.getInstance().remove(derivate, absoluteImagePath);
410 LOGGER.info("removed tiles from {} images", removed);
411 }
412
413 private static void deleteFileAndEmptyDirectories(Path file) throws IOException {
414 if (Files.isRegularFile(file)) {
415 Files.delete(file);
416 }
417 if (Files.isDirectory(file)) {
418 try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(file)) {
419 for (@SuppressWarnings("unused")
420 Path entry : directoryStream) {
421 return;
422 }
423 Files.delete(file);
424 }
425 }
426 Path parent = file.getParent();
427 if (parent != null && parent.getNameCount() > 0) {
428 deleteFileAndEmptyDirectories(parent);
429 }
430 }
431
432 private static List<MCRPath> getSupportedFiles(MCRPath rootNode) throws IOException {
433 final ArrayList<MCRPath> files = new ArrayList<>();
434 SimpleFileVisitor<Path> test = new SimpleFileVisitor<>() {
435 @Override
436 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
437 Objects.requireNonNull(file);
438 Objects.requireNonNull(attrs);
439 if (MCRIView2Tools.isFileSupported(file)) {
440 files.add(MCRPath.toMCRPath(file));
441 }
442 return FileVisitResult.CONTINUE;
443 }
444 };
445 Files.walkFileTree(rootNode, test);
446 return files;
447 }
448
449 }