1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.mycore.frontend.fileupload;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.UncheckedIOException;
24 import java.nio.file.DirectoryStream;
25 import java.nio.file.FileSystemException;
26 import java.nio.file.FileVisitResult;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.nio.file.SimpleFileVisitor;
30 import java.nio.file.StandardCopyOption;
31 import java.nio.file.attribute.BasicFileAttributes;
32 import java.util.Collection;
33 import java.util.LinkedList;
34 import java.util.List;
35 import java.util.Objects;
36 import java.util.function.Supplier;
37 import java.util.stream.Collectors;
38 import java.util.stream.Stream;
39
40 import org.apache.logging.log4j.LogManager;
41 import org.apache.logging.log4j.Logger;
42 import org.mycore.access.MCRAccessInterface;
43 import org.mycore.access.MCRAccessException;
44 import org.mycore.access.MCRRuleAccessInterface;
45 import org.mycore.access.MCRAccessManager;
46 import org.mycore.common.MCRPersistenceException;
47 import org.mycore.common.config.MCRConfiguration2;
48 import org.mycore.common.processing.MCRProcessableStatus;
49 import org.mycore.datamodel.metadata.MCRDerivate;
50 import org.mycore.datamodel.metadata.MCRMetaIFS;
51 import org.mycore.datamodel.metadata.MCRMetaLinkID;
52 import org.mycore.datamodel.metadata.MCRMetadataManager;
53 import org.mycore.datamodel.metadata.MCRObjectDerivate;
54 import org.mycore.datamodel.metadata.MCRObjectID;
55 import org.mycore.datamodel.niofs.MCRFileAttributes;
56 import org.mycore.datamodel.niofs.MCRPath;
57
58
59
60
61
62
63
64
65
66 public class MCRUploadHandlerIFS extends MCRUploadHandler {
67
68 private static final Logger LOGGER = LogManager.getLogger(MCRUploadHandlerIFS.class);
69
70 private static final String ID_TYPE = "derivate";
71
72 private static final String FILE_PROCESSOR_PROPERTY = "MCR.MCRUploadHandlerIFS.FileProcessors";
73
74 private static final List<MCRPostUploadFileProcessor> FILE_PROCESSORS = initProcessorList();
75
76 protected String documentID;
77
78 protected String derivateID;
79
80 protected MCRDerivate derivate;
81
82 protected MCRPath rootDir;
83
84 private int filesUploaded;
85
86 public MCRUploadHandlerIFS(String documentID, String derivateID) {
87 super();
88 this.documentID = Objects.requireNonNull(documentID, "Document ID may not be 'null'.");
89 this.derivateID = derivateID;
90 this.setName(this.derivateID);
91 }
92
93 public MCRUploadHandlerIFS(String documentID, String derivateID, String returnURL) {
94 this(documentID, derivateID);
95 this.url = Objects.requireNonNull(returnURL, "Return URL may not be 'null'.");
96 }
97
98 private static List<MCRPostUploadFileProcessor> initProcessorList() {
99 return MCRConfiguration2.getString(FILE_PROCESSOR_PROPERTY)
100 .map(MCRConfiguration2::splitValue)
101 .orElseGet(Stream::empty)
102 .map(MCRConfiguration2::<MCRPostUploadFileProcessor>instantiateClass)
103 .collect(Collectors.toList());
104 }
105
106 @Override
107 public void startUpload(int numFiles) {
108 super.startUpload(numFiles);
109 this.filesUploaded = 0;
110 this.setStatus(MCRProcessableStatus.processing);
111 this.setProgress(0);
112 this.setProgressText("start upload...");
113 }
114
115 private synchronized void prepareUpload() throws MCRPersistenceException, MCRAccessException, IOException {
116 if (this.derivate != null) {
117 return;
118 }
119 LOGGER.debug("upload starting, expecting {} files", getNumFiles());
120
121 MCRObjectID derivateID = getOrCreateDerivateID();
122
123 if (MCRMetadataManager.exists(derivateID)) {
124 this.derivate = MCRMetadataManager.retrieveMCRDerivate(derivateID);
125 } else {
126 this.derivate = createDerivate(derivateID);
127 }
128
129 getOrCreateRootDirectory();
130
131 LOGGER.debug("uploading into {} of {}", this.derivateID, this.documentID);
132 }
133
134 private MCRObjectID getOrCreateDerivateID() {
135 if (derivateID == null) {
136 String projectID = MCRObjectID.getInstance(this.documentID).getProjectId();
137 MCRObjectID oid = MCRObjectID.getNextFreeId(projectID + '_' + ID_TYPE);
138 this.derivateID = oid.toString();
139 return oid;
140 } else {
141 return MCRObjectID.getInstance(derivateID);
142 }
143 }
144
145 private MCRDerivate createDerivate(MCRObjectID derivateID)
146 throws MCRPersistenceException, IOException, MCRAccessException {
147 MCRDerivate derivate = new MCRDerivate();
148 derivate.setId(derivateID);
149
150 String schema = MCRConfiguration2.getString("MCR.Metadata.Config.derivate")
151 .orElse("datamodel-derivate.xml")
152 .replaceAll(".xml", ".xsd");
153 derivate.setSchema(schema);
154
155 MCRMetaLinkID linkId = new MCRMetaLinkID();
156 linkId.setSubTag("linkmeta");
157 linkId.setReference(documentID, null, null);
158 derivate.getDerivate().setLinkMeta(linkId);
159
160 MCRMetaIFS ifs = new MCRMetaIFS();
161 ifs.setSubTag("internal");
162 ifs.setSourcePath(null);
163 derivate.getDerivate().setInternals(ifs);
164
165 LOGGER.debug("Creating new derivate with ID {}", this.derivateID);
166 MCRMetadataManager.create(derivate);
167
168 setDefaultPermissions(derivateID);
169
170 return derivate;
171 }
172
173 protected void setDefaultPermissions(MCRObjectID derivateID) {
174 if (MCRConfiguration2.getBoolean("MCR.Access.AddDerivateDefaultRule").orElse(true)) {
175 MCRAccessInterface accessImpl = MCRAccessManager.getAccessImpl();
176 if (accessImpl instanceof MCRRuleAccessInterface) {
177 MCRRuleAccessInterface ruleAccessImpl = (MCRRuleAccessInterface) accessImpl;
178 Collection<String> configuredPermissions = ruleAccessImpl.getAccessPermissionsFromConfiguration();
179 for (String permission : configuredPermissions) {
180 MCRAccessManager.addRule(derivateID, permission, MCRAccessManager.getTrueRule(),
181 "default derivate rule");
182 }
183 }
184 }
185 }
186
187 private void getOrCreateRootDirectory() throws FileSystemException {
188 this.rootDir = MCRPath.getPath(derivateID, "/");
189 if (Files.notExists(rootDir)) {
190 rootDir.getFileSystem().createRoot(derivateID);
191 }
192 }
193
194 @Override
195 public boolean acceptFile(String path, String checksum, long length) throws Exception {
196 LOGGER.debug("incoming acceptFile request: {} {} {} bytes", path, checksum, length);
197 boolean shouldAcceptFile = true;
198 if (rootDir != null) {
199 MCRPath child = MCRPath.toMCRPath(rootDir.resolve(path));
200 if (Files.isRegularFile(child)) {
201 @SuppressWarnings("rawtypes")
202 MCRFileAttributes attrs = Files.readAttributes(child, MCRFileAttributes.class);
203 shouldAcceptFile = attrs.size() != length
204 || !(checksum.equals(attrs.md5sum()) && child.getFileSystem().verifies(child, attrs));
205 }
206 }
207 LOGGER.debug("Should the client send this file? {}", shouldAcceptFile);
208 return shouldAcceptFile;
209 }
210
211 @Override
212 public synchronized long receiveFile(String path, InputStream in, long length, String checksum)
213 throws IOException, MCRPersistenceException, MCRAccessException {
214 LOGGER.debug("incoming receiveFile request: {} {} {} bytes", path, checksum, length);
215
216 this.setProgressText(path);
217
218 List<Path> tempFiles = new LinkedList<>();
219 Supplier<Path> tempFileSupplier = () -> {
220 try {
221 Path tempFile = Files.createTempFile(derivateID + "-" + path.hashCode(), ".upload");
222 tempFiles.add(tempFile);
223 return tempFile;
224 } catch (IOException e) {
225 throw new UncheckedIOException("Error while creating temp File!", e);
226 }
227 };
228
229 try (InputStream fIn = preprocessInputStream(path, in, length, tempFileSupplier)) {
230 if (rootDir == null) {
231
232 prepareUpload();
233 }
234 MCRPath file = getFile(path);
235 LOGGER.info("Creating file {}.", file);
236 Files.copy(fIn, file, StandardCopyOption.REPLACE_EXISTING);
237 return tempFiles.isEmpty() ? length : Files.size(tempFiles.stream().reduce((a, b) -> b).get());
238 } finally {
239 tempFiles.stream().filter(Files::exists).forEach((tempFilePath) -> {
240 try {
241 Files.delete(tempFilePath);
242 } catch (IOException e) {
243 LOGGER.error("Could not delete temp file {}", tempFilePath);
244 }
245 });
246 this.filesUploaded++;
247 int progress = (int) (((float) this.filesUploaded / (float) getNumFiles()) * 100f);
248 this.setProgress(progress);
249 }
250 }
251
252 private InputStream preprocessInputStream(String path, InputStream in, long length, Supplier<Path> tempFileSupplier)
253 throws IOException {
254 List<MCRPostUploadFileProcessor> activeProcessors = FILE_PROCESSORS.stream().filter(p -> p.isProcessable(path))
255 .collect(Collectors.toList());
256 if (activeProcessors.isEmpty()) {
257 return in;
258 }
259 Path currentTempFile = tempFileSupplier.get();
260 try (InputStream initialIS = in) {
261 Files.copy(initialIS, currentTempFile, StandardCopyOption.REPLACE_EXISTING);
262 }
263 long myLength = Files.size(currentTempFile);
264 if (length >= 0 && length != myLength) {
265 throw new IOException("Length of transmitted data does not match promised length: " + myLength + "!="
266 + length);
267 }
268 for (MCRPostUploadFileProcessor pufp : activeProcessors) {
269 currentTempFile = pufp.processFile(path, currentTempFile, tempFileSupplier);
270 }
271 return Files.newInputStream(currentTempFile);
272 }
273
274 private MCRPath getFile(String path) throws IOException {
275 MCRPath pathToFile = MCRPath.toMCRPath(rootDir.resolve(path));
276 MCRPath parentDirectory = pathToFile.getParent();
277 if (!Files.isDirectory(parentDirectory)) {
278 Files.createDirectories(parentDirectory);
279 }
280 return pathToFile;
281 }
282
283 @Override
284 public synchronized void finishUpload() throws IOException, MCRAccessException {
285 if (this.derivate == null) {
286 return;
287 }
288 try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(rootDir)) {
289 if (dirStream.iterator().hasNext()) {
290 updateMainFile();
291 } else {
292 throw new IllegalStateException(
293 "No files were uploaded, delete entry in database for " + derivate.getId() + "!");
294 }
295 }
296 this.setStatus(MCRProcessableStatus.successful);
297 }
298
299 private void updateMainFile() throws IOException, MCRAccessException {
300 String mainFile = derivate.getDerivate().getInternals().getMainDoc();
301 MCRObjectDerivate der = MCRMetadataManager.retrieveMCRDerivate(getOrCreateDerivateID()).getDerivate();
302 boolean hasNoMainFile = ((der.getInternals().getMainDoc() == null) || (der.getInternals().getMainDoc().trim()
303 .isEmpty()));
304 if ((mainFile == null) || mainFile.trim().isEmpty() && hasNoMainFile) {
305 mainFile = getPathOfMainFile();
306 LOGGER.debug("Setting main file to {}", mainFile);
307 derivate.getDerivate().getInternals().setMainDoc(mainFile);
308 MCRMetadataManager.update(derivate);
309 }
310 }
311
312 protected String getPathOfMainFile() throws IOException {
313 MainFileFinder mainFileFinder = new MainFileFinder();
314 Files.walkFileTree(rootDir, mainFileFinder);
315 Path mainFile = mainFileFinder.getMainFile();
316 return mainFile == null ? "" : mainFile.subpath(0, mainFile.getNameCount()).toString();
317 }
318
319 public String getDerivateID() {
320 return derivateID;
321 }
322
323 public String getDocumentID() {
324 return documentID;
325 }
326
327 private static class MainFileFinder extends SimpleFileVisitor<Path> {
328
329 private Path mainFile;
330
331 @Override
332 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
333 if (mainFile != null) {
334 if (mainFile.getFileName().compareTo(file.getFileName()) > 0) {
335 mainFile = file;
336 }
337 } else {
338 mainFile = file;
339 }
340 return super.visitFile(file, attrs);
341 }
342
343 @Override
344 public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
345 FileVisitResult result = super.postVisitDirectory(dir, exc);
346 if (mainFile != null) {
347 return FileVisitResult.TERMINATE;
348 }
349 return result;
350 }
351
352 public Path getMainFile() {
353 return mainFile;
354 }
355 }
356 }