You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sling.apache.org by dk...@apache.org on 2023/01/24 03:31:36 UTC
[sling-org-apache-sling-app-cms] branch master updated: SLING-11760 - Improving the metadata parsing
This is an automated email from the ASF dual-hosted git repository.
dklco pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sling-org-apache-sling-app-cms.git
The following commit(s) were added to refs/heads/master by this push:
new 7982ca65 SLING-11760 - Improving the metadata parsing
7982ca65 is described below
commit 7982ca65de69e25056cb447f0b76fb118772c305
Author: Dan Klco <kl...@adobe.com>
AuthorDate: Mon Jan 23 22:31:29 2023 -0500
SLING-11760 - Improving the metadata parsing
---
.../core/internal/FileMetadataExtractorImpl.java | 92 ++++++++++++---
.../internal/FileMetadataExtractorImplTest.java | 124 ++++++++++++++++++++-
feature/src/main/features/cms/cms-repoinit.txt | 34 +++++-
3 files changed, 231 insertions(+), 19 deletions(-)
diff --git a/core/src/main/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImpl.java b/core/src/main/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImpl.java
index 9313d68e..48b14744 100644
--- a/core/src/main/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImpl.java
+++ b/core/src/main/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImpl.java
@@ -22,10 +22,19 @@ import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
+import javax.jcr.NamespaceRegistry;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.JcrConstants;
-import org.apache.jackrabbit.util.Text;
+import org.apache.sling.api.resource.LoginException;
import org.apache.sling.api.resource.ModifiableValueMap;
import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
+import org.apache.sling.api.resource.ResourceResolverFactory;
import org.apache.sling.cms.CMSConstants;
import org.apache.sling.cms.File;
import org.apache.sling.cms.FileMetadataExtractor;
@@ -36,7 +45,9 @@ import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
+import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
@@ -46,11 +57,18 @@ public class FileMetadataExtractorImpl implements FileMetadataExtractor {
private static final Logger log = LoggerFactory.getLogger(FileMetadataExtractorImpl.class);
+ private ResourceResolverFactory resolverFactory;
+
+ @Activate
+ public FileMetadataExtractorImpl(@Reference ResourceResolverFactory resolverFactory) {
+ this.resolverFactory = resolverFactory;
+ }
+
@Override
public Map<String, Object> extractMetadata(File file) throws IOException {
try {
return extractMetadata(file.getResource());
- } catch (SAXException | TikaException e) {
+ } catch (SAXException | TikaException | RepositoryException | LoginException e) {
throw new IOException("Failed to parse metadata", e);
}
}
@@ -80,6 +98,8 @@ public class FileMetadataExtractorImpl implements FileMetadataExtractor {
}
if (properties != null) {
properties.putAll(extractMetadata(file.getResource()));
+ properties.put("SHA1", generateSha1(resource));
+ resource.getResourceResolver().refresh();
if (metadata == null) {
resource.getResourceResolver().create(content, CMSConstants.NN_METADATA, properties);
}
@@ -90,30 +110,74 @@ public class FileMetadataExtractorImpl implements FileMetadataExtractor {
} else {
throw new IOException("Unable to update metadata for " + resource.getPath());
}
- } catch (SAXException | TikaException e) {
+ } catch (SAXException | TikaException | RepositoryException | LoginException e) {
throw new IOException("Failed to parse metadata", e);
}
+ }
+ public String generateSha1(Resource resource) throws IOException {
+ try (InputStream is = resource.adaptTo(InputStream.class)) {
+ String sha1 = DigestUtils.sha1Hex(is);
+ log.info("Generated SHA {} for {}", sha1, resource.getPath());
+ return sha1;
+ }
}
- public Map<String, Object> extractMetadata(Resource resource) throws IOException, SAXException, TikaException {
+ public Map<String, Object> extractMetadata(Resource resource)
+ throws IOException, SAXException, TikaException, RepositoryException, LoginException {
log.info("Extracting metadata from {}", resource.getPath());
- InputStream is = resource.adaptTo(InputStream.class);
Map<String, Object> properties = new HashMap<>();
- Parser parser = new AutoDetectParser();
- BodyContentHandler handler = new BodyContentHandler();
- Metadata md = new Metadata();
- ParseContext context = new ParseContext();
- parser.parse(is, handler, md, context);
- for (String name : md.names()) {
- putMetadata(properties, name, md);
+ try (InputStream is = resource.adaptTo(InputStream.class)) {
+ Parser parser = new AutoDetectParser();
+ BodyContentHandler handler = new BodyContentHandler();
+ Metadata md = new Metadata();
+ ParseContext context = new ParseContext();
+ try {
+ parser.parse(is, handler, md, context);
+ } catch (SAXException se) {
+ if ("WriteLimitReachedException".equals(se.getClass().getSimpleName())) {
+ log.info("Write limit reached for {}", resource.getPath());
+ } else {
+ throw se;
+ }
+ }
+
+ try (ResourceResolver adminResolver = resolverFactory.getAdministrativeResourceResolver(null)) {
+ NamespaceRegistry registry = adminResolver.adaptTo(Session.class).getWorkspace().getNamespaceRegistry();
+ for (String name : md.names()) {
+ putMetadata(properties, name, md, registry);
+ }
+ }
}
return properties;
}
- private void putMetadata(Map<String, Object> properties, String name, Metadata metadata) {
+ protected String formatKey(String initialKey, NamespaceRegistry registry)
+ throws RepositoryException {
+ String namespace = null;
+ String key = null;
+ if (initialKey.contains(":")) {
+ namespace = StringUtils.substringBefore(initialKey, ":");
+ key = StringUtils.substringAfter(initialKey, ":");
+ } else {
+ key = initialKey;
+ }
+ key = key.replace(" ", "").replace("/", "-");
+ if (namespace != null) {
+ namespace = namespace.replace(" ", "").replace("/", "-");
+ if (!ArrayUtils.contains(registry.getPrefixes(), namespace)) {
+ registry.registerNamespace(namespace, "http://sling.apache.org/cms/ns/" + namespace);
+ }
+ return namespace + ":" + key;
+ } else {
+ return key;
+ }
+ }
+
+ private void putMetadata(Map<String, Object> properties, String name, Metadata metadata, NamespaceRegistry registry)
+ throws RepositoryException {
log.trace("Updating property: {}", name);
- String filtered = Text.escapeIllegalJcrChars(name);
+ String filtered = formatKey(name, registry);
Property property = Property.get(name);
if (property != null) {
if (metadata.isMultiValued(property)) {
diff --git a/core/src/test/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImplTest.java b/core/src/test/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImplTest.java
index 9ddf32fb..3600c69d 100644
--- a/core/src/test/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImplTest.java
+++ b/core/src/test/java/org/apache/sling/cms/core/internal/FileMetadataExtractorImplTest.java
@@ -16,16 +16,31 @@
*/
package org.apache.sling.cms.core.internal;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Iterator;
import java.util.Map;
+import java.util.stream.Stream;
+import javax.jcr.NamespaceRegistry;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Workspace;
+
+import org.apache.sling.api.resource.LoginException;
import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
+import org.apache.sling.api.resource.ResourceResolverFactory;
import org.apache.sling.cms.File;
-import org.apache.sling.cms.FileMetadataExtractor;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -38,20 +53,38 @@ public class FileMetadataExtractorImplTest {
private File file;
+ private NamespaceRegistry registry;
+
+ private ResourceResolverFactory resolverFactory;
+
+ private FileMetadataExtractorImpl extractor;
+
@Before
- public void init() {
+ public void init() throws RepositoryException, LoginException {
+
+ ResourceResolver resolver = mock(ResourceResolver.class);
- Resource resource = Mockito.mock(Resource.class);
+ resolverFactory = mock(ResourceResolverFactory.class);
+ when(resolverFactory.getAdministrativeResourceResolver(null)).thenReturn(resolver);
+ registry = mock(NamespaceRegistry.class);
+
+ Workspace workspace = mock(Workspace.class);
+ when(workspace.getNamespaceRegistry()).thenReturn(registry);
+ Session session = mock(Session.class);
+ when(session.getWorkspace()).thenReturn(workspace);
+ when(resolver.adaptTo(Session.class)).thenReturn(session);
+
+ Resource resource = mock(Resource.class);
Mockito.when(resource.adaptTo(InputStream.class))
.thenReturn(FileMetadataExtractorImplTest.class.getClassLoader().getResourceAsStream("apache.png"));
file = Mockito.mock(File.class);
Mockito.when(file.getResource()).thenReturn(resource);
+ extractor = new FileMetadataExtractorImpl(resolverFactory);
}
@Test
public void testExtractMetadata() throws IOException {
- FileMetadataExtractor extractor = new FileMetadataExtractorImpl();
Map<String, Object> metadata = extractor.extractMetadata(file);
assertNotNull(metadata);
@@ -59,4 +92,87 @@ public class FileMetadataExtractorImplTest {
log.info("Extracted metadata: {}", metadata);
}
+
+ @Test
+ public void testFormatKey() throws IOException, RepositoryException {
+
+ when(registry.getPrefixes()).thenReturn(new String[] { "GPS", "ExifSubIFD", "ExifIFD0", "tiff", "dcterms" });
+
+ String[] keys = new String[] {
+ "GPS:GPS Img Direction", "Exif SubIFD:Subject Distance Range", "Compression Type",
+ "Number of Components", "Component 2", "Component 1", "Exif IFD0:X Resolution", "tiff:ResolutionUnit",
+ "Exif SubIFD:Scene Type", "Exif SubIFD:Exposure Mode", "tiff:Make", "Exif SubIFD:Sharpness",
+ "Exif SubIFD:Custom Rendered", "Component 3", "Exif SubIFD:Components Configuration",
+ "Exif SubIFD:Metering Mode", "Exif SubIFD:White Balance Mode", "tiff:BitsPerSample",
+ "Exif SubIFD:Sub-Sec Time Original", "meta:creation-date", "Creation-Date", "tiff:Orientation",
+ "tiff:Software", "geo:long", "Exif SubIFD:Digital Zoom Ratio", "tiff:YResolution", "Y Resolution",
+ "Exif SubIFD:Flash", "Thumbnail Height Pixels", "Last-Modified", "Exif SubIFD:Sub-Sec Time",
+ "exif:ExposureTime", "File Size", "Exif SubIFD:Exif Version", "GPS:GPS Img Direction Ref",
+ "Exif SubIFD:Focal Length", "Exif IFD0:Resolution Unit", "Exif SubIFD:Lens Model",
+ "Exif SubIFD:Date/Time Original", "Exif SubIFD:Sub-Sec Time Digitized", "Resolution Units",
+ "File Modified Date", "Exif SubIFD:Sensing Method", "Image Height", "Thumbnail Width Pixels",
+ "GPS:GPS Longitude", "Exif SubIFD:Time Zone Original", "GPS:GPS Longitude Ref", "tiff:Model",
+ "Exif SubIFD:Brightness Value", "exif:IsoSpeedRatings", "Exif SubIFD:Exposure Program",
+ "Exif IFD0:Make", "GPS:GPS Altitude Ref", "Exif SubIFD:Aperture Value",
+ "Exif SubIFD:Date/Time Digitized", "tiff:ImageWidth", "GPS:GPS Altitude", "Exif IFD0:Y Resolution",
+ "date", "Exif SubIFD:ISO Speed Ratings", "Number of Tables", "Exif SubIFD:Time Zone Digitized",
+ "Exif SubIFD:Exif Image Width", "Exif SubIFD:Contrast", "X Resolution",
+ "Exif SubIFD:Exposure Bias Value", "Exif SubIFD:Saturation", "modified", "exif:FNumber",
+ "Exif SubIFD:Shutter Speed Value", "Exif IFD0:Orientation", "Exif SubIFD:F-Number", "exif:FocalLength",
+ "Exif IFD0:Software", "Exif IFD0:Date/Time", "Exif SubIFD:Scene Capture Type", "Exif SubIFD:Time Zone",
+ "geo:lat", "Data Precision", "tiff:ImageLength", "Exif IFD0:Model", "dcterms:created",
+ "dcterms:modified", "exif:Flash", "Last-Save-Date", "Exif SubIFD:Color Space",
+ "Exif SubIFD:Focal Length 35", "Exif SubIFD:Exposure Time", "meta:save-date", "File Name",
+ "GPS:GPS Latitude Ref", "Content-Type", "X-Parsed-By", "Exif SubIFD:Max Aperture Value",
+ "tiff:XResolution", "exif:DateTimeOriginal", "Exif SubIFD:Subject Distance",
+ "Exif SubIFD:FlashPix Version", "Exif SubIFD:Exif Image Height", "Image Width", "GPS:GPS Latitude",
+ "Exif SubIFD:Lens Make", "GPS:GPS Date Stamp" };
+
+ String[] formatted = new String[keys.length];
+ for (int i = 0; i < keys.length; i++) {
+ formatted[i] = extractor.formatKey(keys[i], registry);
+ }
+
+ String[] expected = new String[] {
+ "GPS:GPSImgDirection", "ExifSubIFD:SubjectDistanceRange", "CompressionType",
+ "NumberofComponents", "Component2", "Component1", "ExifIFD0:XResolution", "tiff:ResolutionUnit",
+ "ExifSubIFD:SceneType", "ExifSubIFD:ExposureMode", "tiff:Make", "ExifSubIFD:Sharpness",
+ "ExifSubIFD:CustomRendered", "Component3", "ExifSubIFD:ComponentsConfiguration",
+ "ExifSubIFD:MeteringMode", "ExifSubIFD:WhiteBalanceMode", "tiff:BitsPerSample",
+ "ExifSubIFD:Sub-SecTimeOriginal", "meta:creation-date", "Creation-Date", "tiff:Orientation",
+ "tiff:Software", "geo:long", "ExifSubIFD:DigitalZoomRatio", "tiff:YResolution", "YResolution",
+ "ExifSubIFD:Flash", "ThumbnailHeightPixels", "Last-Modified", "ExifSubIFD:Sub-SecTime",
+ "exif:ExposureTime", "FileSize", "ExifSubIFD:ExifVersion", "GPS:GPSImgDirectionRef",
+ "ExifSubIFD:FocalLength", "ExifIFD0:ResolutionUnit", "ExifSubIFD:LensModel",
+ "ExifSubIFD:Date-TimeOriginal", "ExifSubIFD:Sub-SecTimeDigitized", "ResolutionUnits",
+ "FileModifiedDate", "ExifSubIFD:SensingMethod", "ImageHeight", "ThumbnailWidthPixels",
+ "GPS:GPSLongitude", "ExifSubIFD:TimeZoneOriginal", "GPS:GPSLongitudeRef", "tiff:Model",
+ "ExifSubIFD:BrightnessValue", "exif:IsoSpeedRatings", "ExifSubIFD:ExposureProgram",
+ "ExifIFD0:Make", "GPS:GPSAltitudeRef", "ExifSubIFD:ApertureValue",
+ "ExifSubIFD:Date-TimeDigitized", "tiff:ImageWidth", "GPS:GPSAltitude", "ExifIFD0:YResolution",
+ "date", "ExifSubIFD:ISOSpeedRatings", "NumberofTables", "ExifSubIFD:TimeZoneDigitized",
+ "ExifSubIFD:ExifImageWidth", "ExifSubIFD:Contrast", "XResolution",
+ "ExifSubIFD:ExposureBiasValue", "ExifSubIFD:Saturation", "modified", "exif:FNumber",
+ "ExifSubIFD:ShutterSpeedValue", "ExifIFD0:Orientation", "ExifSubIFD:F-Number", "exif:FocalLength",
+ "ExifIFD0:Software", "ExifIFD0:Date-Time", "ExifSubIFD:SceneCaptureType", "ExifSubIFD:TimeZone",
+ "geo:lat", "DataPrecision", "tiff:ImageLength", "ExifIFD0:Model", "dcterms:created",
+ "dcterms:modified", "exif:Flash", "Last-Save-Date", "ExifSubIFD:ColorSpace",
+ "ExifSubIFD:FocalLength35", "ExifSubIFD:ExposureTime", "meta:save-date", "FileName",
+ "GPS:GPSLatitudeRef", "Content-Type", "X-Parsed-By", "ExifSubIFD:MaxApertureValue",
+ "tiff:XResolution", "exif:DateTimeOriginal", "ExifSubIFD:SubjectDistance",
+ "ExifSubIFD:FlashPixVersion", "ExifSubIFD:ExifImageHeight", "ImageWidth", "GPS:GPSLatitude",
+ "ExifSubIFD:LensMake", "GPS:GPSDateStamp" };
+ assertArrayEquals(expected, formatted);
+ }
+
+ @Test
+ public void willRegisterNamespace() throws RepositoryException {
+ when(registry.getPrefixes()).thenReturn(new String[] { "GPS" });
+ Iterator<String> keys = Stream
+ .of("GPS:GPSLatitudeRef", "Content-Type", "X-Parsed-By", "ExifSubIFD:MaxApertureValue").iterator();
+ while (keys.hasNext()) {
+ extractor.formatKey(keys.next(), registry);
+ }
+ verify(registry).registerNamespace(eq("ExifSubIFD"), anyString());
+ }
}
diff --git a/feature/src/main/features/cms/cms-repoinit.txt b/feature/src/main/features/cms/cms-repoinit.txt
index 53187c86..03ed22a0 100644
--- a/feature/src/main/features/cms/cms-repoinit.txt
+++ b/feature/src/main/features/cms/cms-repoinit.txt
@@ -89,4 +89,36 @@ create service user sling-cms-versionmgr
set ACL for sling-cms-versionmgr
allow jcr:write,jcr:nodeTypeManagement,jcr:versionManagement on /
allow jcr:read on /jcr:system/jcr:versionStorage
-end
\ No newline at end of file
+end
+
+# Namespaces
+register namespace (acdsee) "http://ns.acdsee.com/iptc/1.0/"
+register namespace (album) "http://ns.adobe.com/album/1.0/"
+register namespace (cc) "http://creativecommons.org/ns#"
+register namespace (crs) "http://ns.adobe.com/camera-raw-settings/1.0/"
+register namespace (dc) "http://purl.org/dc/elements/1.1/"
+register namespace (dcterms) "http://purl.org/dc/terms/"
+register namespace (Iptc4xmpCore) "http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/"
+register namespace (lr) "http://ns.adobe.com/lightroom/1.0/"
+register namespace (mediapro) "http://ns.iview-multimedia.com/mediapro/1.0/"
+register namespace (MP) "http://ns.microsoft.com/photo/1.2/"
+register namespace (photoshop) "http://ns.adobe.com/photoshop/1.0/"
+register namespace (plus) "http://ns.useplus.org/ldf/xmp/1.0/"
+register namespace (pdf) "http://ns.adobe.com/pdf/1.3/"
+register namespace (pdfx) "http://ns.adobe.com/pdfx/1.3/"
+register namespace (prism) "http://prismstandard.org/namespaces/basic/2.1/"
+register namespace (prl) "http://prismstandard.org/namespaces/prl/2.1/"
+register namespace (psAux) "http://ns.adobe.com/exif/1.0/aux/"
+register namespace (rdf) "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+register namespace (stEvt) "http://ns.adobe.com/xap/1.0/sType/ResourceEvent#"
+register namespace (stEvt) "http://ns.adobe.com/xap/1.0/sType/ResourceEvent#"
+register namespace (tiff) "http://ns.adobe.com/tiff/1.0/"
+register namespace (xmp) "http://ns.adobe.com/xap/1.0/"
+register namespace (xmpBJ) "http://ns.adobe.com/xap/1.0/bj/"
+register namespace (xmpDM) "http://ns.adobe.com/xmp/1.0/DynamicMedia/"
+register namespace (xmpMM) "http://ns.adobe.com/xap/1.0/mm/"
+register namespace (xmpNote) "http://ns.adobe.com/xmp/note/"
+register namespace (xmpPLUS) "http://ns.adobe.com/xap/1.0/PLUS/"
+register namespace (xmpRights) "http://ns.adobe.com/xap/1.0/rights/"
+register namespace (xmpTPg) "http://ns.adobe.com/xap/1.0/t/pg/"
+