#78 Support geospatial statistics and bounding box metadata

Author: mghildiy

core/src/main/java/dev/hardwood/internal/thrift/BoundingBoxReader.java

@@ -0,0 +1,81 @@
+package dev.hardwood.internal.thrift;
+
+import java.io.IOException;
+
+import dev.hardwood.metadata.BoundingBox;
+
+/// Reader for the Thrift BoundingBox struct from Parquet metadata.
+public class BoundingBoxReader {
+    public static BoundingBox read(ThriftCompactReader reader) throws IOException {
+        short saved = reader.pushFieldIdContext();
+        try {
+            return readInternal(reader);
+        }
+        finally {
+            reader.popFieldIdContext(saved);
+        }
+    }
+
+    private static BoundingBox readInternal(ThriftCompactReader reader) throws IOException {
+        Double xmin = null;
+        Double xmax = null;
+        Double ymin = null;
+        Double ymax = null;
+        Double zmin = null;
+        Double zmax = null;
+        Double mmin = null;
+        Double mmax = null;
+        while(true) {
+            ThriftCompactReader.FieldHeader header = reader.readFieldHeader();
+            if (header == null) {
+                break;
+            }
+
+            switch (header.fieldId()) {
+                case 1:
+                    xmin = readField(header, reader);
+                    break;
+                case 2:
+                    xmax = readField(header, reader);
+                    break;
+                case 3:
+                    ymin = readField(header, reader);
+                    break;
+                case 4:
+                    ymax = readField(header, reader);
+                    break;
+                case 5:
+                    zmin = readField(header, reader);
+                    break;
+                case 6:
+                    zmax = readField(header, reader);
+                    break;
+                case 7:
+                    mmin = readField(header, reader);
+                    break;
+                case 8:
+                    mmax = readField(header, reader);
+                    break;
+                default:
+                    reader.skipField(header.type());
+                    break;
+            }
+        }
+        if(xmin == null || xmax == null || ymin == null || ymax == null)
+            return null;
+        else
+            return new BoundingBox(xmin, xmax, ymin, ymax, zmin, zmax, mmin, mmax);
+    }
+
+    private static Double readField(ThriftCompactReader.FieldHeader header, ThriftCompactReader reader) throws IOException {
+        Double value = null;
+        if(header.type() == 0x07) {
+            value = reader.readDouble();
+        }
+        else {
+            reader.skipField(header.type());
+        }
+
+        return value;
+    }
+}

core/src/main/java/dev/hardwood/internal/thrift/ColumnMetaDataReader.java

@@ -17,6 +17,7 @@
 import dev.hardwood.metadata.CompressionCodec;
 import dev.hardwood.metadata.Encoding;
 import dev.hardwood.metadata.FieldPath;
+import dev.hardwood.metadata.GeospatialStatistics;
 import dev.hardwood.metadata.PhysicalType;
 import dev.hardwood.metadata.Statistics;
 
@@ -45,6 +46,7 @@ private static ColumnMetaData readInternal(ThriftCompactReader reader) throws IO
         long dataPageOffset = 0;
         Long dictionaryPageOffset = null;
         Statistics statistics = null;
+        GeospatialStatistics geospatialStatistics = null;
 
         while (true) {
             ThriftCompactReader.FieldHeader header = reader.readFieldHeader();
@@ -150,6 +152,14 @@ private static ColumnMetaData readInternal(ThriftCompactReader reader) throws IO
                         reader.skipField(header.type());
                     }
                     break;
+                case 17: // geospatial statistics (optional)
+                    if(header.type() == 0x0C) {
+                        geospatialStatistics = GeospatialStatisticsReader.read(reader);
+                    }
+                    else {
+                        reader.skipField(header.type());
+                    }
+                    break;
                 default:
                     reader.skipField(header.type());
                     break;
@@ -158,6 +168,6 @@ private static ColumnMetaData readInternal(ThriftCompactReader reader) throws IO
 
         return new ColumnMetaData(type, encodings, new FieldPath(List.copyOf(pathInSchema)), codec, numValues,
                 totalUncompressedSize, totalCompressedSize, keyValueMetadata, dataPageOffset, dictionaryPageOffset,
-                statistics);
+                statistics, geospatialStatistics);
     }
 }

core/src/main/java/dev/hardwood/internal/thrift/GeospatialStatisticsReader.java

@@ -0,0 +1,61 @@
+package dev.hardwood.internal.thrift;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import dev.hardwood.metadata.BoundingBox;
+import dev.hardwood.metadata.GeospatialStatistics;
+
+/// Reader for the Thrift GeospatialStatistics struct from Parquet metadata.
+public class GeospatialStatisticsReader {
+
+    public static GeospatialStatistics read(ThriftCompactReader reader) throws IOException {
+        short saved = reader.pushFieldIdContext();
+        try {
+            return readInternal(reader);
+        }
+        finally {
+            reader.popFieldIdContext(saved);
+        }
+    }
+
+    private static GeospatialStatistics readInternal(ThriftCompactReader reader) throws IOException {
+        BoundingBox bbox = null;
+        List<Integer> geospatialTypes = new ArrayList<>();
+
+        while(true) {
+            ThriftCompactReader.FieldHeader header = reader.readFieldHeader();
+            if (header == null) {
+                break;
+            }
+
+            switch (header.fieldId()) {
+                case 1:
+                    if(header.type() == 0x0C) {
+                        bbox = BoundingBoxReader.read(reader);
+                    }
+                    else {
+                        reader.skipField(header.type());
+                    }
+                    break;
+                case 2:
+                    if(header.type() == 0x09) {
+                        ThriftCompactReader.CollectionHeader listHeader = reader.readListHeader();
+                        for (int i = 0; i < listHeader.size(); i++) {
+                            geospatialTypes.add(reader.readI32());
+                        }
+                    }
+                    else {
+                        reader.skipField(header.type());
+                    }
+                    break;
+                default:
+                    reader.skipField(header.type());
+                    break;
+            }
+        }
+
+        return new GeospatialStatistics(bbox, List.copyOf(geospatialTypes));
+    }
+}

core/src/main/java/dev/hardwood/metadata/BoundingBox.java

@@ -0,0 +1,21 @@
+package dev.hardwood.metadata;
+
+/// @param xmin x coordinate of bottom-left vertex
+/// @param ymin y coordinate of bottom-left vertex
+/// @param xmax x coordinate of top-right vertex
+/// @param ymax y coordinate of top-right vertex
+/// @param zmin minimum height of bounded volume, or `null` if absent
+/// @param zmax maximum height of bounded volume, or `null` if absent
+/// @param mmin minimum of a value in 4th dimension, or `null` if absent
+/// @param mmax maximum of a value in 4th dimension, or `null` if absent
+/// @see <a href="https://github.com/apache/parquet-format/blob/master/Geospatial.md#bounding-box">Geospatial - bounding-box</a>
+public record BoundingBox(
+        Double xmin,
+        Double xmax,
+        Double ymin,
+        Double ymax,
+        Double zmin,
+        Double zmax,
+        Double mmin,
+        Double mmax) {
+}

core/src/main/java/dev/hardwood/metadata/ColumnMetaData.java

@@ -23,6 +23,7 @@
 /// @param dataPageOffset byte offset in the file where the first data page begins
 /// @param dictionaryPageOffset byte offset in the file where the dictionary page begins, or `null` if there is no dictionary page
 /// @param statistics column chunk statistics (min/max values, null count, distinct count), or `null` if absent
+/// @param geospatialStatistics column chunk geospatial statistics (bounding box, geospatial types), or `null` if absent
 /// @see <a href="https://parquet.apache.org/docs/file-format/data-pages/columnchunks/">File Format – Column Chunks</a>
 /// @see <a href="https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift">parquet.thrift</a>
 public record ColumnMetaData(
@@ -36,5 +37,6 @@ public record ColumnMetaData(
         Map<String, String> keyValueMetadata,
         long dataPageOffset,
         Long dictionaryPageOffset,
-        Statistics statistics) {
+        Statistics statistics,
+        GeospatialStatistics geospatialStatistics) {
 }

core/src/main/java/dev/hardwood/metadata/GeospatialStatistics.java

@@ -0,0 +1,13 @@
+package dev.hardwood.metadata;
+
+import java.util.List;
+
+/// @param bbox bounding box, or `null` if absent
+/// @param geospatialTypes list of geospatial type for geometry/geography column, empty list if not known
+/// @see <a href="https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics">Geospatial – statistics</a>
+/// @see <a href="https://github.com/apache/parquet-format/blob/master/Geospatial.md#geospatial-types">Geospatial - types</a>
+public record GeospatialStatistics(
+        BoundingBox bbox,
+        List<Integer> geospatialTypes
+) {
+}

core/src/test/java/dev/hardwood/FilterPredicateTest.java

@@ -410,15 +410,15 @@ private static RowGroup createRowGroupWithStats(PhysicalType type, byte[] min, b
         Statistics stats = new Statistics(min, max, 0L, null, false);
         ColumnMetaData cmd = new ColumnMetaData(
                 type, List.of(Encoding.PLAIN), FieldPath.of("col"),
-                CompressionCodec.UNCOMPRESSED, 100, 1000, 1000, Map.of(), 0, null, stats);
+                CompressionCodec.UNCOMPRESSED, 100, 1000, 1000, Map.of(), 0, null, stats, null);
         ColumnChunk chunk = new ColumnChunk(cmd, null, null, null, null);
         return new RowGroup(List.of(chunk), 1000, 100);
     }
 
     private static RowGroup createRowGroupWithoutStatistics() {
         ColumnMetaData cmd = new ColumnMetaData(
                 PhysicalType.INT32, List.of(Encoding.PLAIN), FieldPath.of("col"),
-                CompressionCodec.UNCOMPRESSED, 100, 1000, 1000, Map.of(), 0, null, null);
+                CompressionCodec.UNCOMPRESSED, 100, 1000, 1000, Map.of(), 0, null, null, null);
         ColumnChunk chunk = new ColumnChunk(cmd, null, null, null, null);
         return new RowGroup(List.of(chunk), 1000, 100);
     }

core/src/test/java/dev/hardwood/GeospatialStatisticsTest.java

@@ -0,0 +1,41 @@
+package dev.hardwood;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.Test;
+
+import dev.hardwood.metadata.*;
+import dev.hardwood.reader.ParquetFileReader;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class GeospatialStatisticsTest {
+
+    @Test
+    public void tesGeospatialStatisticsAreReadCorrectly() throws Exception {
+        Path parquetFile = Paths.get("src/test/resources/geospatial_stats_test.parquet");
+
+        try (ParquetFileReader reader = ParquetFileReader.open(InputFile.of(parquetFile))) {
+            FileMetaData metadata = reader.getFileMetaData();
+            assertThat(metadata.rowGroups()).hasSize(1);
+
+            RowGroup rg0 = metadata.rowGroups().get(0);
+            assertThat(rg0.columns().get(0).metaData().geospatialStatistics()).isNull();
+            ColumnChunk cc = rg0.columns().get(1);
+            ColumnMetaData md = cc.metaData();
+            GeospatialStatistics geospatialStatistics =  md.geospatialStatistics();
+            assertThat(geospatialStatistics).isNotNull();
+            assertThat(geospatialStatistics.bbox().xmin()).isEqualTo(-4.0);
+            assertThat(geospatialStatistics.bbox().xmax()).isEqualTo(7.5);
+            assertThat(geospatialStatistics.bbox().ymin()).isEqualTo(20.96);
+            assertThat(geospatialStatistics.bbox().ymax()).isEqualTo(77.08);
+            assertThat(geospatialStatistics.bbox().zmin()).isEqualTo(10.5);
+            assertThat(geospatialStatistics.bbox().zmax()).isEqualTo(90.0);
+            assertThat(geospatialStatistics.bbox().mmin()).isNull();
+            assertThat(geospatialStatistics.bbox().mmax()).isNull();
+            assertThat(geospatialStatistics.geospatialTypes().size()).isEqualTo(2);
+            assertThat(geospatialStatistics.geospatialTypes()).containsExactly(1, 6);
+        }
+    }
+}

core/src/test/java/dev/hardwood/internal/reader/PageRangeTest.java

@@ -206,15 +206,15 @@ private static ColumnChunk chunkWithoutDict(long dataPageOffset) {
         ColumnMetaData meta = new ColumnMetaData(
                 PhysicalType.INT64, List.of(Encoding.PLAIN), FieldPath.of("col"),
                 CompressionCodec.UNCOMPRESSED, 100, 1000, 1000, Map.of(),
-                dataPageOffset, null, null);
+                dataPageOffset, null, null, null);
         return new ColumnChunk(meta, null, null, null, null);
     }
 
     private static ColumnChunk chunkWithDict(long dataPageOffset, long dictOffset) {
         ColumnMetaData meta = new ColumnMetaData(
                 PhysicalType.INT64, List.of(Encoding.PLAIN), FieldPath.of("col"),
                 CompressionCodec.UNCOMPRESSED, 100, 1000, 1000, Map.of(),
-                dataPageOffset, dictOffset, null);
+                dataPageOffset, dictOffset, null, null);
         return new ColumnChunk(meta, null, null, null, null);
     }