GH-3708: Inline parquet.thrift

dev/update-parquet-thrift.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Updates the inlined parquet.thrift and sidecar to a specific parquet-format
+# commit or tag.
+#
+# Usage:
+#   update-parquet-thrift.sh <ref>
+#
+#   <ref> is a full 40-char parquet-format commit SHA, or a parquet-format tag
+#   (e.g. apache-parquet-format-2.13.0).
+#
+#   Set PARQUET_FORMAT_REPO to pull from a fork instead of apache/parquet-format,
+#   e.g. PARQUET_FORMAT_REPO=https://github.com/<user>/parquet-format.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+THRIFT_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet.thrift"
+SIDECAR_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet-format.version"
+
+# Source repo, overridable via the PARQUET_FORMAT_REPO env var to pull from a fork.
+# The raw-download host is derived from it (assumes a GitHub-hosted repo).
+PARQUET_FORMAT_REPO="${PARQUET_FORMAT_REPO:-https://github.com/apache/parquet-format}"
+PARQUET_FORMAT_RAW="${PARQUET_FORMAT_REPO/github.com/raw.githubusercontent.com}"
+THRIFT_PATH_IN_FORMAT="src/main/thrift/parquet.thrift"
+
+usage() {
+  cat <<EOF
+Usage: $0 <ref>
+  <ref>  a full 40-char parquet-format commit SHA, or a parquet-format tag
+         (e.g. apache-parquet-format-2.13.0)
+EOF
+  exit 1
+}
+
+[[ $# -ne 1 ]] && usage
+
+ref="$1"
+
+# Resolve ref to a full commit SHA.
+if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then
+  resolved_sha="$ref"
+else
+  echo "Resolving ${ref} ..."
+  ls_out="$(git ls-remote "$PARQUET_FORMAT_REPO" "$ref" "${ref}^{}")" || {
+    echo "ERROR: git ls-remote failed for '${ref}' in ${PARQUET_FORMAT_REPO}" >&2
+    exit 1
+  }
+  if [[ -z "$ls_out" ]]; then
+    echo "ERROR: ref '${ref}' not found in ${PARQUET_FORMAT_REPO}" >&2
+    exit 1
+  fi
+  resolved_sha="$(printf '%s\n' "$ls_out" \
+    | awk '{ if ($2 ~ /\^\{\}$/) deref=$1; else plain=$1 }
+           END { print (deref != "" ? deref : plain) }')"
+  if [[ -z "$resolved_sha" ]]; then
+    echo "ERROR: could not resolve commit SHA for '${ref}'" >&2
+    exit 1
+  fi
+  echo "  -> commit: ${resolved_sha}"
+fi
+
+# Read the old commit from the sidecar for the summary.
+old_commit="(none)"
+if [[ -f "$SIDECAR_FILE" ]]; then
+  old_commit="$(grep '^parquet-format.commit=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+fi
+
+# Fetch upstream parquet.thrift to a temp file. Only overwrite the inlined file after fetch and
+# validation succeed, so a failure never overwrites the tracked IDL.
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+
+echo "Fetching parquet.thrift at ${resolved_sha} ..."
+url="${PARQUET_FORMAT_RAW}/${resolved_sha}/${THRIFT_PATH_IN_FORMAT}"
+if ! curl -fsSL -o "$tmp" "$url"; then
+  echo "ERROR: could not fetch parquet.thrift at ${resolved_sha}" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+if [[ ! -s "$tmp" ]]; then
+  echo "ERROR: fetched parquet.thrift is empty" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+if ! grep -q 'namespace java org.apache.parquet.format' "$tmp"; then
+  echo "ERROR: fetched file does not look like parquet.thrift (missing namespace declaration)" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+# Write the validated thrift file, then write the sidecar.
+mv "$tmp" "$THRIFT_FILE"
+chmod 644 "$THRIFT_FILE"
+echo "  -> written to ${THRIFT_FILE}"
+
+cat > "$SIDECAR_FILE" <<SIDECAR
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=${resolved_sha}
+SIDECAR
+echo "  -> sidecar updated: ${SIDECAR_FILE}"
+
+# Summary
+echo ""
+echo "Update complete:"
+echo "  old commit: ${old_commit}"
+echo "  new commit: ${resolved_sha}"
+echo ""
+echo "Next steps: rebuild parquet-format-structures and review the diff:"
+echo "  ./mvnw -pl parquet-format-structures -am install -DskipTests"
+echo "  git diff parquet-format-structures/src/main/thrift/parquet.thrift"

parquet-format-structures/pom.xml

@@ -34,44 +34,14 @@
   <url>https://parquet.apache.org/</url>
   <description>Parquet-mr related java classes to use the parquet-format thrift structures.</description>
 
-  <properties>
-    <parquet.thrift.path>${project.build.directory}/parquet-format-thrift</parquet.thrift.path>
-  </properties>
-
   <build>
     <plugins>
-      <!-- Getting the parquet-format thrift file -->
-       <plugin>
-         <groupId>org.apache.maven.plugins</groupId>
-         <artifactId>maven-dependency-plugin</artifactId>
-         <executions>
-           <execution>
-             <id>unpack</id>
-             <phase>initialize</phase>
-             <goals>
-               <goal>unpack</goal>
-             </goals>
-             <configuration>
-               <artifactItems>
-                 <artifactItem>
-                   <groupId>org.apache.parquet</groupId>
-                   <artifactId>parquet-format</artifactId>
-                   <version>${parquet.format.version}</version>
-                   <type>jar</type>
-                 </artifactItem>
-               </artifactItems>
-               <includes>parquet.thrift</includes>
-               <outputDirectory>${parquet.thrift.path}</outputDirectory>
-             </configuration>
-           </execution>
-         </executions>
-       </plugin>
       <!-- thrift -->
       <plugin>
         <groupId>org.apache.thrift</groupId>
         <artifactId>thrift-maven-plugin</artifactId>
         <configuration>
-          <thriftSourceRoot>${parquet.thrift.path}</thriftSourceRoot>
+          <thriftSourceRoot>${project.basedir}/src/main/thrift</thriftSourceRoot>
           <thriftExecutable>${format.thrift.executable}</thriftExecutable>
         </configuration>
         <executions>

parquet-format-structures/src/main/thrift/parquet-format.version

@@ -0,0 +1,2 @@
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=c47e2a66e88943fc46fde1b028a9432f14fdf5c0