Merge branch 'apache:master' into master

.github/pull_request_template.md

@@ -1,5 +1,3 @@
-
-
 ## Did you read the Contributor Guide?
 
 - Yes, I have read the [Contributor Rules](https://sedona.apache.org/latest-snapshot/community/rule/) and [Contributor Development Guide](https://sedona.apache.org/latest-snapshot/community/develop/)

.github/workflows/docs.yml

@@ -54,12 +54,12 @@ jobs:
       - run: pip install mkdocs-git-revision-date-localized-plugin
       - run: pip install mike
       - run: sudo apt update
-      - uses: r-lib/actions/setup-r@v2.10.1
+      - uses: r-lib/actions/setup-r@v2.11.1
         with:
           r-version: release
           use-public-rspm: true
       - name: Query R dependencies
-        uses: r-lib/actions/setup-r-dependencies@v2.10.1
+        uses: r-lib/actions/setup-r-dependencies@v2.11.1
         with:
           cache: true
           extra-packages: |

.github/workflows/java.yml

@@ -81,6 +81,10 @@ jobs:
             scala: 2.12.15
             jdk: '11'
             skipTests: ''
+          - spark: 3.3.0
+            scala: 2.13.8
+            jdk: '8'
+            skipTests: ''
           - spark: 3.3.0
             scala: 2.12.15
             jdk: '8'

.github/workflows/python-wheel.yml

@@ -56,7 +56,7 @@ jobs:
         with:
           platforms: all
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.3
+        uses: pypa/cibuildwheel@v2.22.0
         env:
           CIBW_SKIP: 'pp* *musl*'
           CIBW_ARCHS_LINUX: 'x86_64 aarch64'

.github/workflows/r.yml

@@ -77,20 +77,20 @@ jobs:
           sudo apt-get -y remove --purge default-jdk adoptopenjdk-11-hotspot || :
         shell: bash
       - uses: actions/checkout@v4
-      - uses: r-lib/actions/setup-r@v2.10.1
+      - uses: r-lib/actions/setup-r@v2.11.1
         with:
           r-version: ${{ matrix.r }}
           use-public-rspm: true
       - name: Query R dependencies
-        uses: r-lib/actions/setup-r-dependencies@v2.10.1
+        uses: r-lib/actions/setup-r-dependencies@v2.11.1
         with:
           cache: true
           extra-packages: |
             any::testthat
             any::rcmdcheck
           working-directory : './R'
       - name: Build and check R package
-        uses: r-lib/actions/check-r-package@v2.10.1
+        uses: r-lib/actions/check-r-package@v2.11.1
         with:
           build_args: 'c("--no-build-vignettes", "--no-manual")'
           args: 'c("--no-build-vignettes", "--no-manual", "--no-tests")'

.pre-commit-config.yaml

@@ -30,6 +30,25 @@ repos:
   - repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.5.5
     hooks:
+      - id: insert-license
+        name: Add license for all Markdown files
+        files: \.md$
+        args:
+          - --comment-style
+          - "<!--|| -->"
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+        exclude: ^docs/index\.md$|^\.github/pull_request_template\.md$|\.github/issue_template\.md$
+      - id: insert-license
+        name: Add license for all Makefile files
+        files: ^Makefile$
+        args:
+          - --comment-style
+          - "|#|"
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all TOML files
         files: \.toml$
@@ -53,7 +72,7 @@ repos:
     hooks:
       - id: black-jupyter
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v19.1.1
+    rev: v19.1.4
     hooks:
       - id: clang-format
         args: [--style=Google]
@@ -72,7 +91,7 @@ repos:
         args: [--ignore-words=.github/linters/codespell.txt]
         exclude: ^docs/image|^spark/common/src/test/resources|^docs/usecases|^tools/maven/scalafmt
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.20.1
+    rev: v8.21.2
     hooks:
       - id: gitleaks
   - repo: https://github.com/shssoichiro/oxipng
@@ -129,7 +148,7 @@ repos:
         args: [--markdown-linebreak-ext=md]
         exclude: ^docs-overrides/main\.html$|\.Rd$
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.42.0
+    rev: v0.43.0
     hooks:
       - id: markdownlint
         name: Run markdownlint

Makefile

@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 check :
 	pre-commit run --all-files
 .PHONY : check

R/DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: apache.sedona
 Title: R Interface for Apache Sedona
-Version: 1.6.1
+Version: 1.7.0.900
 Authors@R:
     c(person(family = "Apache Sedona",
              role = c("aut", "cre"),
@@ -38,6 +38,6 @@ Suggests:
     knitr,
     rmarkdown
 Encoding: UTF-8
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.2
 SystemRequirements: 'Apache Spark' 3.x
 Roxygen: list(markdown = TRUE)

R/R/data_interface.R

@@ -389,7 +389,14 @@ sedona_read_shapefile <- function(sc,
 # ------- Read SDF ------------
 #' Read geospatial data into a Spark DataFrame.
 #'
-#' @description Functions to read geospatial data from a variety of formats into Spark DataFrames.
+#' @description `r lifecycle::badge("deprecated")`
+#'
+#' These functions are deprecated and will be removed in a future release. Sedona has
+#' been implementing readers as spark DataFrame sources, so you can use `spark_read_source`
+#' with the right sources ("shapefile", "geojson", "geoparquet") to read geospatial data.
+#'
+#' Functions to read geospatial data from a variety of formats into Spark DataFrames.
+#'
 #'
 #' * `spark_read_shapefile`: from a shapefile
 #' * `spark_read_geojson`: from a geojson file
@@ -420,9 +427,15 @@ spark_read_shapefile <- function(sc,
                                  options = list(),
                                  ...) {
 
+  lifecycle::deprecate_soft(
+    "1.7.1",
+    "spark_read_shapefile()",
+    with = "spark_read_source()"
+  )
+
   lapply(names(options), function(name) {
     if (!name %in% c("")) {
-      warning(paste0("Ignoring unknown option '", name,"'"))
+      warning(paste0("Ignoring unknown option '", name, "'"))
     }
   })
 
@@ -446,12 +459,18 @@ spark_read_geojson <- function(sc,
                                memory = TRUE,
                                overwrite = TRUE) {
 
+  lifecycle::deprecate_soft(
+    "1.7.1",
+    "spark_read_geojson()",
+    with = "spark_read_source()"
+  )
+
   # check options
   if ("allow_invalid_geometries" %in% names(options)) final_allow_invalid <- options[["allow_invalid_geometries"]] else final_allow_invalid <- TRUE
   if ("skip_syntactically_invalid_geometries" %in% names(options)) final_skip <- options[["skip_syntactically_invalid_geometries"]] else final_skip <- TRUE
   lapply(names(options), function(name) {
     if (!name %in% c("allow_invalid_geometries", "skip_syntactically_invalid_geometries")) {
-      warning(paste0("Ignoring unknown option '", name,"'"))
+      warning(paste0("Ignoring unknown option '", name, "'"))
     }
   })
 
@@ -480,6 +499,12 @@ spark_read_geoparquet <- function(sc,
                                   memory = TRUE,
                                   overwrite = TRUE) {
 
+  lifecycle::deprecate_soft(
+    "1.7.1",
+    "spark_read_geoparquet()",
+    with = "spark_read_source()"
+  )
+
   spark_read_source(sc,
                     name = name,
                     path = path,
@@ -602,7 +627,14 @@ sedona_save_spatial_rdd <- function(x,
 
 #' Write geospatial data from a Spark DataFrame.
 #'
-#' @description Functions to write geospatial data into a variety of formats from Spark DataFrames.
+#' @description `r lifecycle::badge("deprecated")`
+#'
+#' These functions are deprecated and will be removed in a future release. Sedona has
+#' been implementing writers as spark DataFrame sources, so you can use `spark_write_source`
+#' with the right sources ("shapefile", "geojson", "geoparquet") to write geospatial data.
+
+#'
+#' Functions to write geospatial data into a variety of formats from Spark DataFrames.
 #'
 #' * `spark_write_geojson`: to GeoJSON
 #' * `spark_write_geoparquet`: to GeoParquet
@@ -644,6 +676,12 @@ spark_write_geojson <- function(x,
                                 partition_by = NULL,
                                 ...) {
 
+  lifecycle::deprecate_soft(
+    "1.7.1",
+    "spark_write_geojson()",
+    with = "spark_read_source()"
+  )
+
   ## find geometry column if not specified
   if (!"spatial_col" %in% names(options)) {
     schema <- x %>% sdf_schema()
@@ -678,6 +716,11 @@ spark_write_geoparquet <- function(x,
                                    options = list(),
                                    partition_by = NULL,
                                    ...) {
+  lifecycle::deprecate_soft(
+    "1.7.1",
+    "spark_write_geoparquet()",
+    with = "spark_read_source()"
+  )
 
   spark_write_source(
     x = x,
@@ -702,6 +745,12 @@ spark_write_raster <- function(x,
                                    partition_by = NULL,
                                    ...) {
 
+  lifecycle::deprecate_soft(
+    "1.7.1",
+    "spark_write_raster()",
+    with = "spark_read_source()"
+  )
+
   spark_write_source(
     x = x,
     source = "raster",

R/R/dependencies.R

@@ -25,7 +25,7 @@ spark_dependencies <- function(spark_version, scala_version, ...) {
   }
 
   packages <- c(
-    "org.datasyslab:geotools-wrapper:1.6.1-28.2"
+    "org.datasyslab:geotools-wrapper:1.7.0-28.5"
   )
   jars <- NULL
 
@@ -38,7 +38,7 @@ spark_dependencies <- function(spark_version, scala_version, ...) {
       paste0(
         "org.apache.sedona:sedona-",
         c("spark-shaded"),
-        sprintf("-%s_%s:1.6.1", spark_version, scala_version)
+        sprintf("-%s_%s:1.7.0", spark_version, scala_version)
       ),
       packages
     )

R/README.md

@@ -1,3 +1,22 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
 # apache.sedona <img src="man/figures/logo.png" align="right" alt="Apache Sedona Logo" width="120"/>
 
 [Apache Sedona](https://sedona.apache.org/) is a cluster computing system for processing large-scale spatial data. Sedona extends existing cluster computing systems, such as Apache Spark and Apache Flink, with a set of out-of-the-box distributed Spatial Datasets and Spatial SQL that efficiently load, process, and analyze large-scale spatial data across machines.