add typing stubs (#29)

* not ready * fix * rename * update * fix * update * remove pybind11 * update headers * fix * update * fix * fix * fix * fix * fix * fix * fix * fix * fix * update * fix * fix * update * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * ifx * fix * fix * fix * fix * fix * fix * update --------- Co-authored-by: tang zhixiong <[email protected]>
cubao · Oct 3, 2024 · a3ce547 · a3ce547
1 parent 87bbca2
commit a3ce547
Show file tree

Hide file tree

Showing 28 changed files with 5,967 additions and 1,205 deletions.
diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml
@@ -12,8 +12,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-20.04, macos-11]
-        python-version: ["3.8", "3.9", "3.10"]
+        platform: [ubuntu-20.04, windows-2019, macos-13]
+        python-version: ["3.9"]
 
     runs-on: ${{ matrix.platform }}
 

diff --git a/.gitignore b/.gitignore
@@ -12,3 +12,4 @@ tests/*.json
 data/suzhoubeizhan.idx
 package-lock.json
 node_modules
+stubs
diff --git a/.gitmodules b/.gitmodules
@@ -7,6 +7,3 @@
 [submodule "headers"]
     path = headers
     url = https://github.com/cubao/headers.git
-[submodule "pybind11"]
-    path = pybind11
-    url = https://github.com/pybind/pybind11.git
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -33,26 +33,14 @@ repos:
   - id: requirements-txt-fixer
   - id: trailing-whitespace
 
-# Black, the code formatter, natively supports pre-commit
-- repo: https://github.com/psf/black
-  rev: 22.3.0
+# Check linting and style issues
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: "v0.6.5"
   hooks:
-  - id: black
-    exclude: ^(docs)
-
-# Sort your imports in a standard form
-- repo: https://github.com/PyCQA/isort
-  rev: 5.11.5
-  hooks:
-  - id: isort
-    args: [--profile, black]
-
-# Upgrade older Python syntax
-- repo: https://github.com/asottile/pyupgrade
-  rev: v2.31.1
-  hooks:
-  - id: pyupgrade
-    args: ["--py36-plus"]
+    - id: ruff
+      args: ["--fix", "--show-fixes"]
+    - id: ruff-format
+      exclude: ^(docs)
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
@@ -61,12 +49,6 @@ repos:
   - id: remove-tabs
     exclude: ^(docs|Makefile|benchmarks/Makefile)
 
-- repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
-  hooks:
-  - id: flake8
-    additional_dependencies: [flake8-bugbear]
-
 # CMake formatting
 - repo: https://github.com/cheshirekow/cmake-format-precommit
   rev: v0.6.13

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,24 +1,41 @@
-cmake_minimum_required(VERSION 3.5)
-project(geobuf)
+cmake_minimum_required(VERSION 3.15...3.27)
 
-set(CUBAO_INCLUDE ${PROJECT_SOURCE_DIR}/headers/include)
-include(${CUBAO_INCLUDE}/utils.cmake)
-# set(CMAKE_BUILD_TYPE Debug)
+project(
+    ${SKBUILD_PROJECT_NAME}
+    VERSION ${SKBUILD_PROJECT_VERSION}
+    LANGUAGES CXX)
 
-activate_common_configuration()
-auto_build_type_and_compile_flags()
-configure_output_directories()
+# set(CMAKE_BUILD_TYPE "Debug")
+if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
+    set(CMAKE_BUILD_TYPE
+        "Release"
+        CACHE STRING "" FORCE)
+    message(STATUS "Set build type to default: ${CMAKE_BUILD_TYPE}")
+else()
+    message(STATUS "Your build type: ${CMAKE_BUILD_TYPE}")
+endif()
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -ggdb")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -ggdb")
+elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
+endif()
 
-if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
-    set(CUBAO_USE_DUMMY_VERSION_H "True")
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-pragma-once-outside-header")
 endif()
-configure_version_h()
 
-# MISC
 add_definitions(-D_CRT_SECURE_NO_WARNINGS)
+add_definitions(-DPROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}")
+add_definitions(-DPROJECT_BINARY_DIR="${PROJECT_BINARY_DIR}")
 
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_CXX_STANDARD 17)
-include_directories(${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/headers/include ${PROJECT_SOURCE_DIR}/src)
+include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/headers/include)
+include_directories(${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/geobuf)
 
 option(BUILD_SHARED_LIBS "Build shared library." OFF)
 set(HEADER_ROOT "${PROJECT_SOURCE_DIR}/src/${PROJECT_NAME}")
@@ -27,25 +44,11 @@ file(GLOB_RECURSE SOURCES src/**/*.cpp)
 add_library(${PROJECT_NAME} ${SOURCES} ${HEADERS})
 target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS})
 set_target_properties(${PROJECT_NAME} PROPERTIES CXX_VISIBILITY_PRESET "hidden")
-print_all_linked_libraries(${PROJECT_NAME})
-install(
-    TARGETS ${PROJECT_NAME}
-    LIBRARY DESTINATION lib
-    ARCHIVE DESTINATION lib)
-foreach(file ${HEADERS})
-    get_filename_component(dir ${file} DIRECTORY)
-    string(
-        REGEX
-        REPLACE "${HEADER_ROOT}"
-                ""
-                dir
-                ${dir})
-    install(FILES ${file} DESTINATION "include/${PROJECT_NAME}/${dir}")
-endforeach()
+install(TARGETS ${PROJECT_NAME} DESTINATION ${PROJECT_NAME})
 
 add_executable(pbf_decoder src/geobuf/pbf_decoder.cpp)
 target_compile_definitions(pbf_decoder PUBLIC -DPBF_DECODER_ENABLE_MAIN)
-install(TARGETS pbf_decoder RUNTIME DESTINATION bin)
+install(TARGETS pbf_decoder RUNTIME DESTINATION ${PROJECT_NAME})
 
 option(BUILD_EXAMPLES "Build examples." ON)
 if(BUILD_EXAMPLES)
@@ -59,7 +62,7 @@ if(BUILD_EXAMPLES)
                     ${src})
         add_executable(${exe} ${src})
         target_link_libraries(${exe} ${PROJECT_NAME})
-        install(TARGETS ${exe} RUNTIME DESTINATION bin)
+        install(TARGETS ${exe} RUNTIME DESTINATION ${PROJECT_NAME})
     endforeach(src)
 endif()
 
@@ -75,22 +78,20 @@ if(BUILD_TESTS)
                     ${src})
         add_executable(${exe} ${src})
         target_link_libraries(${exe} ${PROJECT_NAME})
-        install(TARGETS ${exe} RUNTIME DESTINATION bin)
+        install(TARGETS ${exe} RUNTIME DESTINATION ${PROJECT_NAME})
     endforeach(src)
 endif()
 add_custom_target(test_all COMMAND make -C .. test_all)
 
-# print_include_directories()
-
-# print_all_variables()
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+find_package(pybind11 CONFIG REQUIRED)
 
-set(PYBIND11_CPP_STANDARD -std=c++17)
-add_subdirectory(pybind11)
-pybind11_add_module(
-    _pybind11_geobuf
-    src/main.cpp
-    src/pybind11_rapidjson.cpp
-    src/pybind11_geojson.cpp)
-target_include_directories(_pybind11_geobuf PRIVATE ${HEADER_ROOT})
-target_link_libraries(_pybind11_geobuf PRIVATE ${PROJECT_NAME})
-target_compile_definitions(_pybind11_geobuf PRIVATE VERSION_INFO=${PYBIND11_GEOBUF_VERSION_INFO})
+set(PYBIND_SRCS src/main.cpp src/pybind11_rapidjson.cpp src/pybind11_geojson.cpp)
+python_add_library(
+    _core
+    MODULE
+    ${PYBIND_SRCS}
+    WITH_SOABI)
+target_link_libraries(_core PRIVATE pybind11::headers ${PROJECT_NAME})
+target_compile_definitions(_core PRIVATE VERSION_INFO=${PROJECT_VERSION})
+install(TARGETS _core DESTINATION ${PROJECT_NAME})
diff --git a/Makefile b/Makefile
@@ -2,7 +2,7 @@ PROJECT_SOURCE_DIR ?= $(abspath ./)
 PROJECT_NAME ?= $(shell basename $(PROJECT_SOURCE_DIR))
 BUILD_DIR ?= $(PROJECT_SOURCE_DIR)/build
 INSTALL_DIR ?= $(BUILD_DIR)/install
-NUM_JOB ?= 8
+NUM_JOBS ?= 8
 
 all:
 	@echo nothing special
@@ -17,21 +17,36 @@ reset_submodules:
 	git submodule update --init --recursive
 
 clean:
-	rm -rf $(BUILD_DIR) *.egg-info dist
+	rm -rf $(BUILD_DIR) *.egg-info dist stubs
 force_clean:
 	docker run --rm -v `pwd`:`pwd` -w `pwd` -it alpine/make make clean
 
-CMAKE_ARGS ?= \
-	-DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) \
-	-DBUILD_SHARED_LIBS=OFF
+PYTHON ?= python3
 build:
-	mkdir -p $(BUILD_DIR) && cd $(BUILD_DIR) && \
-	cmake $(PROJECT_SOURCE_DIR) $(CMAKE_ARGS) && \
-	make -j$(NUM_JOB) && make install
-.PHONY: build
+	$(PYTHON) -m pip install scikit_build_core pyproject_metadata pathspec pybind11
+	CMAKE_BUILD_PARALLEL_LEVEL=$(NUM_JOBS) $(PYTHON) -m pip install --no-build-isolation -Ceditable.rebuild=true -Cbuild-dir=build -ve.
+python_install:
+	$(PYTHON) -m pip install . --verbose
+python_wheel:
+	$(PYTHON) -m pip wheel . -w build --verbose
+python_sdist:
+	$(PYTHON) -m pip sdist . --verbose
+python_test: pytest
+test:
+	# make roundtrip_test_js roundtrip_test_cpp diff
+	python3 geobuf-roundtrip-test.py pygeobuf/test/fixtures
+pytest:
+	python3 -m pip install pytest numpy
+	pytest tests/test_basic.py
+cli_test: cli_test1 cli_test2 cli_test3 cli_test4
+.PHONY: build python_install python_wheel python_sdist test pytest cli_test
+
+restub:
+	pybind11-stubgen pybind11_geobuf._core -o stubs
+	cp -rf stubs/pybind11_geobuf/_core src/pybind11_geobuf
 
 test_all:
-	@cd build && for t in $(wildcard $(BUILD_DIR)/bin/test_*); do echo $$t && eval $$t >/dev/null 2>&1 && echo 'ok' || echo $(RED)Not Ok$(NC); done
+	@cd build && for t in $(wildcard $(BUILD_DIR)/test_*); do echo $$t && eval $$t >/dev/null 2>&1 && echo 'ok' || echo $(RED)Not Ok$(NC); done
 
 INPUT_GEOJSON_PATH ?= data/sample1.json
 # INPUT_GEOJSON_PATH := pygeobuf/test/fixtures/geometrycollection.json
@@ -47,34 +62,26 @@ OUTPUT_PBF_CPP = $(OUTPUT_DIR_CPP)/$(GEOJSON_BASENAME).pbf
 OUTPUT_TXT_CPP = $(OUTPUT_PBF_CPP).txt
 OUTPUT_JSN_CPP = $(OUTPUT_PBF_CPP).json
 
-build/bin/json2geobuf: build
+build/json2geobuf: build
 
 # LINTJSON := jq .
-LINTJSON := $(BUILD_DIR)/bin/lintjson
+LINTJSON := $(BUILD_DIR)/lintjson
 roundtrip_test_js:
 	@umask 0000 && mkdir -p $(OUTPUT_DIR_JS)
 	json2geobuf $(INPUT_GEOJSON_PATH) > $(OUTPUT_PBF_JS)
-	build/bin/pbf_decoder $(OUTPUT_PBF_JS) > $(OUTPUT_TXT_JS)
+	build/pbf_decoder $(OUTPUT_PBF_JS) > $(OUTPUT_TXT_JS)
 	geobuf2json $(OUTPUT_PBF_JS) | $(LINTJSON) > $(OUTPUT_JSN_JS)
 	cat $(INPUT_GEOJSON_PATH) | $(LINTJSON) > $(OUTPUT_DIR_JS)/$(GEOJSON_BASENAME)
-roundtrip_test_cpp: build/bin/json2geobuf
+roundtrip_test_cpp: build/json2geobuf
 	@umask 0000 && mkdir -p $(OUTPUT_DIR_CPP)
-	$(BUILD_DIR)/bin/json2geobuf $(INPUT_GEOJSON_PATH) > $(OUTPUT_PBF_CPP)
-	build/bin/pbf_decoder $(OUTPUT_PBF_CPP) > $(OUTPUT_TXT_CPP)
-	$(BUILD_DIR)/bin/geobuf2json $(OUTPUT_PBF_CPP) | $(LINTJSON) > $(OUTPUT_JSN_CPP)
+	$(BUILD_DIR)/json2geobuf $(INPUT_GEOJSON_PATH) > $(OUTPUT_PBF_CPP)
+	build/pbf_decoder $(OUTPUT_PBF_CPP) > $(OUTPUT_TXT_CPP)
+	$(BUILD_DIR)/geobuf2json $(OUTPUT_PBF_CPP) | $(LINTJSON) > $(OUTPUT_JSN_CPP)
 	cat $(INPUT_GEOJSON_PATH) | $(LINTJSON) > $(OUTPUT_DIR_CPP)/$(GEOJSON_BASENAME)
-roundtrip_test_cpp: build/bin/json2geobuf
 diff:
 	# code --diff $(OUTPUT_TXT_JS) $(OUTPUT_TXT_CPP)
 	code --diff $(OUTPUT_JSN_JS) $(OUTPUT_JSN_CPP)
 
-test:
-	# make roundtrip_test_js roundtrip_test_cpp diff
-	python3 geobuf-roundtrip-test.py pygeobuf/test/fixtures
-pytest:
-	python3 -m pip install pytest numpy
-	pytest tests # --capture=tee-sys
-.PHONY: test pytest
 
 clean_test:
 	rm -rf $(OUTPUT_DIR_JS) $(OUTPUT_DIR_CPP) build/roundtrip_test
@@ -104,18 +111,6 @@ test_in_dev_container:
 			--network host --security-opt seccomp=unconfined \
 			-v `pwd`:`pwd` -w `pwd` -it $(DEV_CONTAINER_IMAG) bash
 
-PYTHON ?= python3
-python_install:
-	$(PYTHON) setup.py install --force
-python_build:
-	$(PYTHON) setup.py bdist_wheel
-python_sdist:
-	$(PYTHON) setup.py sdist
-	# tar -tvf dist/geobuf-*.tar.gz
-python_test: pytest
-
-cli_test: cli_test1 cli_test2 cli_test3 cli_test4
-
 cli_test1:
 	python3 -m pybind11_geobuf
 	python3 -m pybind11_geobuf --help
@@ -144,32 +139,32 @@ cli_test3:
 cli_test4:
 	python3 -m pybind11_geobuf is_subset_of data/feature_collection.json data/feature_collection.json
 
-.PHONY: cli_test cli_test1 cli_test2 cli_test3
+.PHONY: cli_test1 cli_test2 cli_test3
 
 geobuf_index.js: geobuf_index.proto
 	pbf $< > $@
 
-# conda create -y -n py36 python=3.6
-# conda create -y -n py37 python=3.7
 # conda create -y -n py38 python=3.8
 # conda create -y -n py39 python=3.9
 # conda create -y -n py310 python=3.10
+# conda create -y -n py311 python=3.11
+# conda create -y -n py312 python=3.12
 # conda env list
-python_build_py36:
-	PYTHON=python conda run --no-capture-output -n py36 make python_build
-python_build_py37:
-	PYTHON=python conda run --no-capture-output -n py37 make python_build
 python_build_py38:
 	PYTHON=python conda run --no-capture-output -n py38 make python_build
 python_build_py39:
 	PYTHON=python conda run --no-capture-output -n py39 make python_build
 python_build_py310:
 	PYTHON=python conda run --no-capture-output -n py310 make python_build
-python_build_all: python_build_py36 python_build_py37 python_build_py38 python_build_py39 python_build_py310
+python_build_py311:
+	PYTHON=python conda run --no-capture-output -n py311 make python_build
+python_build_py312:
+	PYTHON=python conda run --no-capture-output -n py312 make python_build
+python_build_all: python_build_py38 python_build_py39 python_build_py310 python_build_py311 python_build_py312
 python_build_all_in_linux:
 	docker run --rm -w `pwd` -v `pwd`:`pwd` -v `pwd`/build/linux:`pwd`/build -it $(DOCKER_TAG_LINUX) make python_build_all
 	make repair_wheels && rm -rf dist/*.whl && mv wheelhouse/*.whl dist && rm -rf wheelhouse
-python_build_all_in_macos: python_build_py38 python_build_py39 python_build_py310
+python_build_all_in_macos: python_build_py38 python_build_py39 python_build_py310 python_build_py311 python_build_py312
 python_build_all_in_windows: python_build_all
 
 repair_wheels:
@@ -186,6 +181,13 @@ tar.gz:
 	tar -cvz --exclude .git -f ../$(PROJECT_NAME).tar.gz .
 	ls -alh ../$(PROJECT_NAME).tar.gz
 
+SYNC_OUTPUT_DIR ?= headers/include/cubao
+sync_headers:
+	cp src/pybind11_rapidjson.cpp $(SYNC_OUTPUT_DIR)/pybind11_rapidjson.hpp
+	cp src/geobuf/geojson_helpers.hpp $(SYNC_OUTPUT_DIR)
+	cp src/geobuf/pybind11_helpers.hpp $(SYNC_OUTPUT_DIR)
+	cp src/geobuf/rapidjson_helpers.hpp $(SYNC_OUTPUT_DIR)
+
 # https://stackoverflow.com/a/25817631
 echo-%  : ; @echo -n $($*)
 Echo-%  : ; @echo $($*)

diff --git a/README.md b/README.md
@@ -53,6 +53,13 @@ pull all code:
 git submodule update --init --recursive
 ```
 
+install deps:
+
+```
+npm i -g geobuf
+python3 -m pip install geobuf
+```
+
 compile & test:
 
 ```bash