diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe728c06..d34bdcc3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,12 +4,18 @@ on: [push] jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-latest, macos-latest] version: ['Release', 'Debug'] steps: - uses: actions/checkout@v3 - - run: sudo apt-get install libsqlite3-dev + - name: Install dependencies (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get install libsqlite3-dev + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: brew install sqlite3 - run: uname -a; BUILDTYPE=${{ matrix.version }} make - run: make test diff --git a/CHANGELOG.md b/CHANGELOG.md index 81ef86c5..1fd1024b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.73.0 + +* Correctly clip features down to nothing when the clip region doesn't intersect the tile at all + # 2.72.0 * Add --clip-polygon-file and --feature-filter-file options to tippecanoe-overzoom diff --git a/Makefile b/Makefile index d01eb42b..2f522d00 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Iclipper2/include LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -68,7 +68,7 @@ tippecanoe-enumerate: enumerate.o tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o +tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o @@ -439,6 +439,10 @@ overzoom-test: tippecanoe-overzoom ./tippecanoe-decode tests/pbf/countries-8-135-86-bigclip.pbf 8 135 86 > tests/pbf/countries-8-135-86-bigclip.json.check cmp tests/pbf/countries-8-135-86-bigclip.json.check tests/pbf/countries-8-135-86-bigclip.json rm tests/pbf/countries-8-135-86-bigclip.pbf tests/pbf/countries-8-135-86-bigclip.json.check + # Clip region that does not intersect with the tile + ./tippecanoe-overzoom -o tests/pbf/squirrels-13-2413-3077-clip.pbf --clip-polygon-file tests/pbf/squirrels-clip.json tests/pbf/squirrels-13-2413-3077.pbf 13/2413/3077 13/2413/3077 + cmp tests/pbf/squirrels-13-2413-3077-clip.pbf /dev/null # clipped away + rm tests/pbf/squirrels-13-2413-3077-clip.pbf join-test: tippecanoe tippecanoe-decode tile-join ./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}' diff --git a/main.cpp b/main.cpp index 0d0a9ed3..aa6c49f2 100644 --- a/main.cpp +++ b/main.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -67,6 +66,7 @@ #include "sort.hpp" #include "attribute.hpp" #include "thread.hpp" +#include "platform.hpp" static int low_detail = 12; static int full_detail = -1; @@ -187,7 +187,7 @@ void init_cpus() { if (TIPPECANOE_MAX_THREADS != NULL) { CPUS = atoi_require(TIPPECANOE_MAX_THREADS, "TIPPECANOE_MAX_THREADS"); } else { - CPUS = sysconf(_SC_NPROCESSORS_ONLN); + CPUS = get_num_avail_cpus(); } if (CPUS < 1) { @@ -202,13 +202,7 @@ void init_cpus() { // Round down to a power of 2 CPUS = 1 << (int) (log(CPUS) / log(2)); - struct rlimit rl; - if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { - perror("getrlimit"); - exit(EXIT_PTHREAD); - } else { - MAX_FILES = rl.rlim_cur; - } + MAX_FILES = get_max_open_files(); // Don't really want too many temporary files, because the file system // will start to bog down eventually @@ -222,7 +216,7 @@ void init_cpus() { long long fds[MAX_FILES]; long long i; for (i = 0; i < MAX_FILES; i++) { - fds[i] = open("/dev/null", O_RDONLY | O_CLOEXEC); + fds[i] = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (fds[i] < 0) { break; } @@ -899,7 +893,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split std::atomic indexpos(indexst.st_size); int bytes = sizeof(struct index); - int page = sysconf(_SC_PAGESIZE); + int page = get_page_size(); // Don't try to sort more than 2GB at once, // which used to crash Macs and may still long long max_unit = 2LL * 1024 * 1024 * 1024; @@ -1069,31 +1063,6 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d } } -static size_t calc_memsize() { - size_t mem; - -#ifdef __APPLE__ - int64_t hw_memsize; - size_t len = sizeof(int64_t); - if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) { - perror("sysctl hw.memsize"); - exit(EXIT_MEMORY); - } - mem = hw_memsize; -#else - long long pagesize = sysconf(_SC_PAGESIZE); - long long pages = sysconf(_SC_PHYS_PAGES); - if (pages < 0 || pagesize < 0) { - perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES"); - exit(EXIT_MEMORY); - } - - mem = (long long) pages * pagesize; -#endif - - return mem; -} - void radix(std::vector &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic *geompos, int maxzoom, int basezoom, double droprate, double gamma) { // Run through the index and geometry for each reader, // splitting the contents out by index into as many @@ -1446,7 +1415,7 @@ std::pair read_input(std::vector &sources, char *fname, i size_t dist_count = 0; double area_sum = 0; - int files_open_before_reading = open("/dev/null", O_RDONLY | O_CLOEXEC); + int files_open_before_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (files_open_before_reading < 0) { perror("open /dev/null"); exit(EXIT_OPEN); @@ -1887,7 +1856,7 @@ std::pair read_input(std::vector &sources, char *fname, i } } - int files_open_after_reading = open("/dev/null", O_RDONLY | O_CLOEXEC); + int files_open_after_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (files_open_after_reading < 0) { perror("open /dev/null"); exit(EXIT_OPEN); @@ -3728,7 +3697,7 @@ int main(int argc, char **argv) { signal(SIGPIPE, SIG_IGN); - files_open_at_start = open("/dev/null", O_RDONLY | O_CLOEXEC); + files_open_at_start = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (files_open_at_start < 0) { perror("open /dev/null"); exit(EXIT_OPEN); @@ -3878,7 +3847,7 @@ int main(int argc, char **argv) { muntrace(); #endif - i = open("/dev/null", O_RDONLY | O_CLOEXEC); + i = open(get_null_device(), O_RDONLY | O_CLOEXEC); // i < files_open_at_start is not an error, because reading from a pipe closes stdin if (i > files_open_at_start) { fprintf(stderr, "Internal error: did not close all files: %d\n", i); diff --git a/overzoom.cpp b/overzoom.cpp index 112f0187..c7eb922d 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -294,6 +294,7 @@ int main(int argc, char **argv) { // clip the clip polygons, if any, to the tile bounds, // to reduce their complexity + bool clipped_to_nothing = false; if (clipbboxes.size() > 0) { long long wx1 = (nx - buffer / 256.0) * (1LL << (32 - nz)); long long wy1 = (ny - buffer / 256.0) * (1LL << (32 - nz)); @@ -315,38 +316,47 @@ int main(int argc, char **argv) { if (c.dv.size() > 0) { c.dv = clip_poly_poly(c.dv, tile_bounds); + + if (c.dv.size() == 0) { + clipped_to_nothing = true; + break; + } } } } - json_object *json_filter = NULL; - if (filter.size() > 0) { - json_filter = parse_filter(filter.c_str()); - } - - for (auto const &s : sources) { - std::string tile; - char buf[1000]; - int len; + std::string out; - FILE *f = fopen(s.tile.c_str(), "rb"); - if (f == NULL) { - perror(s.tile.c_str()); - exit(EXIT_FAILURE); + if (!clipped_to_nothing) { + json_object *json_filter = NULL; + if (filter.size() > 0) { + json_filter = parse_filter(filter.c_str()); } - while ((len = fread(buf, sizeof(char), 1000, f)) > 0) { - tile.append(std::string(buf, len)); + for (auto const &s : sources) { + std::string tile; + char buf[1000]; + int len; + + FILE *f = fopen(s.tile.c_str(), "rb"); + if (f == NULL) { + perror(s.tile.c_str()); + exit(EXIT_FAILURE); + } + + while ((len = fread(buf, sizeof(char), 1000, f)) > 0) { + tile.append(std::string(buf, len)); + } + fclose(f); + + input_tile t = s; + t.tile = std::move(tile); + its.push_back(std::move(t)); } - fclose(f); - input_tile t = s; - t.tile = std::move(tile); - its.push_back(std::move(t)); + out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes); } - std::string out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes); - FILE *f = fopen(outfile, "wb"); if (f == NULL) { perror(outfile); diff --git a/platform.cpp b/platform.cpp new file mode 100644 index 00000000..9bf7c3ad --- /dev/null +++ b/platform.cpp @@ -0,0 +1,56 @@ +#include "platform.hpp" +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#include +#include +#include +#endif + +#include "errors.hpp" + +long get_num_avail_cpus() { + return sysconf(_SC_NPROCESSORS_ONLN); +} + +long get_page_size() { + return sysconf(_SC_PAGESIZE); +} + +size_t calc_memsize() { + size_t mem; + +#ifdef __APPLE__ + int64_t hw_memsize; + size_t len = sizeof(int64_t); + if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) { + perror("sysctl hw.memsize"); + exit(EXIT_MEMORY); + } + mem = hw_memsize; +#else + long long pagesize = sysconf(_SC_PAGESIZE); + long long pages = sysconf(_SC_PHYS_PAGES); + if (pages < 0 || pagesize < 0) { + perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES"); + exit(EXIT_MEMORY); + } + + mem = (long long) pages * pagesize; +#endif + + return mem; +} + +size_t get_max_open_files() { + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { + perror("getrlimit"); + exit(EXIT_PTHREAD); + } + return rl.rlim_cur; +} \ No newline at end of file diff --git a/platform.hpp b/platform.hpp new file mode 100644 index 00000000..7fa4033b --- /dev/null +++ b/platform.hpp @@ -0,0 +1,18 @@ +#ifndef PLATFORM_HPP +#define PLATFORM_HPP + +#include + +long get_num_avail_cpus(); + +long get_page_size(); + +size_t calc_memsize(); + +size_t get_max_open_files(); + +constexpr const char *get_null_device() { + return "/dev/null"; +} + +#endif // PLATFORM_HPP \ No newline at end of file diff --git a/tests/pbf/squirrels-13-2413-3077.pbf b/tests/pbf/squirrels-13-2413-3077.pbf new file mode 100644 index 00000000..20e02168 Binary files /dev/null and b/tests/pbf/squirrels-13-2413-3077.pbf differ diff --git a/tests/pbf/squirrels-clip.json b/tests/pbf/squirrels-clip.json new file mode 100644 index 00000000..ef0e95fd --- /dev/null +++ b/tests/pbf/squirrels-clip.json @@ -0,0 +1 @@ +{"coordinates":[[[-73.9722549,40.7904659],[-73.9755817,40.7674848],[-73.938783,40.767022],[-73.9457082,40.7791562],[-73.9628855,40.7760201],[-73.9629534,40.7914425],[-73.9722549,40.7904659]]],"crs":{"properties":{"name":"EPSG:4326"},"type":"name"},"type":"Polygon"} diff --git a/tile-join.cpp b/tile-join.cpp index ce4e436a..7fef6fb8 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -43,6 +43,7 @@ #include "errors.hpp" #include "geometry.hpp" #include "thread.hpp" +#include "platform.hpp" int pk = false; int pC = false; @@ -1325,7 +1326,7 @@ int main(int argc, char **argv) { struct tileset_reader *readers = NULL; - CPUS = sysconf(_SC_NPROCESSORS_ONLN); + CPUS = get_num_avail_cpus(); const char *TIPPECANOE_MAX_THREADS = getenv("TIPPECANOE_MAX_THREADS"); if (TIPPECANOE_MAX_THREADS != NULL) { diff --git a/version.hpp b/version.hpp index bf022696..fac06ade 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.72.0" +#define VERSION "v2.73.0" #endif