WIP: Protochunk rendering - addresses #2002, #2024, #2035

docs/config.rst

@@ -270,6 +270,58 @@ General
 
         processes = 2
 
+.. _render_protochunks:
+
+``render_protochunks``
+    This is a boolean. When enabled, the Overviewer will render some proto-chunks, ie,
+    chunks that have not been fully generated yet.
+
+    This is useful for rendering worlds that have pre-1.18 chunks. When Minecraft 1.18
+    or higher loads one of these chunks, it will mark it as a proto-chunk until all of
+    the terrain below Y=0 is generated. So, without this option set, the Overviewer
+    may not render parts of the world at all.
+
+    It's worth noting that if a world has been optimized (``--forceUpgrade``), all
+    upgraded but unvisited chunks will be in this state.
+
+    However, this option will make the edges of the map contain actual proto-chunks, so
+    it isn't recommended for worlds started in 1.18 or newer, and disabled by default.
+
+    e.g.::
+
+        render_protochunks = True
+
+    **Default:** ``False``
+
+.. _prettify_protochunk_lighting:
+
+``prettify_protochunk_lighting``
+    This is a boolean. It is only useful in combination with
+    :ref:`render_protochunks <render_protochunks>`.
+
+    If it is enabled, a heuristic is applied to get rid of certain lighting issues in
+    proto-chunks on map borders.
+
+    Some proto-chunks are partially dark because their skylight has not been
+    calculated yet, but skylight from neighboring chunks has already been used to
+    generate lighting data.
+
+    So, with this option enabled, if a chunk section
+     - is a proto-chunk
+     - has not been visited by a player before
+     - and does not contain a single block with full sky light
+
+    the Overviewer will render it as fully bright instead.
+
+    It is disabled by default as there is no guarantee that no false positives are
+    generated.
+
+    e.g.::
+
+        prettify_protochunk_lighting = True
+
+    **Default:** ``False``
+
 Observers
 ~~~~~~~~~
 

overviewer.py

@@ -477,6 +477,9 @@ def set_renderchecks(checkname, num):
     # TODO: optionally more caching layers here
 
     renders = config['renders']
+    render_protochunks = config["render_protochunks"]
+    prettify_protochunk_lighting = config["prettify_protochunk_lighting"]
+
     for render_name, render in renders.items():
         logging.debug("Found the following render thing: %r", render)
 
@@ -485,7 +488,11 @@ def set_renderchecks(checkname, num):
             w = worldcache[render['world']]
         except KeyError:
             try:
-                w = world.World(render['world'])
+                w = world.World(
+                    render['world'],
+                    render_protochunks,
+                    prettify_protochunk_lighting,
+                )
             except CorruptNBTError as e:
                 logging.error("Failed to open world %r.", render['world'])
                 raise e

overviewer_core/settingsDefinition.py

@@ -109,6 +109,9 @@ def get_default_config():
 
     conf['processes'] = Setting(required=True, validator=int, default=-1)
 
+    conf['render_protochunks'] = Setting(required=False, validator=validateBool, default=False)
+    conf['prettify_protochunk_lighting'] = Setting(required=False, validator=validateBool, default=False)
+
     # TODO clean up this ugly in sys.argv hack
     if platform.system() == 'Windows' or not sys.stdout.isatty() or "--simple" in sys.argv:
         obs = LoggingObserver()

overviewer_core/world.py

@@ -88,7 +88,12 @@ class World(object):
 
     """
 
-    def __init__(self, worlddir):
+    def __init__(
+        self,
+        worlddir,
+        render_protochunks=False,
+        prettify_protochunk_lighting=False,
+    ):
         self.worlddir = worlddir
 
         # This list, populated below, will hold RegionSet files that are in
@@ -137,7 +142,12 @@ def __init__(self, worlddir):
                 # construct a regionset object for this
                 rel = os.path.relpath(root, self.worlddir)
                 if os.path.basename(rel) != "poi":
-                    rset = RegionSet(root, rel)
+                    rset = RegionSet(
+                        root,
+                        rel,
+                        render_protochunks=render_protochunks,
+                        prettify_protochunk_lighting=prettify_protochunk_lighting,
+                    )
                     if root == os.path.join(self.worlddir, "region"):
                         self.regionsets.insert(0, rset)
                     else:
@@ -250,7 +260,7 @@ class RegionSet(object):
 
     """
 
-    def __init__(self, regiondir, rel):
+    def __init__(self, regiondir, rel, render_protochunks=False, prettify_protochunk_lighting=False):
         """Initialize a new RegionSet to access the region files in the given
         directory.
 
@@ -265,6 +275,23 @@ def __init__(self, regiondir, rel):
         """
         self.regiondir = os.path.normpath(regiondir)
         self.rel = os.path.normpath(rel)
+        self.render_protochunks = render_protochunks
+        self.prettify_protochunk_lighting = prettify_protochunk_lighting
+        renderable_statuses = [
+            "full",
+            "postprocessed",
+            "fullchunk",
+            "mobs_spawned",
+            "spawn",
+            "",
+        ]
+        if self.render_protochunks:
+            renderable_statuses += [
+                "structure_starts",
+                "empty",
+            ]
+        self.renderable_statuses = tuple(renderable_statuses)
+
         logging.debug("regiondir is %r" % self.regiondir)
         logging.debug("rel is %r" % self.rel)
 
@@ -1029,9 +1056,17 @@ def __init__(self, regiondir, rel):
 
     # Re-initialize upon unpickling
     def __getstate__(self):
-        return (self.regiondir, self.rel)
+        return (
+            (self.regiondir, self.rel),
+            {
+                "render_protochunks": self.render_protochunks,
+                "prettify_protochunk_lighting": self.prettify_protochunk_lighting,
+            },
+        )
+
     def __setstate__(self, state):
-        return self.__init__(*state)
+        args, kwargs = state
+        return self.__init__(*args, **kwargs)
 
     def __repr__(self):
         return "<RegionSet regiondir=%r>" % self.regiondir
@@ -1730,8 +1765,8 @@ def get_chunk(self, x, z):
         # Empty is self-explanatory, and liquid_carved and carved seem to correspond
         # to SkyLight not being calculated, which results in mostly-black chunks,
         # so we'll just pretend they aren't there.
-        if chunk_data.get("Status", "") not in ("full", "postprocessed", "fullchunk",
-                                                "mobs_spawned", "spawn", ""):
+        chunk_status = chunk_data.get("Status", "")
+        if chunk_status not in self.renderable_statuses:
             raise ChunkDoesntExist("Chunk %s,%s doesn't exist" % (x,z))
 
         # Turn the Biomes array into a 16x16 numpy array
@@ -1752,6 +1787,8 @@ def get_chunk(self, x, z):
         chunk_data['Biomes'] = biomes
         chunk_data['NewBiomes'] = (len(biomes.shape) == 3)
 
+        chunk_was_inhabited = chunk_data.get("InhabitedTime", 0) > 0
+
         unrecognized_block_types = {}
         for section in chunk_data['Sections']:
 
@@ -1761,18 +1798,50 @@ def get_chunk(self, x, z):
                 # Sometimes, Minecraft loves generating chunks with no light info.
                 # These mostly appear to have those two properties, and in this case
                 # we default to full-bright as it's less jarring to look at than all-black.
-                if chunk_data.get("Status", "") == "spawn" and 'Lights' in chunk_data:
+                if chunk_status == "spawn" and 'Lights' in chunk_data:
                     section['SkyLight'] = numpy.full((16,16,16), 255, dtype=numpy.uint8)
                 else:
                     if 'SkyLight' in section:
                         skylight = numpy.frombuffer(section['SkyLight'], dtype=numpy.uint8)
                         skylight = skylight.reshape((16,16,8))
+                    elif chunk_status in ["structure_starts", "empty"]:
+                        # completely black border chunk. fullbright is definitely
+                        # preferable here.
+                        skylight = numpy.full((16,16,8), 255, dtype=numpy.uint8)
                     else:   # Special case introduced with 1.14
                         skylight = numpy.zeros((16,16,8), dtype=numpy.uint8)
                     skylight_expanded = numpy.empty((16,16,16), dtype=numpy.uint8)
                     skylight_expanded[:,:,::2] = skylight & 0x0F
                     skylight_expanded[:,:,1::2] = (skylight & 0xF0) >> 4
                     del skylight
+
+                    if self.prettify_protochunk_lighting:
+                        # this chunk might be a border chunk that was populated with
+                        # sky light from neighboring chunks, but hasn't had its own sky
+                        # light calculated yet.
+                        # first, check if the chunk was inhabited. This is pretty much
+                        # impossible for border chunks.
+                        # then, check if the chunk is even a proto-chunk.
+                        if not chunk_was_inhabited and chunk_status in ["structure_starts", "empty"]:
+                            # and finally, check if there is a block with full skylight
+                            # in the section.
+                            # if there is one, that means that skylight was definitely
+                            # calculated here before.
+                            # if not, we assume that we're in a border chunk.
+                            # this is based on the following observations:
+                            #  - sections that are underground and far from border
+                            #    chunks will not be seen on the map most of the time
+                            #    anyway
+                            #  - sections that are above ground will almost always have
+                            #    a block somewhere with full light level in naturally
+                            #    generated terrain
+                            #  - sections that are above ground and have been modified
+                            #    by a player have almost certainly been inhabited
+                            # so, false positives shouldn't be too common, and those
+                            # that occur shouldn't be in player-visited areas.
+                            if not numpy.any(skylight_expanded == 15):
+                                skylight_expanded = numpy.full((16,16,16), 0x0F, dtype=numpy.uint8)
+
                     section['SkyLight'] = skylight_expanded
 
                 # Turn the BlockLight array into a 16x16x16 matrix, same as SkyLight