Skip to content

Commit

Permalink
sketch of core refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-v-b
d-v-b committed Jan 30, 2025
1 parent fc08f31 commit 7a77fe3
Show file tree
Hide file tree
Showing 31 changed files with 3,606 additions and 3,585 deletions.
9 changes: 2 additions & 7 deletions src/zarr/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,31 @@
from zarr._version import version as __version__
from zarr.api.synchronous import (
array,
consolidate_metadata,
copy,
copy_all,
copy_store,
create,
create_array,
create_group,
empty,
empty_like,
full,
full_like,
group,
load,
ones,
ones_like,
open,
open_array,
open_consolidated,
open_group,
open_like,
save,
save_array,
save_group,
tree,
zeros,
zeros_like,
)
from zarr.core.array import Array, AsyncArray
from zarr.core.config import config
from zarr.core.group import AsyncGroup, Group
from zarr.core.group.async import AsyncGroup
from zarr.core.group.sync import Group, consolidate_metadata, create_group, group, open_group, save_group, tree

# in case setuptools scm screw up and find version to be 0.0.0
assert not __version__.startswith("0.0.0")
Expand Down
250 changes: 4 additions & 246 deletions src/zarr/api/asynchronous.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
_warn_write_empty_chunks_kwarg,
parse_dtype,
)
from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata
from zarr.core.group.core import open_group
from zarr.core.group.metadata import ConsolidatedMetadata
from zarr.core.group import AsyncGroup, GroupMetadata
from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
from zarr.core.metadata.v2 import _default_compressor, _default_filters
from zarr.errors import NodeTypeValidationError
Expand Down Expand Up @@ -52,7 +54,7 @@
"empty_like",
"full",
"full_like",
"group",
,
"load",
"ones",
"ones_like",
Expand Down Expand Up @@ -577,250 +579,6 @@ async def array(
return z


async def group(
*, # Note: this is a change from v2
store: StoreLike | None = None,
overwrite: bool = False,
chunk_store: StoreLike | None = None, # not used
cache_attrs: bool | None = None, # not used, default changed
synchronizer: Any | None = None, # not used
path: str | None = None,
zarr_version: ZarrFormat | None = None, # deprecated
zarr_format: ZarrFormat | None = None,
meta_array: Any | None = None, # not used
attributes: dict[str, JSON] | None = None,
storage_options: dict[str, Any] | None = None,
) -> AsyncGroup:
"""Create a group.
Parameters
----------
store : Store or str, optional
Store or path to directory in file system.
overwrite : bool, optional
If True, delete any pre-existing data in `store` at `path` before
creating the group.
chunk_store : Store, optional
Separate storage for chunks. If not provided, `store` will be used
for storage of both chunks and metadata.
cache_attrs : bool, optional
If True (default), user attributes will be cached for attribute read
operations. If False, user attributes are reloaded from the store prior
to all attribute read operations.
synchronizer : object, optional
Array synchronizer.
path : str, optional
Group path within store.
meta_array : array-like, optional
An array instance to use for determining arrays to create and return
to users. Use `numpy.empty(())` by default.
zarr_format : {2, 3, None}, optional
The zarr format to use when saving.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.
Returns
-------
g : group
The new group.
"""

zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)

mode: AccessModeLiteral
if overwrite:
mode = "w"
else:
mode = "r+"
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

if chunk_store is not None:
warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)
if cache_attrs is not None:
warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2)
if synchronizer is not None:
warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2)
if meta_array is not None:
warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)

if attributes is None:
attributes = {}

try:
return await AsyncGroup.open(store=store_path, zarr_format=zarr_format)
except (KeyError, FileNotFoundError):
_zarr_format = zarr_format or _default_zarr_format()
return await AsyncGroup.from_store(
store=store_path,
zarr_format=_zarr_format,
overwrite=overwrite,
attributes=attributes,
)


async def create_group(
*,
store: StoreLike,
path: str | None = None,
overwrite: bool = False,
zarr_format: ZarrFormat | None = None,
attributes: dict[str, Any] | None = None,
storage_options: dict[str, Any] | None = None,
) -> AsyncGroup:
"""Create a group.
Parameters
----------
store : Store or str
Store or path to directory in file system.
path : str, optional
Group path within store.
overwrite : bool, optional
If True, pre-existing data at ``path`` will be deleted before
creating the group.
zarr_format : {2, 3, None}, optional
The zarr format to use when saving.
If no ``zarr_format`` is provided, the default format will be used.
This default can be changed by modifying the value of ``default_zarr_format``
in :mod:`zarr.core.config`.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.
Returns
-------
AsyncGroup
The new group.
"""

if zarr_format is None:
zarr_format = _default_zarr_format()

mode: Literal["a"] = "a"

store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

return await AsyncGroup.from_store(
store=store_path,
zarr_format=zarr_format,
overwrite=overwrite,
attributes=attributes,
)


async def open_group(
store: StoreLike | None = None,
*, # Note: this is a change from v2
mode: AccessModeLiteral = "a",
cache_attrs: bool | None = None, # not used, default changed
synchronizer: Any = None, # not used
path: str | None = None,
chunk_store: StoreLike | None = None, # not used
storage_options: dict[str, Any] | None = None,
zarr_version: ZarrFormat | None = None, # deprecated
zarr_format: ZarrFormat | None = None,
meta_array: Any | None = None, # not used
attributes: dict[str, JSON] | None = None,
use_consolidated: bool | str | None = None,
) -> AsyncGroup:
"""Open a group using file-mode-like semantics.
Parameters
----------
store : Store, str, or mapping, optional
Store or path to directory in file system or name of zip file.
Strings are interpreted as paths on the local file system
and used as the ``root`` argument to :class:`zarr.storage.LocalStore`.
Dictionaries are used as the ``store_dict`` argument in
:class:`zarr.storage.MemoryStore``.
By default (``store=None``) a new :class:`zarr.storage.MemoryStore`
is created.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
cache_attrs : bool, optional
If True (default), user attributes will be cached for attribute read
operations. If False, user attributes are reloaded from the store prior
to all attribute read operations.
synchronizer : object, optional
Array synchronizer.
path : str, optional
Group path within store.
chunk_store : Store or str, optional
Store or path to directory in file system or name of zip file.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.
meta_array : array-like, optional
An array instance to use for determining arrays to create and return
to users. Use `numpy.empty(())` by default.
attributes : dict
A dictionary of JSON-serializable values with user-defined attributes.
use_consolidated : bool or str, default None
Whether to use consolidated metadata.
By default, consolidated metadata is used if it's present in the
store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file
for Zarr format 2).
To explicitly require consolidated metadata, set ``use_consolidated=True``,
which will raise an exception if consolidated metadata is not found.
To explicitly *not* use consolidated metadata, set ``use_consolidated=False``,
which will fall back to using the regular, non consolidated metadata.
Zarr format 2 allowed configuring the key storing the consolidated metadata
(``.zmetadata`` by default). Specify the custom key as ``use_consolidated``
to load consolidated metadata from a non-default key.
Returns
-------
g : group
The new group.
"""

zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)

if cache_attrs is not None:
warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2)
if synchronizer is not None:
warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2)
if meta_array is not None:
warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)
if chunk_store is not None:
warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)

store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)

if attributes is None:
attributes = {}

try:
if mode in _READ_MODES:
return await AsyncGroup.open(
store_path, zarr_format=zarr_format, use_consolidated=use_consolidated
)
except (KeyError, FileNotFoundError):
pass
if mode in _CREATE_MODES:
overwrite = _infer_overwrite(mode)
_zarr_format = zarr_format or _default_zarr_format()
return await AsyncGroup.from_store(
store_path,
zarr_format=_zarr_format,
overwrite=overwrite,
attributes=attributes,
)
raise FileNotFoundError(f"Unable to find group: {store_path}")


async def create(
shape: ChunkCoords | int,
*, # Note: this is a change from v2
Expand Down
Loading

0 comments on commit 7a77fe3

Please sign in to comment.