Skip to content

Commit

Permalink
remove multi_writer
Browse files Browse the repository at this point in the history
  • Loading branch information
zpz committed Jul 4, 2021
1 parent a23f55e commit 2fbb830
Showing 1 changed file with 6 additions and 20 deletions.
26 changes: 6 additions & 20 deletions src/biglist/_biglist.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,20 +306,7 @@ def new(cls,

return obj

def __init__(self,
path: Union[str, Path, Upath],
*,
multi_writers: bool = True):
'''
`multi_writers`: if `true`, this Biglist is possibly being written to
(i.e. appended to) by other workers besides the current one.
The default value, `True`, is a conservative setting---it does no harm
other than slowing down random access. Setting it to `False` only when
you are sure the current object is the only one that may be writing
to the Biglist at this time.
TODO: find a better name for `multi_writers`.
'''
def __init__(self, path: Union[str, Path, Upath]):
if isinstance(path, str):
path = Path(path)
if isinstance(path, Path):
Expand All @@ -328,7 +315,6 @@ def __init__(self,
# `Upath` protocol.
self.path = path

self._multi_writers = multi_writers
self._data_files: Optional[list] = None

self._read_buffer: Optional[list] = None
Expand Down Expand Up @@ -402,7 +388,7 @@ def __getitem__(self, idx: int): # type: ignore
if n1 <= idx < n2:
return self._read_buffer[idx - n1] # type: ignore

datafiles = self.get_data_files()
datafiles = self.get_data_files(lazy=True)
length = sum(l for _, l in datafiles)
idx = range(length + len(self._append_buffer))[idx]

Expand Down Expand Up @@ -466,7 +452,7 @@ def __iter__(self):
yield from self._append_buffer

def __len__(self) -> int:
z = self.get_data_files()
z = self.get_data_files(lazy=True)
return sum(k for _, k in z) + len(self._append_buffer)

def append(self, x) -> None:
Expand Down Expand Up @@ -519,7 +505,7 @@ def file_iter_stat(self, task_id: str) -> FileIterStat:

def file_view(self, file: Union[Upath, int]) -> FileView:
if isinstance(file, int):
datafiles = self.get_data_files()
datafiles = self.get_data_files(lazy=True)
file = self._data_dir / datafiles[file][0]
return FileView(file, self.load_data_file) # type: ignore

Expand Down Expand Up @@ -582,8 +568,8 @@ def flush(self):
self._flush()
self._file_dumper.wait()

def get_data_files(self) -> list:
if not self._multi_writers and self._data_files is not None:
def get_data_files(self, lazy: bool = False) -> list:
if lazy and self._data_files is not None:
return self._data_files

if self._data_info_file.exists():
Expand Down

0 comments on commit 2fbb830

Please sign in to comment.