update deps (#336)

* update deps

* update pex

* better deps

* deps

* revert torch update

* update torch dep

* torch

.pylintrc

@@ -7,9 +7,6 @@
 # pygtk.require().
 #init-hook=
 
-# Profiled execution.
-profile=no
-
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
 ignore=CVS
@@ -41,22 +38,13 @@ enable=indexing-exception,old-raise-syntax
 disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,not-context-manager,no-else-return,wrong-import-order,unnecessary-pass,logging-fstring-interpolation,logging-format-interpolation,C0330
 
 
-# Set the cache size for astng objects.
-cache-size=500
-
-
 [REPORTS]
 
 # Set the output format. Available formats are text, parseable, colorized, msvs
 # (visual studio) and html. You can also give a reporter class, eg
 # mypackage.mymodule.MyReporterClass.
 output-format=text
 
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=no
 
@@ -67,10 +55,6 @@ reports=no
 # (RP0004).
 evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (RP0004).
-comment=no
-
 # Template used to display messages. This is a python new-style format string
 # used to format the message information. See doc for all details
 #msg-template=
@@ -86,10 +70,6 @@ ignore-mixin-members=yes
 # (useful for classes with attributes dynamically set).
 ignored-classes=SQLObject
 
-# When zope mode is activated, add a predefined set of Zope acquired attributes
-# to generated-members.
-zope=no
-
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E0201 when accessed. Python regular
 # expressions are accepted.
@@ -116,17 +96,6 @@ additional-builtins=
 
 [BASIC]
 
-# Required attributes for module, separated by a comma
-required-attributes=
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=apply,input,reduce
-
-
-# Disable the report(s) with the given id(s).
-# All non-Google reports are disabled by default.
-disable-report=R0001,R0002,R0003,R0004,R0101,R0102,R0201,R0202,R0220,R0401,R0402,R0701,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,R0923
-
 # Regular expression which should only match correct module names
 module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 
@@ -196,9 +165,6 @@ ignore-long-lines=(?x)
 # else.
 single-line-if-stmt=y
 
-# List of optional constructs for which whitespace checking is disabled
-no-space-check=
-
 # Maximum number of lines in a module
 max-module-lines=99999
 
@@ -250,10 +216,6 @@ extension-pkg-whitelist=_jsonnet
 
 [CLASSES]
 
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
-
 # List of method names used to declare (i.e. assign) instance attributes.
 defining-attr-methods=__init__,__new__,setUp
 
@@ -298,31 +260,6 @@ min-public-methods=2
 max-public-methods=20
 
 
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception,StandardError,BaseException
-
-
-[AST]
-
-# Maximum line length for lambdas
-short-func-length=1
-
-# List of module members that should be marked as deprecated.
-# All of the string functions are listed in 4.1.4 Deprecated string functions
-# in the Python 2.4 docs.
-deprecated-members=string.atof,string.atoi,string.atol,string.capitalize,string.expandtabs,string.find,string.rfind,string.index,string.rindex,string.count,string.lower,string.split,string.rsplit,string.splitfields,string.join,string.joinfields,string.lstrip,string.rstrip,string.strip,string.swapcase,string.translate,string.upper,string.ljust,string.rjust,string.center,string.zfill,string.replace,sys.exitfunc
-
-
-[DOCSTRING]
-
-# List of exceptions that do not need to be mentioned in the Raises section of
-# a docstring.
-ignore-exceptions=AssertionError,NotImplementedError,StopIteration,TypeError
-
-
 
 [TOKENS]
 

Makefile

@@ -16,10 +16,10 @@ black: ## [Local development] Auto-format python code using black
 build-pex:
 	python3 -m venv .pexing
 	. .pexing/bin/activate && python -m pip install -U pip && python -m pip install pex
-	. .pexing/bin/activate && python -m pex --layout packed  -f https://download.pytorch.org/whl/cu113/torch_stable.html setuptools gcsfs charset-normalizer==2.1.1 s3fs pyspark==3.2.0 torch==1.10.2+cu113 torchvision==0.11.3+cu113 . -o clip_retrieval.pex -v
+	. .pexing/bin/activate && python -m pex --layout packed -f https://download.pytorch.org/whl/cu113/torch_stable.html setuptools gcsfs charset-normalizer s3fs pyspark torch==1.10.2+cu113 torchvision==0.11.3+cu113 . -o clip_retrieval.pex -v
 	rm -rf .pexing
-	tar czf clip_retrieval_torch.tgz clip_retrieval.pex/.deps/torch-1.10.2+cu113-cp38-cp38-linux_x86_64.whl
-	tar czf clip_retrieval.tgz --exclude clip_retrieval.pex/.deps/torch-1.10.2+cu113-cp38-cp38-linux_x86_64.whl clip_retrieval.pex
+	tar czf clip_retrieval_torch.tgz clip_retrieval.pex/.deps/torch-*
+	tar czf clip_retrieval.tgz --exclude clip_retrieval.pex/.deps/torch-* clip_retrieval.pex
 
 venv-lint-test: ## [Continuous integration]
 	python3 -m venv .env && . .env/bin/activate && make install install-dev lint test && rm -rf .env

clip_retrieval/clip_client.py

@@ -149,4 +149,5 @@ def __search_knn_api__(
                     "num_result_ids": self.num_images,
                 }
             ),
+            timeout=3600,
         ).json()

clip_retrieval/clip_inference/reader.py

@@ -35,7 +35,10 @@ def folder_to_keys(folder, enable_text=True, enable_image=True, enable_metadata=
         metadata_files = {metadata_file.relative_to(path).as_posix(): metadata_file for metadata_file in metadata_files}
 
     keys = None
-    join = lambda new_set: new_set & keys if keys is not None else new_set
+
+    def join(new_set):
+        return new_set & keys if keys is not None else new_set
+
     if enable_text:
         keys = join(text_files.keys())
     elif enable_image:
@@ -137,7 +140,9 @@ def create_webdataset(
     urls = input_sampler(urls)
 
     dataset = wds.WebDataset(urls, cache_dir=cache_path, cache_size=10**10, handler=wds.handlers.warn_and_continue)
-    tokenizer = lambda text: clip.tokenize([text], truncate=True)[0]
+
+    def tokenizer(text):
+        return clip.tokenize([text], truncate=True)[0]
 
     def filter_dataset(item):
         if enable_text and caption_key not in item:

requirements-test.txt

@@ -1,9 +1,9 @@
 img2dataset
-black==22.3.0
+black==23.12.1
 mypy==1.8.0
-pylint==2.13.4
+pylint==3.0.3
 pytest-cov==4.1.0
-pytest-xdist==2.5.0
+pytest-xdist==3.5.0
 pytest==7.4.4
 types-setuptools
 types-requests

requirements.txt

@@ -6,23 +6,23 @@ torch>=1.7.1,<2
 torchvision>=0.10.1,<2
 numpy>=1.19.5,<2
 faiss-cpu>=1.7.2,<2
-flask>=2.0.3,<3
+flask>=3.0.0,<4
 flask_restful>=0.3.9,<1
-flask_cors>=3.0.10,<4
+flask_cors>=4.0.0,<5
 pandas>=1.1.5,<2
-pyarrow>=6.0.1,<8
+pyarrow>=6.0.1,<13
 autofaiss>=2.9.6,<3
 webdataset>=0.2,<0.3
 h5py>=3.1.0,<4
 prometheus-client>=0.13.1,<1
 fsspec
 sentence-transformers>=2.2.0,<3
-wandb>=0.12.10,<0.13
+wandb>=0.12.0,<0.17
 open-clip-torch>=2.0.0,<3.0.0
 requests>=2.27.1,<3
 aiohttp>=3.8.1,<4
 multilingual-clip>=1.0.10,<2
 transformers
 urllib3<2
-scipy<1.11.5
+scipy<1.12
 all_clip<2

setup.py

@@ -10,7 +10,7 @@
 
     def package_files(directory):
         paths = []
-        for (path, _, filenames) in os.walk(directory):
+        for path, _, filenames in os.walk(directory):
             for filename in filenames:
                 paths.append(os.path.join("..", path, filename))
         return paths

tests/test_clip_inference/test_worker.py

@@ -11,7 +11,6 @@ def test_worker():
     input_dataset = os.path.join(current_folder, "test_images")
 
     with tempfile.TemporaryDirectory() as tmpdir:
-
         worker(
             tasks=[0, 1],
             input_dataset=input_dataset,