From 2c2b84af4a32cf1502eae60709854987f7b21669 Mon Sep 17 00:00:00 2001 From: Anatoly Belikov Date: Thu, 4 Apr 2024 11:12:21 +0300 Subject: [PATCH 1/4] throw excpetion if lpw doen't match with model --- multigen/pipes.py | 9 ++++++++- multigen/sessions.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/multigen/pipes.py b/multigen/pipes.py index e67f503..27acb82 100755 --- a/multigen/pipes.py +++ b/multigen/pipes.py @@ -102,6 +102,10 @@ def __init__(self, model_id: str, except ImportError as e: logging.warning("xformers not found, can't use efficient attention") + if hasattr(self.pipe, 'text_encoder_2'): + if self.pipe.text_encoder_2 is None: + raise AttributeError("text_encoder_2 is None") + @property def scheduler(self): return self._scheduler @@ -188,7 +192,10 @@ def __init__(self, model_id: str, super().__init__(model_id=model_id, pipe=pipe, **args) else: #StableDiffusionKDiffusionPipeline - super().__init__(model_id=model_id, pipe=pipe, custom_pipeline="lpw_stable_diffusion", **args) + try: + super().__init__(model_id=model_id, pipe=pipe, custom_pipeline="lpw_stable_diffusion_xl", **args) + except AttributeError as e: + super().__init__(model_id=model_id, pipe=pipe, custom_pipeline="lpw_stable_diffusion", **args) def setup(self, width=768, height=768, guidance_scale=7.5, **args): super().setup(**args) diff --git a/multigen/sessions.py b/multigen/sessions.py index 540667f..5e8945d 100755 --- a/multigen/sessions.py +++ b/multigen/sessions.py @@ -10,7 +10,7 @@ class GenSession: def __init__(self, session_dir, pipe, config: Cfgen, name_prefix=""): self.session_dir = session_dir self.pipe = pipe - self.model_id = pipe._model_id + self.model_id = pipe.model_id self.confg = config self.last_conf = None self.name_prefix = name_prefix From fd3ddd48fea842dd83b8d41f242df133dac708fb Mon Sep 17 00:00:00 2001 From: Anatoly Belikov Date: Thu, 4 Apr 2024 11:16:33 +0300 Subject: [PATCH 2/4] load lora --- multigen/pipes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/multigen/pipes.py b/multigen/pipes.py index 27acb82..d3eb069 100755 --- a/multigen/pipes.py +++ b/multigen/pipes.py @@ -127,6 +127,10 @@ def try_set_scheduler(self, inputs): self._scheduler = sch_set inputs.pop('scheduler') + def load_lora(self, path, multiplier=1.0): + self.pipe.load_lora_weights(path) + self.pipe_params['cross_attention_kwargs'] = {"scale": multiplier} + def add_hypernet(self, path, multiplier=None): from . hypernet import add_hypernet, clear_hypernets, Hypernetwork hypernetwork = Hypernetwork() From 70ac55d5bd678ecd8ef2f0987e8c5aaf7654a50a Mon Sep 17 00:00:00 2001 From: Anatoly Belikov Date: Thu, 4 Apr 2024 13:37:09 +0300 Subject: [PATCH 3/4] expand user path in model_id --- multigen/pipes.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/multigen/pipes.py b/multigen/pipes.py index d3eb069..9edfefa 100755 --- a/multigen/pipes.py +++ b/multigen/pipes.py @@ -6,6 +6,7 @@ from PIL import Image, ImageFilter import cv2 +import os import copy import numpy as np from typing import Optional, Type @@ -57,7 +58,7 @@ def __init__(self, model_id: str, self.pipe = pipe self._scheduler = None self._hypernets = [] - self._model_id = model_id + self._model_id = os.path.expanduser(model_id) self.pipe_params = dict() # Creating a stable diffusion pipeine args = {**args} @@ -72,24 +73,24 @@ def __init__(self, model_id: str, constructor_args['controlnet'] = args['controlnet'] if sd_pipe_class is None: - if model_id.endswith('.safetensors'): + if self.model_id.endswith('.safetensors'): try: - self.pipe = StableDiffusionPipeline.from_single_file(model_id, **args) + self.pipe = StableDiffusionPipeline.from_single_file(self.model_id, **args) except TypeError as e: - self.pipe = StableDiffusionXLPipeline.from_single_file(model_id, **args) + self.pipe = StableDiffusionXLPipeline.from_single_file(self.model_id, **args) else: # we can't use specific class, because we dont know if it is sdxl - self.pipe = DiffusionPipeline.from_pretrained(model_id, **args) + self.pipe = DiffusionPipeline.from_pretrained(self.model_id, **args) if 'custom_pipeline' not in args: # create correct class if custom_pipeline is not specified # at this stage we know that the model is sdxl or sd self.pipe = self.from_pipe(self.pipe, **constructor_args) else: - if model_id.endswith('.safetensors'): - self.pipe = sd_pipe_class.from_single_file(model_id, **args) + if self.model_id.endswith('.safetensors'): + self.pipe = sd_pipe_class.from_single_file(self.model_id, **args) else: - self.pipe = sd_pipe_class.from_pretrained(model_id, **args) + self.pipe = sd_pipe_class.from_pretrained(self.model_id, **args) self.pipe.to(device) # self.pipe.enable_attention_slicing() # self.pipe.enable_vae_slicing() From 7d592ebe31e2990ad1fc0b38f1615565fb821828 Mon Sep 17 00:00:00 2001 From: Anatoly Belikov Date: Thu, 4 Apr 2024 13:37:38 +0300 Subject: [PATCH 4/4] lora example --- examples/prompt2im.py | 2 +- examples/prompt2im_lora.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 examples/prompt2im_lora.py diff --git a/examples/prompt2im.py b/examples/prompt2im.py index 5517187..908618b 100755 --- a/examples/prompt2im.py +++ b/examples/prompt2im.py @@ -15,7 +15,7 @@ ["surrealism", "impressionism", "high tech", "cyberpunk"]] -pipe = Prompt2ImPipe(model_dir+model_id) +pipe = Prompt2ImPipe(model_dir+model_id, lpw=True) pipe.setup(width=768, height=768) gs = GenSession("./_projects/biolab", pipe, Cfgen(prompt, nprompt)) gs.gen_sess(add_count=10) diff --git a/examples/prompt2im_lora.py b/examples/prompt2im_lora.py new file mode 100755 index 0000000..d96b64d --- /dev/null +++ b/examples/prompt2im_lora.py @@ -0,0 +1,21 @@ +from multigen.prompting import Cfgen +from multigen.sessions import GenSession +from multigen.pipes import Prompt2ImPipe +from diffusers import DPMSolverMultistepScheduler + + +prompt = "digitalben is hugging his lamb, farm in background, animal photography, big sheep, full-height photo, best quality, camera low, camera close, best quality, amazing,ultra high res, masterpiece, round glasses, long hair" + + +negative_prompt = "deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, (mutated hands and fingers:1.2), disconnected limbs, mutation, mutated, ugly, disgusting, blurry, amputation" + + +model_name = '~/models/SDXL/juggernautXL_v8Rundiffusion/' + +path_lora = 'checkpoint-20500' +pipe = Prompt2ImPipe(model_name, lpw=True) +pipe.setup(width=1024, height=1024, guidance_scale=4, clip_skip=1) +pipe.load_lora(path_lora, 0.9) + +gs = GenSession("./benben", pipe, Cfgen(prompt, negative_prompt, seeds=[1877029948 + i for i in range(10)])) +gs.gen_sess(add_count=10)