Merge pull request #49 from noskill/fixxl

add lora example

examples/prompt2im.py

@@ -15,7 +15,7 @@
           ["surrealism", "impressionism", "high tech", "cyberpunk"]]
 
 
-pipe = Prompt2ImPipe(model_dir+model_id)
+pipe = Prompt2ImPipe(model_dir+model_id, lpw=True)
 pipe.setup(width=768, height=768)
 gs = GenSession("./_projects/biolab", pipe, Cfgen(prompt, nprompt))
 gs.gen_sess(add_count=10)

examples/prompt2im_lora.py

@@ -0,0 +1,21 @@
+from multigen.prompting import Cfgen
+from multigen.sessions import GenSession
+from multigen.pipes import Prompt2ImPipe
+from diffusers import DPMSolverMultistepScheduler
+
+
+prompt = "digitalben is hugging his lamb, farm in background, animal photography, big sheep, full-height photo, best quality, camera low, camera close, best quality, amazing,ultra high res, masterpiece, round glasses, long hair"
+
+
+negative_prompt = "deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, (mutated hands and fingers:1.2), disconnected limbs, mutation, mutated, ugly, disgusting, blurry, amputation" 
+
+
+model_name = '~/models/SDXL/juggernautXL_v8Rundiffusion/'
+
+path_lora = 'checkpoint-20500'
+pipe = Prompt2ImPipe(model_name, lpw=True)
+pipe.setup(width=1024, height=1024, guidance_scale=4, clip_skip=1)
+pipe.load_lora(path_lora, 0.9)
+
+gs = GenSession("./benben", pipe, Cfgen(prompt, negative_prompt, seeds=[1877029948 + i for i in range(10)]))
+gs.gen_sess(add_count=10)

multigen/pipes.py

@@ -6,6 +6,7 @@
 
 from PIL import Image, ImageFilter
 import cv2
+import os
 import copy
 import numpy as np
 from typing import Optional, Type
@@ -57,7 +58,7 @@ def __init__(self, model_id: str,
         self.pipe = pipe
         self._scheduler = None
         self._hypernets = []
-        self._model_id = model_id
+        self._model_id = os.path.expanduser(model_id)
         self.pipe_params = dict()
         # Creating a stable diffusion pipeine
         args = {**args}
@@ -72,24 +73,24 @@ def __init__(self, model_id: str,
                 constructor_args['controlnet'] = args['controlnet']
 
             if sd_pipe_class is None:
-                if model_id.endswith('.safetensors'):
+                if self.model_id.endswith('.safetensors'):
                     try:
-                        self.pipe = StableDiffusionPipeline.from_single_file(model_id, **args)
+                        self.pipe = StableDiffusionPipeline.from_single_file(self.model_id, **args)
                     except TypeError as e:
-                        self.pipe = StableDiffusionXLPipeline.from_single_file(model_id, **args)
+                        self.pipe = StableDiffusionXLPipeline.from_single_file(self.model_id, **args)
                 else:
                     # we can't use specific class, because we dont know if it is sdxl
-                    self.pipe = DiffusionPipeline.from_pretrained(model_id, **args)
+                    self.pipe = DiffusionPipeline.from_pretrained(self.model_id, **args)
                     if 'custom_pipeline' not in args:
                         # create correct class if custom_pipeline is not specified
                         # at this stage we know that the model is sdxl or sd
                         self.pipe = self.from_pipe(self.pipe, **constructor_args)
 
             else:
-                if model_id.endswith('.safetensors'):
-                    self.pipe = sd_pipe_class.from_single_file(model_id, **args)
+                if self.model_id.endswith('.safetensors'):
+                    self.pipe = sd_pipe_class.from_single_file(self.model_id, **args)
                 else:
-                    self.pipe = sd_pipe_class.from_pretrained(model_id, **args)
+                    self.pipe = sd_pipe_class.from_pretrained(self.model_id, **args)
         self.pipe.to(device)
         # self.pipe.enable_attention_slicing()
         # self.pipe.enable_vae_slicing()
@@ -102,6 +103,10 @@ def __init__(self, model_id: str,
         except ImportError as e:
             logging.warning("xformers not found, can't use efficient attention")
 
+        if hasattr(self.pipe, 'text_encoder_2'):
+            if self.pipe.text_encoder_2 is None:
+                raise AttributeError("text_encoder_2 is None")
+
     @property
     def scheduler(self):
         return self._scheduler
@@ -123,6 +128,10 @@ def try_set_scheduler(self, inputs):
                 self._scheduler = sch_set
             inputs.pop('scheduler')
 
+    def load_lora(self, path, multiplier=1.0):
+        self.pipe.load_lora_weights(path)
+        self.pipe_params['cross_attention_kwargs'] = {"scale": multiplier}
+
     def add_hypernet(self, path, multiplier=None):
         from . hypernet import add_hypernet, clear_hypernets, Hypernetwork
         hypernetwork = Hypernetwork()
@@ -188,7 +197,10 @@ def __init__(self, model_id: str,
             super().__init__(model_id=model_id, pipe=pipe, **args)
         else:
             #StableDiffusionKDiffusionPipeline
-            super().__init__(model_id=model_id, pipe=pipe, custom_pipeline="lpw_stable_diffusion", **args)
+            try:
+                super().__init__(model_id=model_id, pipe=pipe, custom_pipeline="lpw_stable_diffusion_xl", **args)
+            except AttributeError as e:
+                super().__init__(model_id=model_id, pipe=pipe, custom_pipeline="lpw_stable_diffusion", **args)
 
     def setup(self, width=768, height=768, guidance_scale=7.5, **args):
         super().setup(**args)

multigen/sessions.py

@@ -10,7 +10,7 @@ class GenSession:
     def __init__(self, session_dir, pipe, config: Cfgen, name_prefix=""):
         self.session_dir = session_dir
         self.pipe = pipe
-        self.model_id = pipe._model_id
+        self.model_id = pipe.model_id
         self.confg = config
         self.last_conf = None
         self.name_prefix = name_prefix